1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(O&: llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(i: 0);
86 if (Arg0Type != FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(GV: F);
91 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 i: F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(BitWidth: 32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(GV: F);
107 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(GV: F);
120 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 8))
135 return false;
136
137 rename(GV: F);
138 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 16))
153 return false;
154
155 rename(GV: F);
156 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(GV: F);
166 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(i: 1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(GV: F);
176 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front(Prefix: "avx."))
187 return (Name.starts_with(Prefix: "blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with(Prefix: "movnt.") || // Added in 3.2
192 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
193 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
194 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 3.5
195 Name.starts_with(Prefix: "vbroadcastf128") || // Added in 4.0
196 Name.starts_with(Prefix: "vextractf128.") || // Added in 3.7
197 Name.starts_with(Prefix: "vinsertf128.") || // Added in 3.7
198 Name.starts_with(Prefix: "vperm2f128.") || // Added in 6.0
199 Name.starts_with(Prefix: "vpermil.")); // Added in 3.1
200
201 if (Name.consume_front(Prefix: "avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
204 Name.starts_with(Prefix: "padds.") || // Added in 8.0
205 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
206 Name.starts_with(Prefix: "pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.8
209 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
210 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
211 Name.starts_with(Prefix: "pmax") || // Added in 3.9
212 Name.starts_with(Prefix: "pmin") || // Added in 3.9
213 Name.starts_with(Prefix: "pmovsx") || // Added in 3.9
214 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
218 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
219 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
220 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
221 Name.starts_with(Prefix: "vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front(Prefix: "avx512.")) {
228 if (Name.consume_front(Prefix: "mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with(Prefix: "add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with(Prefix: "and.") || // Added in 3.9
232 Name.starts_with(Prefix: "andn.") || // Added in 3.9
233 Name.starts_with(Prefix: "broadcast.s") || // Added in 3.9
234 Name.starts_with(Prefix: "broadcastf32x4.") || // Added in 6.0
235 Name.starts_with(Prefix: "broadcastf32x8.") || // Added in 6.0
236 Name.starts_with(Prefix: "broadcastf64x2.") || // Added in 6.0
237 Name.starts_with(Prefix: "broadcastf64x4.") || // Added in 6.0
238 Name.starts_with(Prefix: "broadcasti32x4.") || // Added in 6.0
239 Name.starts_with(Prefix: "broadcasti32x8.") || // Added in 6.0
240 Name.starts_with(Prefix: "broadcasti64x2.") || // Added in 6.0
241 Name.starts_with(Prefix: "broadcasti64x4.") || // Added in 6.0
242 Name.starts_with(Prefix: "cmp.b") || // Added in 5.0
243 Name.starts_with(Prefix: "cmp.d") || // Added in 5.0
244 Name.starts_with(Prefix: "cmp.q") || // Added in 5.0
245 Name.starts_with(Prefix: "cmp.w") || // Added in 5.0
246 Name.starts_with(Prefix: "compress.b") || // Added in 9.0
247 Name.starts_with(Prefix: "compress.d") || // Added in 9.0
248 Name.starts_with(Prefix: "compress.p") || // Added in 9.0
249 Name.starts_with(Prefix: "compress.q") || // Added in 9.0
250 Name.starts_with(Prefix: "compress.store.") || // Added in 7.0
251 Name.starts_with(Prefix: "compress.w") || // Added in 9.0
252 Name.starts_with(Prefix: "conflict.") || // Added in 9.0
253 Name.starts_with(Prefix: "cvtdq2pd.") || // Added in 4.0
254 Name.starts_with(Prefix: "cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with(Prefix: "cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with(Prefix: "cvtudq2pd.") || // Added in 4.0
266 Name.starts_with(Prefix: "cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with(Prefix: "cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with(Prefix: "dbpsadbw.") || // Added in 7.0
271 Name.starts_with(Prefix: "div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with(Prefix: "expand.b") || // Added in 9.0
273 Name.starts_with(Prefix: "expand.d") || // Added in 9.0
274 Name.starts_with(Prefix: "expand.load.") || // Added in 7.0
275 Name.starts_with(Prefix: "expand.p") || // Added in 9.0
276 Name.starts_with(Prefix: "expand.q") || // Added in 9.0
277 Name.starts_with(Prefix: "expand.w") || // Added in 9.0
278 Name.starts_with(Prefix: "fpclass.p") || // Added in 7.0
279 Name.starts_with(Prefix: "insert") || // Added in 4.0
280 Name.starts_with(Prefix: "load.") || // Added in 3.9
281 Name.starts_with(Prefix: "loadu.") || // Added in 3.9
282 Name.starts_with(Prefix: "lzcnt.") || // Added in 5.0
283 Name.starts_with(Prefix: "max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with(Prefix: "min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with(Prefix: "movddup") || // Added in 3.9
286 Name.starts_with(Prefix: "move.s") || // Added in 4.0
287 Name.starts_with(Prefix: "movshdup") || // Added in 3.9
288 Name.starts_with(Prefix: "movsldup") || // Added in 3.9
289 Name.starts_with(Prefix: "mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with(Prefix: "or.") || // Added in 3.9
291 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
292 Name.starts_with(Prefix: "packssdw.") || // Added in 5.0
293 Name.starts_with(Prefix: "packsswb.") || // Added in 5.0
294 Name.starts_with(Prefix: "packusdw.") || // Added in 5.0
295 Name.starts_with(Prefix: "packuswb.") || // Added in 5.0
296 Name.starts_with(Prefix: "padd.") || // Added in 4.0
297 Name.starts_with(Prefix: "padds.") || // Added in 8.0
298 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
299 Name.starts_with(Prefix: "palignr.") || // Added in 3.9
300 Name.starts_with(Prefix: "pand.") || // Added in 3.9
301 Name.starts_with(Prefix: "pandn.") || // Added in 3.9
302 Name.starts_with(Prefix: "pavg") || // Added in 6.0
303 Name.starts_with(Prefix: "pbroadcast") || // Added in 6.0
304 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.9
305 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.9
306 Name.starts_with(Prefix: "perm.df.") || // Added in 3.9
307 Name.starts_with(Prefix: "perm.di.") || // Added in 3.9
308 Name.starts_with(Prefix: "permvar.") || // Added in 7.0
309 Name.starts_with(Prefix: "pmaddubs.w.") || // Added in 7.0
310 Name.starts_with(Prefix: "pmaddw.d.") || // Added in 7.0
311 Name.starts_with(Prefix: "pmax") || // Added in 4.0
312 Name.starts_with(Prefix: "pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with(Prefix: "pmovsx") || // Added in 4.0
318 Name.starts_with(Prefix: "pmovzx") || // Added in 4.0
319 Name.starts_with(Prefix: "pmul.dq.") || // Added in 4.0
320 Name.starts_with(Prefix: "pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with(Prefix: "pmulh.w.") || // Added in 7.0
322 Name.starts_with(Prefix: "pmulhu.w.") || // Added in 7.0
323 Name.starts_with(Prefix: "pmull.") || // Added in 4.0
324 Name.starts_with(Prefix: "pmultishift.qb.") || // Added in 8.0
325 Name.starts_with(Prefix: "pmulu.dq.") || // Added in 4.0
326 Name.starts_with(Prefix: "por.") || // Added in 3.9
327 Name.starts_with(Prefix: "prol.") || // Added in 8.0
328 Name.starts_with(Prefix: "prolv.") || // Added in 8.0
329 Name.starts_with(Prefix: "pror.") || // Added in 8.0
330 Name.starts_with(Prefix: "prorv.") || // Added in 8.0
331 Name.starts_with(Prefix: "pshuf.b.") || // Added in 4.0
332 Name.starts_with(Prefix: "pshuf.d.") || // Added in 3.9
333 Name.starts_with(Prefix: "pshufh.w.") || // Added in 3.9
334 Name.starts_with(Prefix: "pshufl.w.") || // Added in 3.9
335 Name.starts_with(Prefix: "psll.d") || // Added in 4.0
336 Name.starts_with(Prefix: "psll.q") || // Added in 4.0
337 Name.starts_with(Prefix: "psll.w") || // Added in 4.0
338 Name.starts_with(Prefix: "pslli") || // Added in 4.0
339 Name.starts_with(Prefix: "psllv") || // Added in 4.0
340 Name.starts_with(Prefix: "psra.d") || // Added in 4.0
341 Name.starts_with(Prefix: "psra.q") || // Added in 4.0
342 Name.starts_with(Prefix: "psra.w") || // Added in 4.0
343 Name.starts_with(Prefix: "psrai") || // Added in 4.0
344 Name.starts_with(Prefix: "psrav") || // Added in 4.0
345 Name.starts_with(Prefix: "psrl.d") || // Added in 4.0
346 Name.starts_with(Prefix: "psrl.q") || // Added in 4.0
347 Name.starts_with(Prefix: "psrl.w") || // Added in 4.0
348 Name.starts_with(Prefix: "psrli") || // Added in 4.0
349 Name.starts_with(Prefix: "psrlv") || // Added in 4.0
350 Name.starts_with(Prefix: "psub.") || // Added in 4.0
351 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
352 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
353 Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
354 Name.starts_with(Prefix: "punpckh") || // Added in 3.9
355 Name.starts_with(Prefix: "punpckl") || // Added in 3.9
356 Name.starts_with(Prefix: "pxor.") || // Added in 3.9
357 Name.starts_with(Prefix: "shuf.f") || // Added in 6.0
358 Name.starts_with(Prefix: "shuf.i") || // Added in 6.0
359 Name.starts_with(Prefix: "shuf.p") || // Added in 4.0
360 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
361 Name.starts_with(Prefix: "store.b.") || // Added in 3.9
362 Name.starts_with(Prefix: "store.d.") || // Added in 3.9
363 Name.starts_with(Prefix: "store.p") || // Added in 3.9
364 Name.starts_with(Prefix: "store.q.") || // Added in 3.9
365 Name.starts_with(Prefix: "store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
368 Name.starts_with(Prefix: "sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with(Prefix: "ucmp.") || // Added in 5.0
370 Name.starts_with(Prefix: "unpckh.") || // Added in 3.9
371 Name.starts_with(Prefix: "unpckl.") || // Added in 3.9
372 Name.starts_with(Prefix: "valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with(Prefix: "vextract") || // Added in 4.0
376 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
377 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
378 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
379 Name.starts_with(Prefix: "vfnmsub.") || // Added in 7.0
380 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
381 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
382 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
383 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
384 Name.starts_with(Prefix: "vpermi2var.") || // Added in 7.0
385 Name.starts_with(Prefix: "vpermil.p") || // Added in 3.9
386 Name.starts_with(Prefix: "vpermilvar.") || // Added in 4.0
387 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
388 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
389 Name.starts_with(Prefix: "vpshld.") || // Added in 7.0
390 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
391 Name.starts_with(Prefix: "vpshrd.") || // Added in 7.0
392 Name.starts_with(Prefix: "vpshrdv.") || // Added in 8.0
393 Name.starts_with(Prefix: "vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with(Prefix: "xor.")); // Added in 3.9
395
396 if (Name.consume_front(Prefix: "mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
399 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
400 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
401 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
402 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front(Prefix: "maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
407 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
408 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
409 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
410 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
411 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
412 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
413 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
414 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
415 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
416 Name.starts_with(Prefix: "vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with(Prefix: "broadcastm") || // Added in 6.0
423 Name.starts_with(Prefix: "cmp.p") || // Added in 12.0
424 Name.starts_with(Prefix: "cvtb2mask.") || // Added in 7.0
425 Name.starts_with(Prefix: "cvtd2mask.") || // Added in 7.0
426 Name.starts_with(Prefix: "cvtmask2") || // Added in 5.0
427 Name.starts_with(Prefix: "cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with(Prefix: "cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with(Prefix: "kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with(Prefix: "padds.") || // Added in 8.0
440 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.9
441 Name.starts_with(Prefix: "prol") || // Added in 8.0
442 Name.starts_with(Prefix: "pror") || // Added in 8.0
443 Name.starts_with(Prefix: "psll.dq") || // Added in 3.9
444 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.9
445 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
446 Name.starts_with(Prefix: "ptestm") || // Added in 6.0
447 Name.starts_with(Prefix: "ptestnm") || // Added in 6.0
448 Name.starts_with(Prefix: "storent.") || // Added in 3.9
449 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 7.0
450 Name.starts_with(Prefix: "vpshld.") || // Added in 8.0
451 Name.starts_with(Prefix: "vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front(Prefix: "fma."))
455 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
456 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
457 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
458 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
459 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front(Prefix: "fma4."))
462 return Name.starts_with(Prefix: "vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front(Prefix: "sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front(Prefix: "sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with(Prefix: "padds.") || // Added in 8.0
486 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
487 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
488 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with(Prefix: "pshuf") || // Added in 3.9
495 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
496 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
497 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
498 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
499 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front(Prefix: "sse41."))
506 return (Name.starts_with(Prefix: "blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with(Prefix: "pmovsx") || // Added in 3.8
518 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front(Prefix: "sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front(Prefix: "sse4a."))
525 return Name.starts_with(Prefix: "movnt."); // Added in 3.9
526
527 if (Name.consume_front(Prefix: "ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front(Prefix: "xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with(Prefix: "vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with(Prefix: "vprot")); // Added in 8.0
537
538 if (Name.consume_front(Prefix: "bmi."))
539 return (Name.starts_with(Prefix: "pdep.") || // Added in 23.0
540 Name.starts_with(Prefix: "pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with(Prefix: "vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front(Prefix: "x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(GV: F);
568 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
569 id: Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front(Prefix: "sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case(S: "c", Value: Intrinsic::x86_sse41_ptestc)
579 .Case(S: "z", Value: Intrinsic::x86_sse41_ptestz)
580 .Case(S: "nzc", Value: Intrinsic::x86_sse41_ptestnzc)
581 .Default(Value: Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, IID: ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case(S: "sse41.insertps", Value: Intrinsic::x86_sse41_insertps)
594 .Case(S: "sse41.dppd", Value: Intrinsic::x86_sse41_dppd)
595 .Case(S: "sse41.dpps", Value: Intrinsic::x86_sse41_dpps)
596 .Case(S: "sse41.mpsadbw", Value: Intrinsic::x86_sse41_mpsadbw)
597 .Case(S: "avx.dp.ps.256", Value: Intrinsic::x86_avx_dp_ps_256)
598 .Case(S: "avx2.mpsadbw", Value: Intrinsic::x86_avx2_mpsadbw)
599 .Default(Value: Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, IID: ID, NewFn);
602
603 if (Name.consume_front(Prefix: "avx512.")) {
604 if (Name.consume_front(Prefix: "mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case(S: "pd.128", Value: Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case(S: "pd.256", Value: Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case(S: "pd.512", Value: Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case(S: "ps.128", Value: Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case(S: "ps.256", Value: Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case(S: "ps.512", Value: Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Value: Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, IID: ID, NewFn);
616 } else if (Name.starts_with(Prefix: "vpdpbusd.") ||
617 Name.starts_with(Prefix: "vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case(S: "vpdpbusd.128", Value: Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case(S: "vpdpbusd.256", Value: Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case(S: "vpdpbusd.512", Value: Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case(S: "vpdpbusds.128", Value: Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case(S: "vpdpbusds.256", Value: Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case(S: "vpdpbusds.512", Value: Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Value: Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
629 } else if (Name.starts_with(Prefix: "vpdpwssd.") ||
630 Name.starts_with(Prefix: "vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case(S: "vpdpwssd.128", Value: Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case(S: "vpdpwssd.256", Value: Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case(S: "vpdpwssd.512", Value: Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case(S: "vpdpwssds.128", Value: Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case(S: "vpdpwssds.256", Value: Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case(S: "vpdpwssds.512", Value: Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Value: Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front(Prefix: "avx2.")) {
647 if (Name.consume_front(Prefix: "vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case(S: "ssd.128", Value: Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case(S: "ssd.256", Value: Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case(S: "ssds.128", Value: Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case(S: "ssds.256", Value: Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Value: Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
665 } else if (Name.consume_front(Prefix: "vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case(S: "usd.128", Value: Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case(S: "usd.256", Value: Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case(S: "usds.128", Value: Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case(S: "usds.256", Value: Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Value: Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front(Prefix: "avx10.")) {
688 if (Name.consume_front(Prefix: "vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case(S: "ssd.512", Value: Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case(S: "ssds.512", Value: Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Value: Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
700 } else if (Name.consume_front(Prefix: "vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case(S: "usd.512", Value: Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case(S: "usds.512", Value: Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Value: Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front(Prefix: "avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case(S: "cvtne2ps2bf16.128",
719 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case(S: "cvtne2ps2bf16.256",
721 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case(S: "cvtne2ps2bf16.512",
723 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case(S: "mask.cvtneps2bf16.128",
725 Value: Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case(S: "cvtneps2bf16.256",
727 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case(S: "cvtneps2bf16.512",
729 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Value: Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, IID: ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case(S: "dpbf16ps.128", Value: Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case(S: "dpbf16ps.256", Value: Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case(S: "dpbf16ps.512", Value: Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Value: Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, IID: ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front(Prefix: "xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with(Prefix: "vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(i: 2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case(S: "vfrcz.ss", Value: Intrinsic::x86_xop_vfrcz_ss)
767 .Case(S: "vfrcz.sd", Value: Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Value: Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(GV: F);
772 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
780 id: Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with(Prefix: "rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 M: F->getParent(), id: Intrinsic::bitreverse, OverloadTys: F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front(Prefix: "neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front(Prefix: "bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases(CaseStrings: {"v2f32.v8i8", "v4f32.v16i8"},
816 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Value: Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 ._M_elems: {F->getReturnType(),
826 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), NumElts: OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front(Prefix: "bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(Suffix: ".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case(S: "mla",
842 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case(S: "lalb",
845 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case(S: "lalt",
848 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Value: Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith(S: "vclz.", Value: Intrinsic::ctlz)
869 .StartsWith(S: "vcnt.", Value: Intrinsic::ctpop)
870 .StartsWith(S: "vqadds.", Value: Intrinsic::sadd_sat)
871 .StartsWith(S: "vqaddu.", Value: Intrinsic::uadd_sat)
872 .StartsWith(S: "vqsubs.", Value: Intrinsic::ssub_sat)
873 .StartsWith(S: "vqsubu.", Value: Intrinsic::usub_sat)
874 .StartsWith(S: "vrinta.", Value: Intrinsic::round)
875 .StartsWith(S: "vrintn.", Value: Intrinsic::roundeven)
876 .StartsWith(S: "vrintm.", Value: Intrinsic::floor)
877 .StartsWith(S: "vrintp.", Value: Intrinsic::ceil)
878 .StartsWith(S: "vrintx.", Value: Intrinsic::rint)
879 .StartsWith(S: "vrintz.", Value: Intrinsic::trunc)
880 .Default(Value: Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
883 OverloadTys: F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front(Prefix: "vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(String: Name, Matches: &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 M: F->getParent(), id: StoreInts[fArgs.size() - 3], OverloadTys: Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 M: F->getParent(), id: StoreLaneInts[fArgs.size() - 5], OverloadTys: Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front(Prefix: "mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(Val: F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(GV: F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with(Prefix: "vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 M: F->getParent(), id: Intrinsic::roundeven, OverloadTys: F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(Suffix: ".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(Suffix: ".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front(Prefix: "vldr.gather.");
944 if (IsGather || Name.consume_front(Prefix: "vstr.scatter.")) {
945 if (Name.consume_front(Prefix: "base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front(Prefix: "wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front(Prefix: "offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front(Prefix: "cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith(S: "frintn", Value: Intrinsic::roundeven)
983 .StartsWith(S: "rbit", Value: Intrinsic::bitreverse)
984 .Default(Value: Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
987 OverloadTys: F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with(Prefix: "addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(Val: F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 M: F->getParent(), id: Intrinsic::aarch64_neon_faddp, OverloadTys: Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with(Prefix: "bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front(Prefix: "sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front(Prefix: "bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(x: F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 M: F->getParent(), id: Intrinsic::aarch64_sve_fmmla, OverloadTys: Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(Suffix: ".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case(S: "dot", Value: Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case(S: "mlalb", Value: Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case(S: "mlalt", Value: Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Value: Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front(Prefix: "convert.from.svbool")) {
1051 // 'aarch64.sve.convert.from.svbool'
1052 auto *TTy = dyn_cast<TargetExtType>(Val: F->getReturnType());
1053 if (!TTy || TTy->getName() != "aarch64.svcount")
1054 return false;
1055
1056 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1057 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1058 return true;
1059 }
1060
1061 if (Name.consume_front(Prefix: "convert.to.svbool")) {
1062 // 'aarch64.sve.convert.to.svbool'
1063 auto *TTy = dyn_cast<TargetExtType>(Val: F->arg_begin()->getType());
1064 if (!TTy || TTy->getName() != "aarch64.svcount")
1065 return false;
1066
1067 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1068 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1069 return true;
1070 }
1071
1072 if (Name.consume_front(Prefix: "addqv")) {
1073 // 'aarch64.sve.addqv'.
1074 if (!F->getReturnType()->isFPOrFPVectorTy())
1075 return false;
1076
1077 auto Args = F->getFunctionType()->params();
1078 Type *Tys[] = {F->getReturnType(), Args[1]};
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 M: F->getParent(), id: Intrinsic::aarch64_sve_faddqv, OverloadTys: Tys);
1081 return true;
1082 }
1083
1084 if (Name.consume_front(Prefix: "ld")) {
1085 // 'aarch64.sve.ld*'.
1086 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1087 if (LdRegex.match(String: Name)) {
1088 Type *ScalarTy =
1089 cast<VectorType>(Val: F->getReturnType())->getElementType();
1090 ElementCount EC =
1091 cast<VectorType>(Val: F->arg_begin()->getType())->getElementCount();
1092 assert(F->arg_size() == 2 &&
1093 "Expected 2 arguments for ld* intrinsic.");
1094 Type *PtrTy = F->getArg(i: 1)->getType();
1095 Type *Ty = VectorType::get(ElementType: ScalarTy, EC);
1096 static const Intrinsic::ID LoadIDs[] = {
1097 Intrinsic::aarch64_sve_ld2_sret,
1098 Intrinsic::aarch64_sve_ld3_sret,
1099 Intrinsic::aarch64_sve_ld4_sret,
1100 };
1101 NewFn = Intrinsic::getOrInsertDeclaration(
1102 M: F->getParent(), id: LoadIDs[Name[0] - '2'], OverloadTys: {Ty, PtrTy});
1103 return true;
1104 }
1105 return false; // No other 'aarch64.sve.ld*'.
1106 }
1107
1108 if (Name.consume_front(Prefix: "tuple.")) {
1109 // 'aarch64.sve.tuple.*'.
1110 if (Name.starts_with(Prefix: "get")) {
1111 // 'aarch64.sve.tuple.get*'.
1112 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1113 NewFn = Intrinsic::getOrInsertDeclaration(
1114 M: F->getParent(), id: Intrinsic::vector_extract, OverloadTys: Tys);
1115 return true;
1116 }
1117
1118 if (Name.starts_with(Prefix: "set")) {
1119 // 'aarch64.sve.tuple.set*'.
1120 auto Args = F->getFunctionType()->params();
1121 Type *Tys[] = {Args[0], Args[2], Args[1]};
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1124 return true;
1125 }
1126
1127 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1128 if (CreateTupleRegex.match(String: Name)) {
1129 // 'aarch64.sve.tuple.create*'.
1130 auto Args = F->getFunctionType()->params();
1131 Type *Tys[] = {F->getReturnType(), Args[1]};
1132 NewFn = Intrinsic::getOrInsertDeclaration(
1133 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1134 return true;
1135 }
1136 return false; // No other 'aarch64.sve.tuple.*'.
1137 }
1138
1139 if (Name.starts_with(Prefix: "rev.nxv")) {
1140 // 'aarch64.sve.rev.<Ty>'
1141 NewFn = Intrinsic::getOrInsertDeclaration(
1142 M: F->getParent(), id: Intrinsic::vector_reverse, OverloadTys: F->getReturnType());
1143 return true;
1144 }
1145
1146 return false; // No other 'aarch64.sve.*'.
1147 }
1148 if (Name.consume_front(Prefix: "sme.")) {
1149 // 'aarch64.sme.*'.
1150 if (Name.consume_front(Prefix: "ftmopa.")) {
1151 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1152 // intrinsics to model their FPMR dependency.
1153 Intrinsic::ID ID =
1154 StringSwitch<Intrinsic::ID>(Name)
1155 .Case(S: "za16.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1156 .Case(S: "za32.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1157 .Default(Value: Intrinsic::not_intrinsic);
1158 if (ID != Intrinsic::not_intrinsic) {
1159 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1160 return true;
1161 }
1162 return false; // No other 'aarch64.sme.ftmopa.*'.
1163 }
1164
1165 return false; // No other 'aarch64.sme.*'.
1166 }
1167 }
1168 return false; // No other 'arm.*', 'aarch64.*'.
1169}
1170
1171// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1172// declarations requiring an auto-upgrade. The same set applies to the
1173// cluster (g2s) and CTA (g2s_cta) variants.
1174#define NVVM_TMA_G2S_MODES(M) \
1175 M(tile_1d, "tile.1d") \
1176 M(tile_2d, "tile.2d") \
1177 M(tile_3d, "tile.3d") \
1178 M(tile_4d, "tile.4d") \
1179 M(tile_5d, "tile.5d") \
1180 M(tile_gather4_2d, "tile.gather4.2d") \
1181 M(im2col_3d, "im2col.3d") \
1182 M(im2col_4d, "im2col.4d") \
1183 M(im2col_5d, "im2col.5d") \
1184 M(im2col_w_3d, "im2col.w.3d") \
1185 M(im2col_w_4d, "im2col.w.4d") \
1186 M(im2col_w_5d, "im2col.w.5d") \
1187 M(im2col_w_128_3d, "im2col.w.128.3d") \
1188 M(im2col_w_128_4d, "im2col.w.128.4d") \
1189 M(im2col_w_128_5d, "im2col.w.128.5d")
1190
1191// Two legacy tails are:
1192//
1193// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1194// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1195//
1196// The current tail appends a trailing i32 %flag_valid_pattern, so both
1197// legacy tails are recognized by an i1 at parameter N-2.
1198static Intrinsic::ID
1199shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1200 SmallVectorImpl<Type *> &OvlTys) {
1201 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s."))
1202 return Intrinsic::not_intrinsic;
1203
1204#define G2S_ID(ID_SUFFIX, NAME) \
1205 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1206 // clang-format off
1207 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1208 NVVM_TMA_G2S_MODES(G2S_ID)
1209 .Default(Value: Intrinsic::not_intrinsic);
1210#undef G2S_ID
1211 // clang-format on
1212 if (ID == Intrinsic::not_intrinsic)
1213 return ID;
1214
1215 size_t NumParams = F->getFunctionType()->getNumParams();
1216
1217 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1218 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1219 if (!F->getFunctionType()->getParamType(i: NumParams - 2)->isIntegerTy(BitWidth: 1))
1220 return Intrinsic::not_intrinsic;
1221
1222 // The multicast mask is the parameter immediately before the i64
1223 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1224 ArrayRef<Type *> Params = F->getFunctionType()->params();
1225 size_t MaskIdx =
1226 Params[NumParams - 1]->isIntegerTy(BitWidth: 1) ? NumParams - 4 : NumParams - 5;
1227 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1228 "expected the i64 cache-hint after the multicast mask");
1229 Type *MaskTy = Params[MaskIdx];
1230 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1231 OvlTys.push_back(Elt: MaskTy);
1232
1233 return ID;
1234}
1235
1236// The legacy tail is:
1237//
1238// arg1, arg2, .. i64 %ch, i1 %flag_ch
1239//
1240// The current tail appends a trailing i32 %flag_valid_pattern, so the
1241// legacy tail is recognized by an i1 at parameter N-1.
1242static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1243 StringRef Name) {
1244 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s.cta."))
1245 return Intrinsic::not_intrinsic;
1246
1247#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1248 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1249 // clang-format off
1250 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1251 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1252 .Default(Value: Intrinsic::not_intrinsic);
1253#undef G2S_CTA_ID
1254 // clang-format on
1255 if (ID == Intrinsic::not_intrinsic)
1256 return ID;
1257
1258 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1259 // with i32 %flag_valid_pattern, for which N-1 is i32.
1260 if (!F->getFunctionType()
1261 ->getParamType(i: F->getFunctionType()->getNumParams() - 1)
1262 ->isIntegerTy(BitWidth: 1))
1263 return Intrinsic::not_intrinsic;
1264
1265 return ID;
1266}
1267// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1268//
1269// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1270//
1271// The current intrinsic is overloaded on the multicast-mask type and takes a
1272// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1273// parameter N-1.
1274static Intrinsic::ID
1275shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1276 SmallVectorImpl<Type *> &OvlTys) {
1277 if (!Name.consume_front(Prefix: "cp.async.bulk.global.to.shared.cluster"))
1278 return Intrinsic::not_intrinsic;
1279
1280 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1281 // i32 %flag_valid_pattern, for which N-1 is i32.
1282 size_t NumParams = F->getFunctionType()->getNumParams();
1283 if (!F->getFunctionType()->getParamType(i: NumParams - 1)->isIntegerTy(BitWidth: 1))
1284 return Intrinsic::not_intrinsic;
1285
1286 // The multicast mask is parameter 4; legacy IR only uses i16.
1287 Type *MaskTy = F->getFunctionType()->getParamType(i: NumParams - 4);
1288 if (!MaskTy->isIntegerTy(BitWidth: 16))
1289 return Intrinsic::not_intrinsic;
1290 OvlTys.push_back(Elt: MaskTy);
1291
1292 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1293}
1294
1295// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1296//
1297// ..., i64 %ch, i1 %flag_ch
1298//
1299// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1300// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1301// recognized by an i1 at parameter N-1, whereas the current tail ends
1302// with an i32.
1303static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1304 StringRef Name) {
1305 if (!Name.consume_front(Prefix: "cp.async.bulk.global.to.shared.cta"))
1306 return Intrinsic::not_intrinsic;
1307
1308 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1309 // i32 %flag_valid_pattern, for which N-1 is i32.
1310 if (!F->getFunctionType()->getParamType(i: 5)->isIntegerTy(BitWidth: 1))
1311 return Intrinsic::not_intrinsic;
1312
1313 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1314}
1315
1316// The legacy TMA reduction intrinsics encode the reduction operator in their
1317// name, while the current ones take it as an immediate argument. Map the
1318// operator part of a legacy name to the corresponding immediate value.
1319static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1320 return StringSwitch<std::optional<unsigned>>(Name)
1321 .Case(S: "add", Value: static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1322 .Case(S: "min", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1323 .Case(S: "max", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1324 .Case(S: "inc", Value: static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1325 .Case(S: "dec", Value: static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1326 .Case(S: "and", Value: static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1327 .Case(S: "or", Value: static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1328 .Case(S: "xor", Value: static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1329 .Default(Value: std::nullopt);
1330}
1331
1332static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1333 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.reduce."))
1334 return Intrinsic::not_intrinsic;
1335
1336 auto [RedOpName, ShapeName] = Name.split(Separator: '.');
1337 if (!getNVPTXTMAReductionOp(Name: RedOpName))
1338 return Intrinsic::not_intrinsic;
1339
1340 return StringSwitch<Intrinsic::ID>(ShapeName)
1341 .Case(S: "tile.1d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1342 .Case(S: "tile.2d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1343 .Case(S: "tile.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1344 .Case(S: "tile.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1345 .Case(S: "tile.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1346 .Case(S: "im2col.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1347 .Case(S: "im2col.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1348 .Case(S: "im2col.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1349 .Default(Value: Intrinsic::not_intrinsic);
1350}
1351
1352static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1353 StringRef Name) {
1354 if (Name.consume_front(Prefix: "mapa.shared.cluster"))
1355 if (F->getReturnType()->getPointerAddressSpace() ==
1356 NVPTXAS::ADDRESS_SPACE_SHARED)
1357 return Intrinsic::nvvm_mapa_shared_cluster;
1358
1359 if (Name.consume_front(Prefix: "cp.async.bulk.")) {
1360 Intrinsic::ID ID =
1361 StringSwitch<Intrinsic::ID>(Name)
1362 .Case(S: "shared.cta.to.cluster",
1363 Value: Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1364 .Default(Value: Intrinsic::not_intrinsic);
1365
1366 if (ID != Intrinsic::not_intrinsic)
1367 if (F->getArg(i: 0)->getType()->getPointerAddressSpace() ==
1368 NVPTXAS::ADDRESS_SPACE_SHARED)
1369 return ID;
1370 }
1371
1372 return Intrinsic::not_intrinsic;
1373}
1374
1375static Intrinsic::ID
1376shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1377 if (!Name.consume_front(Prefix: "tcgen05.commit."))
1378 return Intrinsic::not_intrinsic;
1379
1380 if (Name.consume_front(Prefix: "shared."))
1381 return StringSwitch<Intrinsic::ID>(Name)
1382 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_cg1)
1383 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_cg2)
1384 .Default(Value: Intrinsic::not_intrinsic);
1385
1386 if (Name.consume_front(Prefix: "mc.shared.")) {
1387 // Only upgrade older i16 mc variants.
1388 if (!F->getArg(i: 1)->getType()->isIntegerTy(BitWidth: 16))
1389 return Intrinsic::not_intrinsic;
1390
1391 return StringSwitch<Intrinsic::ID>(Name)
1392 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1393 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1394 .Default(Value: Intrinsic::not_intrinsic);
1395 }
1396
1397 return Intrinsic::not_intrinsic;
1398}
1399
1400static Intrinsic::ID
1401shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1402 if (F->arg_size() != 2)
1403 return Intrinsic::not_intrinsic;
1404
1405 if (Name.consume_front(Prefix: "tcgen05.alloc.shared.") ||
1406 Name.consume_front(Prefix: "tcgen05.alloc."))
1407 return StringSwitch<Intrinsic::ID>(Name)
1408 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_alloc_cg1)
1409 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_alloc_cg2)
1410 .Default(Value: Intrinsic::not_intrinsic);
1411
1412 if (Name.consume_front(Prefix: "tcgen05.dealloc."))
1413 return StringSwitch<Intrinsic::ID>(Name)
1414 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_dealloc_cg1)
1415 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_dealloc_cg2)
1416 .Default(Value: Intrinsic::not_intrinsic);
1417
1418 return Intrinsic::not_intrinsic;
1419}
1420
1421static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1422 if (Name.consume_front(Prefix: "fma.rn."))
1423 return StringSwitch<Intrinsic::ID>(Name)
1424 .Case(S: "bf16", Value: Intrinsic::nvvm_fma_rn_bf16)
1425 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fma_rn_bf16x2)
1426 .Case(S: "relu.bf16", Value: Intrinsic::nvvm_fma_rn_relu_bf16)
1427 .Case(S: "relu.bf16x2", Value: Intrinsic::nvvm_fma_rn_relu_bf16x2)
1428 .Default(Value: Intrinsic::not_intrinsic);
1429
1430 if (Name.consume_front(Prefix: "fmax."))
1431 return StringSwitch<Intrinsic::ID>(Name)
1432 .Case(S: "bf16", Value: Intrinsic::nvvm_fmax_bf16)
1433 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmax_bf16x2)
1434 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmax_ftz_bf16)
1435 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_bf16x2)
1436 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16)
1437 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1438 .Case(S: "ftz.nan.xorsign.abs.bf16",
1439 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1440 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1441 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1442 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1443 .Case(S: "ftz.xorsign.abs.bf16x2",
1444 Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1445 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmax_nan_bf16)
1446 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmax_nan_bf16x2)
1447 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1448 .Case(S: "nan.xorsign.abs.bf16x2",
1449 Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1450 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1451 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1452 .Default(Value: Intrinsic::not_intrinsic);
1453
1454 if (Name.consume_front(Prefix: "fmin."))
1455 return StringSwitch<Intrinsic::ID>(Name)
1456 .Case(S: "bf16", Value: Intrinsic::nvvm_fmin_bf16)
1457 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmin_bf16x2)
1458 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmin_ftz_bf16)
1459 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_bf16x2)
1460 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16)
1461 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1462 .Case(S: "ftz.nan.xorsign.abs.bf16",
1463 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1464 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1465 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1466 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1467 .Case(S: "ftz.xorsign.abs.bf16x2",
1468 Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1469 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmin_nan_bf16)
1470 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmin_nan_bf16x2)
1471 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1472 .Case(S: "nan.xorsign.abs.bf16x2",
1473 Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1474 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1475 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1476 .Default(Value: Intrinsic::not_intrinsic);
1477
1478 if (Name.consume_front(Prefix: "neg."))
1479 return StringSwitch<Intrinsic::ID>(Name)
1480 .Case(S: "bf16", Value: Intrinsic::nvvm_neg_bf16)
1481 .Case(S: "bf16x2", Value: Intrinsic::nvvm_neg_bf16x2)
1482 .Default(Value: Intrinsic::not_intrinsic);
1483
1484 return Intrinsic::not_intrinsic;
1485}
1486
1487static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1488 FunctionType *NewFnTy = Intrinsic::getType(Context&: F->getContext(), id: IID);
1489 FunctionType *OldFnTy = F->getFunctionType();
1490 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1491 return OldTy->getScalarType()->isIntegerTy() &&
1492 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1493 };
1494
1495 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1496 return false;
1497
1498 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1499 return false;
1500
1501 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1502 if (!IsOldBF16StorageTy(OldFnTy->getParamType(i: I), NewFnTy->getParamType(i: I)))
1503 return false;
1504
1505 return true;
1506}
1507
1508static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1509getNVVMFAddUpgrade(StringRef Name) {
1510 auto [Modifiers, Type] = Name.rsplit(Separator: '.');
1511 if (!is_contained(Set: {"f", "d", "f16", "v2f16"}, Element: Type))
1512 return std::nullopt;
1513
1514 std::optional<llvm::RoundingMode> RoundingMode =
1515 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(N: 2))
1516 .Case(S: "rn", Value: llvm::RoundingMode::NearestTiesToEven)
1517 .Case(S: "rz", Value: llvm::RoundingMode::TowardZero)
1518 .Case(S: "rm", Value: llvm::RoundingMode::TowardNegative)
1519 .Case(S: "rp", Value: llvm::RoundingMode::TowardPositive)
1520 .Default(Value: std::nullopt);
1521 if (!RoundingMode)
1522 return std::nullopt;
1523
1524 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(N: 2))
1525 .Case(S: "", Value: Intrinsic::nvvm_fadd)
1526 .Case(S: ".ftz", Value: Intrinsic::nvvm_fadd_ftz)
1527 .Case(S: ".sat", Value: Intrinsic::nvvm_fadd_sat)
1528 .Case(S: ".ftz.sat", Value: Intrinsic::nvvm_fadd_ftz_sat)
1529 .Default(Value: Intrinsic::not_intrinsic);
1530 if (IID == Intrinsic::not_intrinsic)
1531 return std::nullopt;
1532
1533 return std::make_pair(x&: IID, y&: *RoundingMode);
1534}
1535
1536static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1537 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1538 return Intrinsic::not_intrinsic;
1539
1540 return Intrinsic::nvvm_mbarrier_init;
1541}
1542
1543static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1544 return Name.consume_front(Prefix: "local") || Name.consume_front(Prefix: "shared") ||
1545 Name.consume_front(Prefix: "global") || Name.consume_front(Prefix: "constant") ||
1546 Name.consume_front(Prefix: "param");
1547}
1548
1549static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1550 if (!Name.consume_front(Prefix: "vp."))
1551 return 0;
1552 return StringSwitch<unsigned>(Name)
1553 .StartsWith(S: "select", Value: Instruction::Select)
1554 .StartsWith(S: "add", Value: Instruction::Add)
1555 .StartsWith(S: "sub", Value: Instruction::Sub)
1556 .StartsWith(S: "mul", Value: Instruction::Mul)
1557 .StartsWith(S: "ashr", Value: Instruction::AShr)
1558 .StartsWith(S: "lshr", Value: Instruction::LShr)
1559 .StartsWith(S: "shl", Value: Instruction::Shl)
1560 .StartsWith(S: "or", Value: Instruction::Or)
1561 .StartsWith(S: "and", Value: Instruction::And)
1562 .StartsWith(S: "xor", Value: Instruction::Xor)
1563 .StartsWith(S: "fadd", Value: Instruction::FAdd)
1564 .StartsWith(S: "fsub", Value: Instruction::FSub)
1565 .StartsWith(S: "fmuladd", Value: 0)
1566 .StartsWith(S: "fmul", Value: Instruction::FMul)
1567 .StartsWith(S: "fdiv", Value: Instruction::FDiv)
1568 .StartsWith(S: "frem", Value: Instruction::FRem)
1569 .StartsWith(S: "fneg", Value: Instruction::FNeg)
1570 .StartsWith(S: "trunc", Value: Instruction::Trunc)
1571 .StartsWith(S: "zext", Value: Instruction::ZExt)
1572 .StartsWith(S: "sext", Value: Instruction::SExt)
1573 .StartsWith(S: "fptrunc", Value: Instruction::FPTrunc)
1574 .StartsWith(S: "fpext", Value: Instruction::FPExt)
1575 .StartsWith(S: "fptoui", Value: Instruction::FPToUI)
1576 .StartsWith(S: "fptosi", Value: Instruction::FPToSI)
1577 .StartsWith(S: "uitofp", Value: Instruction::UIToFP)
1578 .StartsWith(S: "sitofp", Value: Instruction::SIToFP)
1579 .StartsWith(S: "ptrtoint", Value: Instruction::PtrToInt)
1580 .StartsWith(S: "inttoptr", Value: Instruction::IntToPtr)
1581 .StartsWith(S: "icmp", Value: Instruction::ICmp)
1582 .StartsWith(S: "fcmp", Value: Instruction::FCmp)
1583 .Default(Value: 0);
1584}
1585
1586static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1587 if (!Name.consume_front(Prefix: "vp."))
1588 return 0;
1589 return StringSwitch<Intrinsic::ID>(Name)
1590 .StartsWith(S: "abs", Value: Intrinsic::abs)
1591 .StartsWith(S: "smax", Value: Intrinsic::smax)
1592 .StartsWith(S: "smin", Value: Intrinsic::smin)
1593 .StartsWith(S: "umax", Value: Intrinsic::umax)
1594 .StartsWith(S: "umin", Value: Intrinsic::umin)
1595 .StartsWith(S: "copysign", Value: Intrinsic::copysign)
1596 .StartsWith(S: "minnum", Value: Intrinsic::minnum)
1597 .StartsWith(S: "maxnum", Value: Intrinsic::maxnum)
1598 .StartsWith(S: "minimum", Value: Intrinsic::minimum)
1599 .StartsWith(S: "maximum", Value: Intrinsic::maximum)
1600 .StartsWith(S: "fabs", Value: Intrinsic::fabs)
1601 .StartsWith(S: "sqrt", Value: Intrinsic::sqrt)
1602 .StartsWith(S: "fma", Value: Intrinsic::fma)
1603 .StartsWith(S: "fmuladd", Value: Intrinsic::fmuladd)
1604 .StartsWith(S: "ceil", Value: Intrinsic::ceil)
1605 .StartsWith(S: "floor", Value: Intrinsic::floor)
1606 .StartsWith(S: "rint", Value: Intrinsic::rint)
1607 .StartsWith(S: "nearbyint", Value: Intrinsic::nearbyint)
1608 .StartsWith(S: "roundeven", Value: Intrinsic::roundeven)
1609 .StartsWith(S: "roundtozero", Value: Intrinsic::trunc)
1610 .StartsWith(S: "round", Value: Intrinsic::round)
1611 .StartsWith(S: "lrint", Value: Intrinsic::lrint)
1612 .StartsWith(S: "llrint", Value: Intrinsic::llrint)
1613 .StartsWith(S: "bitreverse", Value: Intrinsic::bitreverse)
1614 .StartsWith(S: "bswap", Value: Intrinsic::bswap)
1615 .StartsWith(S: "ctpop", Value: Intrinsic::ctpop)
1616 .StartsWith(S: "ctlz", Value: Intrinsic::ctlz)
1617 .StartsWith(S: "cttz.elts", Value: 0)
1618 .StartsWith(S: "cttz", Value: Intrinsic::cttz)
1619 .StartsWith(S: "sadd.sat", Value: Intrinsic::sadd_sat)
1620 .StartsWith(S: "uadd.sat", Value: Intrinsic::uadd_sat)
1621 .StartsWith(S: "ssub.sat", Value: Intrinsic::ssub_sat)
1622 .StartsWith(S: "usub.sat", Value: Intrinsic::usub_sat)
1623 .StartsWith(S: "fshl", Value: Intrinsic::fshl)
1624 .StartsWith(S: "fshr", Value: Intrinsic::fshr)
1625 .StartsWith(S: "is.fpclass", Value: Intrinsic::is_fpclass)
1626 .Default(Value: 0);
1627}
1628
1629static bool shouldUpgradeVPIntrinsic(StringRef Name) {
1630 return getFunctionalOpcodeForVP(Name) || getFunctionalIntrinsicIDForVP(Name);
1631}
1632
1633static bool convertIntrinsicValidType(StringRef Name,
1634 const FunctionType *FuncTy) {
1635 Type *HalfTy = Type::getHalfTy(C&: FuncTy->getContext());
1636 if (Name.starts_with(Prefix: "to.fp16")) {
1637 return CastInst::castIsValid(op: Instruction::FPTrunc, SrcTy: FuncTy->getParamType(i: 0),
1638 DstTy: HalfTy) &&
1639 CastInst::castIsValid(op: Instruction::BitCast, SrcTy: HalfTy,
1640 DstTy: FuncTy->getReturnType());
1641 }
1642
1643 if (Name.starts_with(Prefix: "from.fp16")) {
1644 return CastInst::castIsValid(op: Instruction::BitCast, SrcTy: FuncTy->getParamType(i: 0),
1645 DstTy: HalfTy) &&
1646 CastInst::castIsValid(op: Instruction::FPExt, SrcTy: HalfTy,
1647 DstTy: FuncTy->getReturnType());
1648 }
1649
1650 return false;
1651}
1652
1653static unsigned getFullArgCountForDefaultArgUpgrade(Function *F,
1654 Intrinsic::ID IID) {
1655 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1656 if (Defaults.empty())
1657 return 0;
1658
1659 if (Intrinsic::isOverloaded(id: IID))
1660 return 0;
1661
1662 unsigned FullArgCount = FirstDefault + Defaults.size();
1663
1664 // Only trailing default arguments can be missing.
1665 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1666 return 0;
1667
1668 return FullArgCount;
1669}
1670
1671static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1672 Intrinsic::ID IID = F->getIntrinsicID();
1673
1674 unsigned FullArgCount = getFullArgCountForDefaultArgUpgrade(F, IID);
1675 if (FullArgCount == 0)
1676 return false;
1677
1678 rename(GV: F);
1679 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1680 assert(NewFn->arg_size() == FullArgCount &&
1681 "total number of default args does not match intrinsic signature");
1682 return true;
1683}
1684
1685static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1686 bool CanUpgradeDebugIntrinsicsToRecords) {
1687 assert(F && "Illegal to upgrade a non-existent Function.");
1688
1689 StringRef Name = F->getName();
1690
1691 // Quickly eliminate it, if it's not a candidate.
1692 if (!Name.consume_front(Prefix: "llvm.") || Name.empty())
1693 return false;
1694
1695 switch (Name[0]) {
1696 default: break;
1697 case 'a': {
1698 bool IsArm = Name.consume_front(Prefix: "arm.");
1699 if (IsArm || Name.consume_front(Prefix: "aarch64.")) {
1700 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1701 return true;
1702 break;
1703 }
1704
1705 if (Name.consume_front(Prefix: "amdgcn.")) {
1706 if (Name == "alignbit") {
1707 // Target specific intrinsic became redundant
1708 NewFn = Intrinsic::getOrInsertDeclaration(
1709 M: F->getParent(), id: Intrinsic::fshr, OverloadTys: {F->getReturnType()});
1710 return true;
1711 }
1712
1713 if (Name.consume_front(Prefix: "atomic.")) {
1714 if (Name.starts_with(Prefix: "inc") || Name.starts_with(Prefix: "dec") ||
1715 Name.starts_with(Prefix: "cond.sub") || Name.starts_with(Prefix: "csub")) {
1716 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1717 // and usub_sat so there's no new declaration.
1718 NewFn = nullptr;
1719 return true;
1720 }
1721 break; // No other 'amdgcn.atomic.*'
1722 }
1723
1724 if (Name.starts_with(Prefix: "addrspacecast.nonnull")) {
1725 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1726 // so there's no new declaration.
1727 NewFn = nullptr;
1728 return true;
1729 }
1730
1731 switch (F->getIntrinsicID()) {
1732 default:
1733 break;
1734 // Legacy wmma iu intrinsics without the optional clamp operand.
1735 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1736 if (F->arg_size() == 7) {
1737 NewFn = nullptr;
1738 return true;
1739 }
1740 break;
1741 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1742 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1743 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1744 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1745 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1746 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1747 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1748 if (F->arg_size() == 8) {
1749 NewFn = nullptr;
1750 return true;
1751 }
1752 break;
1753 }
1754
1755 if (Name.consume_front(Prefix: "ds.") || Name.consume_front(Prefix: "global.atomic.") ||
1756 Name.consume_front(Prefix: "flat.atomic.")) {
1757 if (Name.starts_with(Prefix: "fadd") ||
1758 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1759 (Name.starts_with(Prefix: "fmin") && !Name.starts_with(Prefix: "fmin.num")) ||
1760 (Name.starts_with(Prefix: "fmax") && !Name.starts_with(Prefix: "fmax.num"))) {
1761 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1762 // declaration.
1763 NewFn = nullptr;
1764 return true;
1765 }
1766 }
1767
1768 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
1769 NewFn = nullptr;
1770 return true;
1771 }
1772
1773 if (Name.starts_with(Prefix: "ldexp.")) {
1774 // Target specific intrinsic became redundant
1775 NewFn = Intrinsic::getOrInsertDeclaration(
1776 M: F->getParent(), id: Intrinsic::ldexp,
1777 OverloadTys: {F->getReturnType(), F->getArg(i: 1)->getType()});
1778 return true;
1779 }
1780 break; // No other 'amdgcn.*'
1781 }
1782
1783 break;
1784 }
1785 case 'c': {
1786 if (F->arg_size() == 1) {
1787 if (Name.consume_front(Prefix: "convert.")) {
1788 if (convertIntrinsicValidType(Name, FuncTy: F->getFunctionType())) {
1789 NewFn = nullptr;
1790 return true;
1791 }
1792 }
1793
1794 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1795 .StartsWith(S: "ctlz.", Value: Intrinsic::ctlz)
1796 .StartsWith(S: "cttz.", Value: Intrinsic::cttz)
1797 .Default(Value: Intrinsic::not_intrinsic);
1798 if (ID != Intrinsic::not_intrinsic) {
1799 rename(GV: F);
1800 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1801 OverloadTys: F->arg_begin()->getType());
1802 return true;
1803 }
1804 }
1805
1806 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1807 if (Name == "coro.end" &&
1808 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(BitWidth: 1)))
1809 CoroEndID = Intrinsic::coro_end;
1810 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(BitWidth: 1))
1811 CoroEndID = Intrinsic::coro_end_async;
1812
1813 if (CoroEndID != Intrinsic::not_intrinsic) {
1814 rename(GV: F);
1815 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: CoroEndID);
1816 return true;
1817 }
1818
1819 break;
1820 }
1821 case 'd':
1822 if (Name.consume_front(Prefix: "dbg.")) {
1823 // Mark debug intrinsics for upgrade to new debug format.
1824 if (CanUpgradeDebugIntrinsicsToRecords) {
1825 if (Name == "addr" || Name == "value" || Name == "assign" ||
1826 Name == "declare" || Name == "label") {
1827 // There's no function to replace these with.
1828 NewFn = nullptr;
1829 // But we do want these to get upgraded.
1830 return true;
1831 }
1832 }
1833 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1834 // converted to DbgVariableRecords later.
1835 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1836 rename(GV: F);
1837 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1838 id: Intrinsic::dbg_value);
1839 return true;
1840 }
1841 break; // No other 'dbg.*'.
1842 }
1843 break;
1844 case 'e':
1845 if (Name.consume_front(Prefix: "experimental.vector.")) {
1846 Intrinsic::ID ID =
1847 StringSwitch<Intrinsic::ID>(Name)
1848 // Skip over extract.last.active, otherwise it will be 'upgraded'
1849 // to a regular vector extract which is a different operation.
1850 .StartsWith(S: "extract.last.active.", Value: Intrinsic::not_intrinsic)
1851 .StartsWith(S: "extract.", Value: Intrinsic::vector_extract)
1852 .StartsWith(S: "insert.", Value: Intrinsic::vector_insert)
1853 .StartsWith(S: "reverse.", Value: Intrinsic::vector_reverse)
1854 .StartsWith(S: "interleave2.", Value: Intrinsic::vector_interleave2)
1855 .StartsWith(S: "deinterleave2.", Value: Intrinsic::vector_deinterleave2)
1856 .StartsWith(S: "partial.reduce.add",
1857 Value: Intrinsic::vector_partial_reduce_add)
1858 .Default(Value: Intrinsic::not_intrinsic);
1859 if (ID != Intrinsic::not_intrinsic) {
1860 const auto *FT = F->getFunctionType();
1861 SmallVector<Type *, 2> Tys;
1862 if (ID == Intrinsic::vector_extract ||
1863 ID == Intrinsic::vector_interleave2)
1864 // Extracting overloads the return type.
1865 Tys.push_back(Elt: FT->getReturnType());
1866 if (ID != Intrinsic::vector_interleave2)
1867 Tys.push_back(Elt: FT->getParamType(i: 0));
1868 if (ID == Intrinsic::vector_insert ||
1869 ID == Intrinsic::vector_partial_reduce_add)
1870 // Inserting overloads the inserted type.
1871 Tys.push_back(Elt: FT->getParamType(i: 1));
1872 rename(GV: F);
1873 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
1874 return true;
1875 }
1876
1877 if (Name.consume_front(Prefix: "reduce.")) {
1878 SmallVector<StringRef, 2> Groups;
1879 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1880 if (R.match(String: Name, Matches: &Groups))
1881 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1882 .Case(S: "add", Value: Intrinsic::vector_reduce_add)
1883 .Case(S: "mul", Value: Intrinsic::vector_reduce_mul)
1884 .Case(S: "and", Value: Intrinsic::vector_reduce_and)
1885 .Case(S: "or", Value: Intrinsic::vector_reduce_or)
1886 .Case(S: "xor", Value: Intrinsic::vector_reduce_xor)
1887 .Case(S: "smax", Value: Intrinsic::vector_reduce_smax)
1888 .Case(S: "smin", Value: Intrinsic::vector_reduce_smin)
1889 .Case(S: "umax", Value: Intrinsic::vector_reduce_umax)
1890 .Case(S: "umin", Value: Intrinsic::vector_reduce_umin)
1891 .Case(S: "fmax", Value: Intrinsic::vector_reduce_fmax)
1892 .Case(S: "fmin", Value: Intrinsic::vector_reduce_fmin)
1893 .Default(Value: Intrinsic::not_intrinsic);
1894
1895 bool V2 = false;
1896 if (ID == Intrinsic::not_intrinsic) {
1897 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1898 Groups.clear();
1899 V2 = true;
1900 if (R2.match(String: Name, Matches: &Groups))
1901 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1902 .Case(S: "fadd", Value: Intrinsic::vector_reduce_fadd)
1903 .Case(S: "fmul", Value: Intrinsic::vector_reduce_fmul)
1904 .Default(Value: Intrinsic::not_intrinsic);
1905 }
1906 if (ID != Intrinsic::not_intrinsic) {
1907 rename(GV: F);
1908 auto Args = F->getFunctionType()->params();
1909 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1910 OverloadTys: {Args[V2 ? 1 : 0]});
1911 return true;
1912 }
1913 break; // No other 'expermental.vector.reduce.*'.
1914 }
1915
1916 if (Name.consume_front(Prefix: "splice"))
1917 return true;
1918 break; // No other 'experimental.vector.*'.
1919 }
1920 if (Name.consume_front(Prefix: "experimental.stepvector.")) {
1921 Intrinsic::ID ID = Intrinsic::stepvector;
1922 rename(GV: F);
1923 NewFn = Intrinsic::getOrInsertDeclaration(
1924 M: F->getParent(), id: ID, OverloadTys: F->getFunctionType()->getReturnType());
1925 return true;
1926 }
1927 break; // No other 'e*'.
1928 case 'f':
1929 if (Name.starts_with(Prefix: "flt.rounds")) {
1930 rename(GV: F);
1931 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1932 id: Intrinsic::get_rounding);
1933 return true;
1934 }
1935 break;
1936 case 'i':
1937 if (Name.starts_with(Prefix: "invariant.group.barrier")) {
1938 // Rename invariant.group.barrier to launder.invariant.group
1939 auto Args = F->getFunctionType()->params();
1940 Type* ObjectPtr[1] = {Args[0]};
1941 rename(GV: F);
1942 NewFn = Intrinsic::getOrInsertDeclaration(
1943 M: F->getParent(), id: Intrinsic::launder_invariant_group, OverloadTys: ObjectPtr);
1944 return true;
1945 }
1946 break;
1947 case 'l': {
1948 bool IsLifetimeStart = Name.consume_front(Prefix: "lifetime.start");
1949 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front(Prefix: "lifetime.end");
1950 if (IsLifetimeStart || IsLifetimeEnd) {
1951 if (F->arg_size() == 2) {
1952 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1953 : Intrinsic::lifetime_end;
1954 rename(GV: F);
1955 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1956 // arguments. Use the Ptr argument to create new declaration.
1957 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1958 OverloadTys: F->getArg(i: 1)->getType());
1959 return true;
1960 } else if (F->arg_size() == 1 && Name == ".i64") {
1961 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1962 // Autoupgrade prior to
1963 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1964 // intrinsic with no expected calls. To allow auto-upgrade process to
1965 // delete such invalid intrinsic declaration, set NewFn = nullptr
1966 // and return true here. If there are actual calls to this intrinsic
1967 // (which is not expected), they will be deleted in
1968 // UpgradeIntrinsicCall.
1969 NewFn = nullptr;
1970 return true;
1971 }
1972 }
1973 break;
1974 }
1975 case 'm': {
1976 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1977 // alignment parameter to embedding the alignment as an attribute of
1978 // the pointer args.
1979 if (unsigned ID = StringSwitch<unsigned>(Name)
1980 .StartsWith(S: "memcpy.", Value: Intrinsic::memcpy)
1981 .StartsWith(S: "memmove.", Value: Intrinsic::memmove)
1982 .Default(Value: 0)) {
1983 if (F->arg_size() == 5) {
1984 rename(GV: F);
1985 // Get the types of dest, src, and len
1986 ArrayRef<Type *> ParamTypes =
1987 F->getFunctionType()->params().slice(N: 0, M: 3);
1988 NewFn =
1989 Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: ParamTypes);
1990 return true;
1991 }
1992 }
1993 if (Name.starts_with(Prefix: "memset.") && F->arg_size() == 5) {
1994 rename(GV: F);
1995 // Get the types of dest, and len
1996 const auto *FT = F->getFunctionType();
1997 Type *ParamTypes[2] = {
1998 FT->getParamType(i: 0), // Dest
1999 FT->getParamType(i: 2) // len
2000 };
2001 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2002 id: Intrinsic::memset, OverloadTys: ParamTypes);
2003 return true;
2004 }
2005
2006 unsigned MaskedID =
2007 StringSwitch<unsigned>(Name)
2008 .StartsWith(S: "masked.load", Value: Intrinsic::masked_load)
2009 .StartsWith(S: "masked.gather", Value: Intrinsic::masked_gather)
2010 .StartsWith(S: "masked.store", Value: Intrinsic::masked_store)
2011 .StartsWith(S: "masked.scatter", Value: Intrinsic::masked_scatter)
2012 .Default(Value: 0);
2013 if (MaskedID && F->arg_size() == 4) {
2014 rename(GV: F);
2015 if (MaskedID == Intrinsic::masked_load ||
2016 MaskedID == Intrinsic::masked_gather) {
2017 NewFn = Intrinsic::getOrInsertDeclaration(
2018 M: F->getParent(), id: MaskedID,
2019 OverloadTys: {F->getReturnType(), F->getArg(i: 0)->getType()});
2020 return true;
2021 }
2022 NewFn = Intrinsic::getOrInsertDeclaration(
2023 M: F->getParent(), id: MaskedID,
2024 OverloadTys: {F->getArg(i: 0)->getType(), F->getArg(i: 1)->getType()});
2025 return true;
2026 }
2027 break;
2028 }
2029 case 'n': {
2030 if (Name.consume_front(Prefix: "nvvm.")) {
2031 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2032 if (F->arg_size() == 1) {
2033 Intrinsic::ID IID =
2034 StringSwitch<Intrinsic::ID>(Name)
2035 .Cases(CaseStrings: {"brev32", "brev64"}, Value: Intrinsic::bitreverse)
2036 .Case(S: "clz.i", Value: Intrinsic::ctlz)
2037 .Case(S: "popc.i", Value: Intrinsic::ctpop)
2038 .Default(Value: Intrinsic::not_intrinsic);
2039 if (IID != Intrinsic::not_intrinsic) {
2040 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2041 OverloadTys: {F->getReturnType()});
2042 return true;
2043 }
2044 } else if (F->arg_size() == 2) {
2045 Intrinsic::ID IID =
2046 StringSwitch<Intrinsic::ID>(Name)
2047 .Cases(CaseStrings: {"max.s", "max.i", "max.ll"}, Value: Intrinsic::smax)
2048 .Cases(CaseStrings: {"min.s", "min.i", "min.ll"}, Value: Intrinsic::smin)
2049 .Cases(CaseStrings: {"max.us", "max.ui", "max.ull"}, Value: Intrinsic::umax)
2050 .Cases(CaseStrings: {"min.us", "min.ui", "min.ull"}, Value: Intrinsic::umin)
2051 .Default(Value: Intrinsic::not_intrinsic);
2052 if (IID != Intrinsic::not_intrinsic) {
2053 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2054 OverloadTys: {F->getReturnType()});
2055 return true;
2056 }
2057 }
2058
2059 // Check for nvvm intrinsics that need a return type adjustment.
2060 {
2061 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2062 if (IID != Intrinsic::not_intrinsic &&
2063 isLegacyNVPTXBF16IntSignature(F, IID)) {
2064 NewFn = nullptr;
2065 return true;
2066 }
2067 }
2068
2069 // Upgrade Distributed Shared Memory Intrinsics
2070 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2071 if (IID != Intrinsic::not_intrinsic) {
2072 rename(GV: F);
2073 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2074 return true;
2075 }
2076
2077 // Upgrade TMA reduction intrinsics
2078 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2079 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2080 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2081 if (IID != Intrinsic::not_intrinsic) {
2082 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2083 return true;
2084 }
2085
2086 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2087 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2088 if (IID != Intrinsic::not_intrinsic) {
2089 rename(GV: F);
2090 NewFn = Intrinsic::getOrInsertDeclaration(
2091 M: F->getParent(), IID, RetTy: F->getReturnType(),
2092 ArgTys: F->getFunctionType()->params());
2093 return true;
2094 }
2095
2096 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2097 // tcgen05.alloc shared variants to anyptr intrinsics.
2098 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2099 if (IID != Intrinsic::not_intrinsic) {
2100 rename(GV: F);
2101 if (Intrinsic::isOverloaded(id: IID))
2102 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2103 OverloadTys: {F->getArg(i: 0)->getType()});
2104 else
2105 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2106 return true;
2107 }
2108
2109 // Upgrade TMA copy G2S CTA intrinsics.
2110 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2111 if (IID != Intrinsic::not_intrinsic) {
2112 rename(GV: F);
2113 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2114 return true;
2115 }
2116
2117 // Upgrade TMA copy G2S (cluster) intrinsics.
2118 SmallVector<Type *, 1> OvlTys;
2119 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2120 if (IID != Intrinsic::not_intrinsic) {
2121 rename(GV: F);
2122 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID, OverloadTys: OvlTys);
2123 return true;
2124 }
2125
2126 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2127 // (multicast-mask overloading + trailing flag_valid_pattern).
2128 SmallVector<Type *, 1> BulkG2SOvlTys;
2129 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, OvlTys&: BulkG2SOvlTys);
2130 if (IID != Intrinsic::not_intrinsic) {
2131 rename(GV: F);
2132 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2133 OverloadTys: BulkG2SOvlTys);
2134 return true;
2135 }
2136
2137 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2138 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2139 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2140 if (IID != Intrinsic::not_intrinsic) {
2141 rename(GV: F);
2142 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2143 return true;
2144 }
2145
2146 // Upgrade mbarrier.init intrinsics missing the layout operand.
2147 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2148 if (IID != Intrinsic::not_intrinsic) {
2149 rename(GV: F);
2150 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2151 OverloadTys: F->getArg(i: 0)->getType());
2152 return true;
2153 }
2154
2155 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2156 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2157 //
2158 // TODO: We could add lohi.i2d.
2159 bool Expand = false;
2160 if (Name.consume_front(Prefix: "abs."))
2161 // nvvm.abs.{i,ii}
2162 Expand =
2163 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2164 else if (Name.consume_front(Prefix: "fabs."))
2165 // nvvm.fabs.{f,ftz.f,d}
2166 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2167 else if (Name.consume_front(Prefix: "add."))
2168 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2169 Expand = getNVVMFAddUpgrade(Name).has_value();
2170 else if (Name.consume_front(Prefix: "ex2.approx."))
2171 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2172 Expand =
2173 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2174 else if (Name.consume_front(Prefix: "atomic.load."))
2175 // nvvm.atomic.load.add.{f32,f64}.p
2176 // nvvm.atomic.load.{inc,dec}.32.p
2177 Expand = StringSwitch<bool>(Name)
2178 .StartsWith(S: "add.f32.p", Value: true)
2179 .StartsWith(S: "add.f64.p", Value: true)
2180 .StartsWith(S: "inc.32.p", Value: true)
2181 .StartsWith(S: "dec.32.p", Value: true)
2182 .Default(Value: false);
2183 else if (Name.consume_front(Prefix: "atomic."))
2184 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2185 // nvvm.atomic.cas.gen.i.{cta,sys}
2186 Expand = StringSwitch<bool>(Name)
2187 .StartsWith(S: "add.gen.", Value: true)
2188 .StartsWith(S: "exch.gen.", Value: true)
2189 .StartsWith(S: "max.gen.", Value: true)
2190 .StartsWith(S: "min.gen.", Value: true)
2191 .StartsWith(S: "inc.gen.", Value: true)
2192 .StartsWith(S: "dec.gen.", Value: true)
2193 .StartsWith(S: "and.gen.", Value: true)
2194 .StartsWith(S: "or.gen.", Value: true)
2195 .StartsWith(S: "xor.gen.", Value: true)
2196 .StartsWith(S: "cas.gen.", Value: true)
2197 .Default(Value: false);
2198 else if (Name.consume_front(Prefix: "bitcast."))
2199 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2200 Expand =
2201 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2202 else if (Name.consume_front(Prefix: "rotate."))
2203 // nvvm.rotate.{b32,b64,right.b64}
2204 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2205 else if (Name.consume_front(Prefix: "ptr.gen.to."))
2206 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2207 Expand = consumeNVVMPtrAddrSpace(Name);
2208 else if (Name.consume_front(Prefix: "ptr."))
2209 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2210 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(Prefix: ".to.gen");
2211 else if (Name.consume_front(Prefix: "ldg.global."))
2212 // nvvm.ldg.global.{i,p,f}
2213 Expand = (Name.starts_with(Prefix: "i.") || Name.starts_with(Prefix: "f.") ||
2214 Name.starts_with(Prefix: "p."));
2215 else
2216 Expand = StringSwitch<bool>(Name)
2217 .Case(S: "barrier0", Value: true)
2218 .Case(S: "barrier.n", Value: true)
2219 .Case(S: "barrier.sync.cnt", Value: true)
2220 .Case(S: "barrier.sync", Value: true)
2221 .Case(S: "barrier", Value: true)
2222 .Case(S: "bar.sync", Value: true)
2223 .Case(S: "barrier0.popc", Value: true)
2224 .Case(S: "barrier0.and", Value: true)
2225 .Case(S: "barrier0.or", Value: true)
2226 .Case(S: "clz.ll", Value: true)
2227 .Case(S: "popc.ll", Value: true)
2228 .Case(S: "h2f", Value: true)
2229 .Case(S: "swap.lo.hi.b64", Value: true)
2230 .Case(S: "tanh.approx.f32", Value: true)
2231 .Default(Value: false);
2232
2233 if (Expand) {
2234 NewFn = nullptr;
2235 return true;
2236 }
2237 break; // No other 'nvvm.*'.
2238 }
2239 break;
2240 }
2241 case 'o':
2242 if (Name.starts_with(Prefix: "objectsize.")) {
2243 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2244 if (F->arg_size() == 2 || F->arg_size() == 3) {
2245 rename(GV: F);
2246 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2247 id: Intrinsic::objectsize, OverloadTys: Tys);
2248 return true;
2249 }
2250 }
2251 break;
2252
2253 case 'p':
2254 if (Name.starts_with(Prefix: "ptr.annotation.") && F->arg_size() == 4) {
2255 rename(GV: F);
2256 NewFn = Intrinsic::getOrInsertDeclaration(
2257 M: F->getParent(), id: Intrinsic::ptr_annotation,
2258 OverloadTys: {F->arg_begin()->getType(), F->getArg(i: 1)->getType()});
2259 return true;
2260 }
2261 break;
2262
2263 case 'r': {
2264 if (Name.consume_front(Prefix: "riscv.")) {
2265 Intrinsic::ID ID;
2266 ID = StringSwitch<Intrinsic::ID>(Name)
2267 .Case(S: "aes32dsi", Value: Intrinsic::riscv_aes32dsi)
2268 .Case(S: "aes32dsmi", Value: Intrinsic::riscv_aes32dsmi)
2269 .Case(S: "aes32esi", Value: Intrinsic::riscv_aes32esi)
2270 .Case(S: "aes32esmi", Value: Intrinsic::riscv_aes32esmi)
2271 .Default(Value: Intrinsic::not_intrinsic);
2272 if (ID != Intrinsic::not_intrinsic) {
2273 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32)) {
2274 rename(GV: F);
2275 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2276 return true;
2277 }
2278 break; // No other applicable upgrades.
2279 }
2280
2281 ID = StringSwitch<Intrinsic::ID>(Name)
2282 .StartsWith(S: "sm4ks", Value: Intrinsic::riscv_sm4ks)
2283 .StartsWith(S: "sm4ed", Value: Intrinsic::riscv_sm4ed)
2284 .Default(Value: Intrinsic::not_intrinsic);
2285 if (ID != Intrinsic::not_intrinsic) {
2286 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32) ||
2287 F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2288 rename(GV: F);
2289 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2290 return true;
2291 }
2292 break; // No other applicable upgrades.
2293 }
2294
2295 ID = StringSwitch<Intrinsic::ID>(Name)
2296 .StartsWith(S: "sha256sig0", Value: Intrinsic::riscv_sha256sig0)
2297 .StartsWith(S: "sha256sig1", Value: Intrinsic::riscv_sha256sig1)
2298 .StartsWith(S: "sha256sum0", Value: Intrinsic::riscv_sha256sum0)
2299 .StartsWith(S: "sha256sum1", Value: Intrinsic::riscv_sha256sum1)
2300 .StartsWith(S: "sm3p0", Value: Intrinsic::riscv_sm3p0)
2301 .StartsWith(S: "sm3p1", Value: Intrinsic::riscv_sm3p1)
2302 .Default(Value: Intrinsic::not_intrinsic);
2303 if (ID != Intrinsic::not_intrinsic) {
2304 if (F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2305 rename(GV: F);
2306 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2307 return true;
2308 }
2309 break; // No other applicable upgrades.
2310 }
2311
2312 // Replace llvm.riscv.clmul with llvm.clmul.
2313 if (Name == "clmul.i32" || Name == "clmul.i64") {
2314 NewFn = Intrinsic::getOrInsertDeclaration(
2315 M: F->getParent(), id: Intrinsic::clmul, OverloadTys: {F->getReturnType()});
2316 return true;
2317 }
2318
2319 break; // No other 'riscv.*' intrinsics
2320 }
2321 } break;
2322
2323 case 's':
2324 if (Name == "stackprotectorcheck") {
2325 NewFn = nullptr;
2326 return true;
2327 }
2328 break;
2329
2330 case 't':
2331 if (Name == "thread.pointer") {
2332 NewFn = Intrinsic::getOrInsertDeclaration(
2333 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
2334 return true;
2335 }
2336 break;
2337
2338 case 'v': {
2339 if (Name == "var.annotation" && F->arg_size() == 4) {
2340 rename(GV: F);
2341 NewFn = Intrinsic::getOrInsertDeclaration(
2342 M: F->getParent(), id: Intrinsic::var_annotation,
2343 OverloadTys: {{F->arg_begin()->getType(), F->getArg(i: 1)->getType()}});
2344 return true;
2345 }
2346 if (Name.consume_front(Prefix: "vector.splice")) {
2347 if (Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"))
2348 break;
2349 return true;
2350 }
2351 if (shouldUpgradeVPIntrinsic(Name))
2352 return true;
2353 break;
2354 }
2355
2356 case 'w':
2357 if (Name.consume_front(Prefix: "wasm.")) {
2358 Intrinsic::ID ID =
2359 StringSwitch<Intrinsic::ID>(Name)
2360 .StartsWith(S: "fma.", Value: Intrinsic::wasm_relaxed_madd)
2361 .StartsWith(S: "fms.", Value: Intrinsic::wasm_relaxed_nmadd)
2362 .StartsWith(S: "laneselect.", Value: Intrinsic::wasm_relaxed_laneselect)
2363 .Default(Value: Intrinsic::not_intrinsic);
2364 if (ID != Intrinsic::not_intrinsic) {
2365 rename(GV: F);
2366 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
2367 OverloadTys: F->getReturnType());
2368 return true;
2369 }
2370
2371 if (Name.consume_front(Prefix: "dot.i8x16.i7x16.")) {
2372 ID = StringSwitch<Intrinsic::ID>(Name)
2373 .Case(S: "signed", Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2374 .Case(S: "add.signed",
2375 Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2376 .Default(Value: Intrinsic::not_intrinsic);
2377 if (ID != Intrinsic::not_intrinsic) {
2378 rename(GV: F);
2379 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2380 return true;
2381 }
2382 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2383 }
2384 break; // No other 'wasm.*'.
2385 }
2386 break;
2387
2388 case 'x':
2389 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2390 return true;
2391 }
2392
2393 auto *ST = dyn_cast<StructType>(Val: F->getReturnType());
2394 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2395 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2396 // Replace return type with literal non-packed struct. Only do this for
2397 // intrinsics declared to return a struct, not for intrinsics with
2398 // overloaded return type, in which case the exact struct type will be
2399 // mangled into the name.
2400 if (Intrinsic::hasStructReturnType(id: F->getIntrinsicID())) {
2401 FunctionType *FT = F->getFunctionType();
2402 auto *NewST = StructType::get(Context&: ST->getContext(), Elements: ST->elements());
2403 auto *NewFT = FunctionType::get(Result: NewST, Params: FT->params(), isVarArg: FT->isVarArg());
2404 std::string Name = F->getName().str();
2405 rename(GV: F);
2406 NewFn = Function::Create(Ty: NewFT, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
2407 N: Name, M: F->getParent());
2408
2409 // The new function may also need remangling.
2410 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F: NewFn))
2411 NewFn = *Result;
2412 return true;
2413 }
2414 }
2415
2416 // Remangle our intrinsic since we upgrade the mangling
2417 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2418 if (Result != std::nullopt) {
2419 NewFn = *Result;
2420 return true;
2421 }
2422
2423 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2424 return true;
2425
2426 // This may not belong here. This function is effectively being overloaded
2427 // to both detect an intrinsic which needs upgrading, and to provide the
2428 // upgraded form of the intrinsic. We should perhaps have two separate
2429 // functions for this.
2430
2431 return false;
2432}
2433
2434bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2435 bool CanUpgradeDebugIntrinsicsToRecords) {
2436 NewFn = nullptr;
2437 bool Upgraded =
2438 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2439
2440 // Upgrade intrinsic attributes. This does not change the function.
2441 if (NewFn)
2442 F = NewFn;
2443 if (Intrinsic::ID id = F->getIntrinsicID()) {
2444 // Only do this if the intrinsic signature is valid.
2445 SmallVector<Type *> OverloadTys;
2446 if (Intrinsic::isSignatureValid(ID: id, FT: F->getFunctionType(), OverloadTys))
2447 F->setAttributes(
2448 Intrinsic::getAttributes(C&: F->getContext(), id, FT: F->getFunctionType()));
2449 }
2450 return Upgraded;
2451}
2452
2453GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2454 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2455 GV->getName() == "llvm.global_dtors")) ||
2456 !GV->hasInitializer())
2457 return nullptr;
2458 ArrayType *ATy = dyn_cast<ArrayType>(Val: GV->getValueType());
2459 if (!ATy)
2460 return nullptr;
2461 StructType *STy = dyn_cast<StructType>(Val: ATy->getElementType());
2462 if (!STy || STy->getNumElements() != 2)
2463 return nullptr;
2464
2465 LLVMContext &C = GV->getContext();
2466 IRBuilder<> IRB(C);
2467 auto EltTy = StructType::get(elt1: STy->getElementType(N: 0), elts: STy->getElementType(N: 1),
2468 elts: IRB.getPtrTy());
2469 Constant *Init = GV->getInitializer();
2470 unsigned N = Init->getNumOperands();
2471 std::vector<Constant *> NewCtors(N);
2472 for (unsigned i = 0; i != N; ++i) {
2473 auto Ctor = cast<Constant>(Val: Init->getOperand(i));
2474 NewCtors[i] = ConstantStruct::get(T: EltTy, Vs: Ctor->getAggregateElement(Elt: 0u),
2475 Vs: Ctor->getAggregateElement(Elt: 1),
2476 Vs: ConstantPointerNull::get(T: IRB.getPtrTy()));
2477 }
2478 Constant *NewInit = ConstantArray::get(T: ArrayType::get(ElementType: EltTy, NumElements: N), V: NewCtors);
2479
2480 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2481 NewInit, GV->getName());
2482}
2483
2484// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2485// to byte shuffles.
2486static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2487 unsigned Shift) {
2488 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2489 unsigned NumElts = ResultTy->getNumElements() * 8;
2490
2491 // Bitcast from a 64-bit element type to a byte element type.
2492 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2493 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2494
2495 // We'll be shuffling in zeroes.
2496 Value *Res = Constant::getNullValue(Ty: VecTy);
2497
2498 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2499 // we'll just return the zero vector.
2500 if (Shift < 16) {
2501 int Idxs[64];
2502 // 256/512-bit version is split into 2/4 16-byte lanes.
2503 for (unsigned l = 0; l != NumElts; l += 16)
2504 for (unsigned i = 0; i != 16; ++i) {
2505 unsigned Idx = NumElts + i - Shift;
2506 if (Idx < NumElts)
2507 Idx -= NumElts - 16; // end of lane, switch operand.
2508 Idxs[l + i] = Idx + l;
2509 }
2510
2511 Res = Builder.CreateShuffleVector(V1: Res, V2: Op, Mask: ArrayRef(Idxs, NumElts));
2512 }
2513
2514 // Bitcast back to a 64-bit element type.
2515 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2516}
2517
2518// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2519// to byte shuffles.
2520static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2521 unsigned Shift) {
2522 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2523 unsigned NumElts = ResultTy->getNumElements() * 8;
2524
2525 // Bitcast from a 64-bit element type to a byte element type.
2526 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2527 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2528
2529 // We'll be shuffling in zeroes.
2530 Value *Res = Constant::getNullValue(Ty: VecTy);
2531
2532 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2533 // we'll just return the zero vector.
2534 if (Shift < 16) {
2535 int Idxs[64];
2536 // 256/512-bit version is split into 2/4 16-byte lanes.
2537 for (unsigned l = 0; l != NumElts; l += 16)
2538 for (unsigned i = 0; i != 16; ++i) {
2539 unsigned Idx = i + Shift;
2540 if (Idx >= 16)
2541 Idx += NumElts - 16; // end of lane, switch operand.
2542 Idxs[l + i] = Idx + l;
2543 }
2544
2545 Res = Builder.CreateShuffleVector(V1: Op, V2: Res, Mask: ArrayRef(Idxs, NumElts));
2546 }
2547
2548 // Bitcast back to a 64-bit element type.
2549 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2550}
2551
2552static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2553 unsigned NumElts) {
2554 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2555 llvm::VectorType *MaskTy = FixedVectorType::get(
2556 ElementType: Builder.getInt1Ty(), NumElts: cast<IntegerType>(Val: Mask->getType())->getBitWidth());
2557 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2558
2559 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2560 // i8 and we need to extract down to the right number of elements.
2561 if (NumElts <= 4) {
2562 int Indices[4];
2563 for (unsigned i = 0; i != NumElts; ++i)
2564 Indices[i] = i;
2565 Mask = Builder.CreateShuffleVector(V1: Mask, V2: Mask, Mask: ArrayRef(Indices, NumElts),
2566 Name: "extract");
2567 }
2568
2569 return Mask;
2570}
2571
2572static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2573 Value *Op1) {
2574 // If the mask is all ones just emit the first operation.
2575 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2576 if (C->isAllOnesValue())
2577 return Op0;
2578
2579 Mask = getX86MaskVec(Builder, Mask,
2580 NumElts: cast<FixedVectorType>(Val: Op0->getType())->getNumElements());
2581 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2582}
2583
2584static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2585 Value *Op1) {
2586 // If the mask is all ones just emit the first operation.
2587 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2588 if (C->isAllOnesValue())
2589 return Op0;
2590
2591 auto *MaskTy = FixedVectorType::get(ElementType: Builder.getInt1Ty(),
2592 NumElts: Mask->getType()->getIntegerBitWidth());
2593 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2594 Mask = Builder.CreateExtractElement(Vec: Mask, Idx: (uint64_t)0);
2595 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2596}
2597
2598// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2599// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2600// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2601static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2602 Value *Op1, Value *Shift,
2603 Value *Passthru, Value *Mask,
2604 bool IsVALIGN) {
2605 unsigned ShiftVal = cast<llvm::ConstantInt>(Val: Shift)->getZExtValue();
2606
2607 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2608 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2609 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2610 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2611
2612 // Mask the immediate for VALIGN.
2613 if (IsVALIGN)
2614 ShiftVal &= (NumElts - 1);
2615
2616 // If palignr is shifting the pair of vectors more than the size of two
2617 // lanes, emit zero.
2618 if (ShiftVal >= 32)
2619 return llvm::Constant::getNullValue(Ty: Op0->getType());
2620
2621 // If palignr is shifting the pair of input vectors more than one lane,
2622 // but less than two lanes, convert to shifting in zeroes.
2623 if (ShiftVal > 16) {
2624 ShiftVal -= 16;
2625 Op1 = Op0;
2626 Op0 = llvm::Constant::getNullValue(Ty: Op0->getType());
2627 }
2628
2629 int Indices[64];
2630 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2631 for (unsigned l = 0; l < NumElts; l += 16) {
2632 for (unsigned i = 0; i != 16; ++i) {
2633 unsigned Idx = ShiftVal + i;
2634 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2635 Idx += NumElts - 16; // End of lane, switch operand.
2636 Indices[l + i] = Idx + l;
2637 }
2638 }
2639
2640 Value *Align = Builder.CreateShuffleVector(
2641 V1: Op1, V2: Op0, Mask: ArrayRef(Indices, NumElts), Name: "palignr");
2642
2643 return emitX86Select(Builder, Mask, Op0: Align, Op1: Passthru);
2644}
2645
2646static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2647 bool ZeroMask, bool IndexForm) {
2648 Type *Ty = CI.getType();
2649 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2650 unsigned EltWidth = Ty->getScalarSizeInBits();
2651 bool IsFloat = Ty->isFPOrFPVectorTy();
2652 Intrinsic::ID IID;
2653 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2654 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2655 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2656 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2657 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2658 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2659 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2660 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2661 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2662 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2663 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2664 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2665 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2666 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2667 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2668 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2669 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2670 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2671 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2672 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2673 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2674 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2675 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2676 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2677 else if (VecWidth == 128 && EltWidth == 16)
2678 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2679 else if (VecWidth == 256 && EltWidth == 16)
2680 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2681 else if (VecWidth == 512 && EltWidth == 16)
2682 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2683 else if (VecWidth == 128 && EltWidth == 8)
2684 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2685 else if (VecWidth == 256 && EltWidth == 8)
2686 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2687 else if (VecWidth == 512 && EltWidth == 8)
2688 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2689 else
2690 llvm_unreachable("Unexpected intrinsic");
2691
2692 Value *Args[] = { CI.getArgOperand(i: 0) , CI.getArgOperand(i: 1),
2693 CI.getArgOperand(i: 2) };
2694
2695 // If this isn't index form we need to swap operand 0 and 1.
2696 if (!IndexForm)
2697 std::swap(a&: Args[0], b&: Args[1]);
2698
2699 Value *V = Builder.CreateIntrinsic(ID: IID, Args);
2700 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2701 : Builder.CreateBitCast(V: CI.getArgOperand(i: 1),
2702 DestTy: Ty);
2703 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: V, Op1: PassThru);
2704}
2705
2706static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2707 Intrinsic::ID IID) {
2708 Type *Ty = CI.getType();
2709 Value *Op0 = CI.getOperand(i_nocapture: 0);
2710 Value *Op1 = CI.getOperand(i_nocapture: 1);
2711 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1});
2712
2713 if (CI.arg_size() == 4) { // For masked intrinsics.
2714 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2715 Value *Mask = CI.getOperand(i_nocapture: 3);
2716 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2717 }
2718 return Res;
2719}
2720
2721static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2722 bool IsRotateRight) {
2723 Type *Ty = CI.getType();
2724 Value *Src = CI.getArgOperand(i: 0);
2725 Value *Amt = CI.getArgOperand(i: 1);
2726
2727 // Amount may be scalar immediate, in which case create a splat vector.
2728 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2729 // we only care about the lowest log2 bits anyway.
2730 if (Amt->getType() != Ty) {
2731 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2732 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2733 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2734 }
2735
2736 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2737 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Src, Src, Amt});
2738
2739 if (CI.arg_size() == 4) { // For masked intrinsics.
2740 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2741 Value *Mask = CI.getOperand(i_nocapture: 3);
2742 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2743 }
2744 return Res;
2745}
2746
2747static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2748 bool IsSigned) {
2749 Type *Ty = CI.getType();
2750 Value *LHS = CI.getArgOperand(i: 0);
2751 Value *RHS = CI.getArgOperand(i: 1);
2752
2753 CmpInst::Predicate Pred;
2754 switch (Imm) {
2755 case 0x0:
2756 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2757 break;
2758 case 0x1:
2759 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2760 break;
2761 case 0x2:
2762 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2763 break;
2764 case 0x3:
2765 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2766 break;
2767 case 0x4:
2768 Pred = ICmpInst::ICMP_EQ;
2769 break;
2770 case 0x5:
2771 Pred = ICmpInst::ICMP_NE;
2772 break;
2773 case 0x6:
2774 return Constant::getNullValue(Ty); // FALSE
2775 case 0x7:
2776 return Constant::getAllOnesValue(Ty); // TRUE
2777 default:
2778 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2779 }
2780
2781 Value *Cmp = Builder.CreateICmp(P: Pred, LHS, RHS);
2782 Value *Ext = Builder.CreateSExt(V: Cmp, DestTy: Ty);
2783 return Ext;
2784}
2785
2786static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2787 bool IsShiftRight, bool ZeroMask) {
2788 Type *Ty = CI.getType();
2789 Value *Op0 = CI.getArgOperand(i: 0);
2790 Value *Op1 = CI.getArgOperand(i: 1);
2791 Value *Amt = CI.getArgOperand(i: 2);
2792
2793 if (IsShiftRight)
2794 std::swap(a&: Op0, b&: Op1);
2795
2796 // Amount may be scalar immediate, in which case create a splat vector.
2797 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2798 // we only care about the lowest log2 bits anyway.
2799 if (Amt->getType() != Ty) {
2800 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2801 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2802 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2803 }
2804
2805 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2806 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1, Amt});
2807
2808 unsigned NumArgs = CI.arg_size();
2809 if (NumArgs >= 4) { // For masked intrinsics.
2810 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(i: 3) :
2811 ZeroMask ? ConstantAggregateZero::get(Ty: CI.getType()) :
2812 CI.getArgOperand(i: 0);
2813 Value *Mask = CI.getOperand(i_nocapture: NumArgs - 1);
2814 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2815 }
2816 return Res;
2817}
2818
2819static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2820 Value *Mask, bool Aligned) {
2821 const Align Alignment =
2822 Aligned
2823 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2824 : Align(1);
2825
2826 // If the mask is all ones just emit a regular store.
2827 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2828 if (C->isAllOnesValue())
2829 return Builder.CreateAlignedStore(Val: Data, Ptr, Align: Alignment);
2830
2831 // Convert the mask from an integer type to a vector of i1.
2832 unsigned NumElts = cast<FixedVectorType>(Val: Data->getType())->getNumElements();
2833 Mask = getX86MaskVec(Builder, Mask, NumElts);
2834 return Builder.CreateMaskedStore(Val: Data, Ptr, Alignment, Mask);
2835}
2836
2837static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2838 Value *Passthru, Value *Mask, bool Aligned) {
2839 Type *ValTy = Passthru->getType();
2840 const Align Alignment =
2841 Aligned
2842 ? Align(
2843 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2844 8)
2845 : Align(1);
2846
2847 // If the mask is all ones just emit a regular store.
2848 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2849 if (C->isAllOnesValue())
2850 return Builder.CreateAlignedLoad(Ty: ValTy, Ptr, Align: Alignment);
2851
2852 // Convert the mask from an integer type to a vector of i1.
2853 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
2854 Mask = getX86MaskVec(Builder, Mask, NumElts);
2855 return Builder.CreateMaskedLoad(Ty: ValTy, Ptr, Alignment, Mask, PassThru: Passthru);
2856}
2857
2858static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2859 Type *Ty = CI.getType();
2860 Value *Op0 = CI.getArgOperand(i: 0);
2861 Value *Res = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: Ty,
2862 Args: {Op0, Builder.getInt1(V: false)});
2863 if (CI.arg_size() == 3)
2864 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 2), Op0: Res, Op1: CI.getArgOperand(i: 1));
2865 return Res;
2866}
2867
2868static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2869 Type *Ty = CI.getType();
2870
2871 // Arguments have a vXi32 type so cast to vXi64.
2872 Value *LHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 0), DestTy: Ty);
2873 Value *RHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 1), DestTy: Ty);
2874
2875 if (IsSigned) {
2876 // Shift left then arithmetic shift right.
2877 Constant *ShiftAmt = ConstantInt::get(Ty, V: 32);
2878 LHS = Builder.CreateShl(LHS, RHS: ShiftAmt);
2879 LHS = Builder.CreateAShr(LHS, RHS: ShiftAmt);
2880 RHS = Builder.CreateShl(LHS: RHS, RHS: ShiftAmt);
2881 RHS = Builder.CreateAShr(LHS: RHS, RHS: ShiftAmt);
2882 } else {
2883 // Clear the upper bits.
2884 Constant *Mask = ConstantInt::get(Ty, V: 0xffffffff);
2885 LHS = Builder.CreateAnd(LHS, RHS: Mask);
2886 RHS = Builder.CreateAnd(LHS: RHS, RHS: Mask);
2887 }
2888
2889 Value *Res = Builder.CreateMul(LHS, RHS);
2890
2891 if (CI.arg_size() == 4)
2892 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Res, Op1: CI.getArgOperand(i: 2));
2893
2894 return Res;
2895}
2896
2897// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2898static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2899 Value *Mask) {
2900 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2901 if (Mask) {
2902 const auto *C = dyn_cast<Constant>(Val: Mask);
2903 if (!C || !C->isAllOnesValue())
2904 Vec = Builder.CreateAnd(LHS: Vec, RHS: getX86MaskVec(Builder, Mask, NumElts));
2905 }
2906
2907 if (NumElts < 8) {
2908 int Indices[8];
2909 for (unsigned i = 0; i != NumElts; ++i)
2910 Indices[i] = i;
2911 for (unsigned i = NumElts; i != 8; ++i)
2912 Indices[i] = NumElts + i % NumElts;
2913 Vec = Builder.CreateShuffleVector(V1: Vec,
2914 V2: Constant::getNullValue(Ty: Vec->getType()),
2915 Mask: Indices);
2916 }
2917 return Builder.CreateBitCast(V: Vec, DestTy: Builder.getIntNTy(N: std::max(a: NumElts, b: 8U)));
2918}
2919
2920static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2921 unsigned CC, bool Signed) {
2922 Value *Op0 = CI.getArgOperand(i: 0);
2923 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2924
2925 Value *Cmp;
2926 if (CC == 3) {
2927 Cmp = Constant::getNullValue(
2928 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2929 } else if (CC == 7) {
2930 Cmp = Constant::getAllOnesValue(
2931 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2932 } else {
2933 ICmpInst::Predicate Pred;
2934 switch (CC) {
2935 default: llvm_unreachable("Unknown condition code");
2936 case 0: Pred = ICmpInst::ICMP_EQ; break;
2937 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2938 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2939 case 4: Pred = ICmpInst::ICMP_NE; break;
2940 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2941 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2942 }
2943 Cmp = Builder.CreateICmp(P: Pred, LHS: Op0, RHS: CI.getArgOperand(i: 1));
2944 }
2945
2946 Value *Mask = CI.getArgOperand(i: CI.arg_size() - 1);
2947
2948 return applyX86MaskOn1BitsVec(Builder, Vec: Cmp, Mask);
2949}
2950
2951// Replace a masked intrinsic with an older unmasked intrinsic.
2952static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2953 Intrinsic::ID IID) {
2954 Value *Rep =
2955 Builder.CreateIntrinsic(ID: IID, Args: {CI.getArgOperand(i: 0), CI.getArgOperand(i: 1)});
2956 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Rep, Op1: CI.getArgOperand(i: 2));
2957}
2958
2959static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2960 Value* A = CI.getArgOperand(i: 0);
2961 Value* B = CI.getArgOperand(i: 1);
2962 Value* Src = CI.getArgOperand(i: 2);
2963 Value* Mask = CI.getArgOperand(i: 3);
2964
2965 Value* AndNode = Builder.CreateAnd(LHS: Mask, RHS: APInt(8, 1));
2966 Value* Cmp = Builder.CreateIsNotNull(Arg: AndNode);
2967 Value* Extract1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
2968 Value* Extract2 = Builder.CreateExtractElement(Vec: Src, Idx: (uint64_t)0);
2969 Value* Select = Builder.CreateSelect(C: Cmp, True: Extract1, False: Extract2);
2970 return Builder.CreateInsertElement(Vec: A, NewElt: Select, Idx: (uint64_t)0);
2971}
2972
2973static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2974 Value* Op = CI.getArgOperand(i: 0);
2975 Type* ReturnOp = CI.getType();
2976 unsigned NumElts = cast<FixedVectorType>(Val: CI.getType())->getNumElements();
2977 Value *Mask = getX86MaskVec(Builder, Mask: Op, NumElts);
2978 return Builder.CreateSExt(V: Mask, DestTy: ReturnOp, Name: "vpmovm2");
2979}
2980
2981// Replace intrinsic with unmasked version and a select.
2982static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2983 CallBase &CI, Value *&Rep) {
2984 Name = Name.substr(Start: 12); // Remove avx512.mask.
2985
2986 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2987 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2988 Intrinsic::ID IID;
2989 if (Name.starts_with(Prefix: "max.p")) {
2990 if (VecWidth == 128 && EltWidth == 32)
2991 IID = Intrinsic::x86_sse_max_ps;
2992 else if (VecWidth == 128 && EltWidth == 64)
2993 IID = Intrinsic::x86_sse2_max_pd;
2994 else if (VecWidth == 256 && EltWidth == 32)
2995 IID = Intrinsic::x86_avx_max_ps_256;
2996 else if (VecWidth == 256 && EltWidth == 64)
2997 IID = Intrinsic::x86_avx_max_pd_256;
2998 else
2999 llvm_unreachable("Unexpected intrinsic");
3000 } else if (Name.starts_with(Prefix: "min.p")) {
3001 if (VecWidth == 128 && EltWidth == 32)
3002 IID = Intrinsic::x86_sse_min_ps;
3003 else if (VecWidth == 128 && EltWidth == 64)
3004 IID = Intrinsic::x86_sse2_min_pd;
3005 else if (VecWidth == 256 && EltWidth == 32)
3006 IID = Intrinsic::x86_avx_min_ps_256;
3007 else if (VecWidth == 256 && EltWidth == 64)
3008 IID = Intrinsic::x86_avx_min_pd_256;
3009 else
3010 llvm_unreachable("Unexpected intrinsic");
3011 } else if (Name.starts_with(Prefix: "pshuf.b.")) {
3012 if (VecWidth == 128)
3013 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3014 else if (VecWidth == 256)
3015 IID = Intrinsic::x86_avx2_pshuf_b;
3016 else if (VecWidth == 512)
3017 IID = Intrinsic::x86_avx512_pshuf_b_512;
3018 else
3019 llvm_unreachable("Unexpected intrinsic");
3020 } else if (Name.starts_with(Prefix: "pmul.hr.sw.")) {
3021 if (VecWidth == 128)
3022 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3023 else if (VecWidth == 256)
3024 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3025 else if (VecWidth == 512)
3026 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3027 else
3028 llvm_unreachable("Unexpected intrinsic");
3029 } else if (Name.starts_with(Prefix: "pmulh.w.")) {
3030 if (VecWidth == 128)
3031 IID = Intrinsic::x86_sse2_pmulh_w;
3032 else if (VecWidth == 256)
3033 IID = Intrinsic::x86_avx2_pmulh_w;
3034 else if (VecWidth == 512)
3035 IID = Intrinsic::x86_avx512_pmulh_w_512;
3036 else
3037 llvm_unreachable("Unexpected intrinsic");
3038 } else if (Name.starts_with(Prefix: "pmulhu.w.")) {
3039 if (VecWidth == 128)
3040 IID = Intrinsic::x86_sse2_pmulhu_w;
3041 else if (VecWidth == 256)
3042 IID = Intrinsic::x86_avx2_pmulhu_w;
3043 else if (VecWidth == 512)
3044 IID = Intrinsic::x86_avx512_pmulhu_w_512;
3045 else
3046 llvm_unreachable("Unexpected intrinsic");
3047 } else if (Name.starts_with(Prefix: "pmaddw.d.")) {
3048 if (VecWidth == 128)
3049 IID = Intrinsic::x86_sse2_pmadd_wd;
3050 else if (VecWidth == 256)
3051 IID = Intrinsic::x86_avx2_pmadd_wd;
3052 else if (VecWidth == 512)
3053 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3054 else
3055 llvm_unreachable("Unexpected intrinsic");
3056 } else if (Name.starts_with(Prefix: "pmaddubs.w.")) {
3057 if (VecWidth == 128)
3058 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3059 else if (VecWidth == 256)
3060 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3061 else if (VecWidth == 512)
3062 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3063 else
3064 llvm_unreachable("Unexpected intrinsic");
3065 } else if (Name.starts_with(Prefix: "packsswb.")) {
3066 if (VecWidth == 128)
3067 IID = Intrinsic::x86_sse2_packsswb_128;
3068 else if (VecWidth == 256)
3069 IID = Intrinsic::x86_avx2_packsswb;
3070 else if (VecWidth == 512)
3071 IID = Intrinsic::x86_avx512_packsswb_512;
3072 else
3073 llvm_unreachable("Unexpected intrinsic");
3074 } else if (Name.starts_with(Prefix: "packssdw.")) {
3075 if (VecWidth == 128)
3076 IID = Intrinsic::x86_sse2_packssdw_128;
3077 else if (VecWidth == 256)
3078 IID = Intrinsic::x86_avx2_packssdw;
3079 else if (VecWidth == 512)
3080 IID = Intrinsic::x86_avx512_packssdw_512;
3081 else
3082 llvm_unreachable("Unexpected intrinsic");
3083 } else if (Name.starts_with(Prefix: "packuswb.")) {
3084 if (VecWidth == 128)
3085 IID = Intrinsic::x86_sse2_packuswb_128;
3086 else if (VecWidth == 256)
3087 IID = Intrinsic::x86_avx2_packuswb;
3088 else if (VecWidth == 512)
3089 IID = Intrinsic::x86_avx512_packuswb_512;
3090 else
3091 llvm_unreachable("Unexpected intrinsic");
3092 } else if (Name.starts_with(Prefix: "packusdw.")) {
3093 if (VecWidth == 128)
3094 IID = Intrinsic::x86_sse41_packusdw;
3095 else if (VecWidth == 256)
3096 IID = Intrinsic::x86_avx2_packusdw;
3097 else if (VecWidth == 512)
3098 IID = Intrinsic::x86_avx512_packusdw_512;
3099 else
3100 llvm_unreachable("Unexpected intrinsic");
3101 } else if (Name.starts_with(Prefix: "vpermilvar.")) {
3102 if (VecWidth == 128 && EltWidth == 32)
3103 IID = Intrinsic::x86_avx_vpermilvar_ps;
3104 else if (VecWidth == 128 && EltWidth == 64)
3105 IID = Intrinsic::x86_avx_vpermilvar_pd;
3106 else if (VecWidth == 256 && EltWidth == 32)
3107 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3108 else if (VecWidth == 256 && EltWidth == 64)
3109 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3110 else if (VecWidth == 512 && EltWidth == 32)
3111 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3112 else if (VecWidth == 512 && EltWidth == 64)
3113 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3114 else
3115 llvm_unreachable("Unexpected intrinsic");
3116 } else if (Name == "cvtpd2dq.256") {
3117 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3118 } else if (Name == "cvtpd2ps.256") {
3119 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3120 } else if (Name == "cvttpd2dq.256") {
3121 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3122 } else if (Name == "cvttps2dq.128") {
3123 IID = Intrinsic::x86_sse2_cvttps2dq;
3124 } else if (Name == "cvttps2dq.256") {
3125 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3126 } else if (Name.starts_with(Prefix: "permvar.")) {
3127 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3128 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3129 IID = Intrinsic::x86_avx2_permps;
3130 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3131 IID = Intrinsic::x86_avx2_permd;
3132 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3133 IID = Intrinsic::x86_avx512_permvar_df_256;
3134 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3135 IID = Intrinsic::x86_avx512_permvar_di_256;
3136 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3137 IID = Intrinsic::x86_avx512_permvar_sf_512;
3138 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3139 IID = Intrinsic::x86_avx512_permvar_si_512;
3140 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3141 IID = Intrinsic::x86_avx512_permvar_df_512;
3142 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3143 IID = Intrinsic::x86_avx512_permvar_di_512;
3144 else if (VecWidth == 128 && EltWidth == 16)
3145 IID = Intrinsic::x86_avx512_permvar_hi_128;
3146 else if (VecWidth == 256 && EltWidth == 16)
3147 IID = Intrinsic::x86_avx512_permvar_hi_256;
3148 else if (VecWidth == 512 && EltWidth == 16)
3149 IID = Intrinsic::x86_avx512_permvar_hi_512;
3150 else if (VecWidth == 128 && EltWidth == 8)
3151 IID = Intrinsic::x86_avx512_permvar_qi_128;
3152 else if (VecWidth == 256 && EltWidth == 8)
3153 IID = Intrinsic::x86_avx512_permvar_qi_256;
3154 else if (VecWidth == 512 && EltWidth == 8)
3155 IID = Intrinsic::x86_avx512_permvar_qi_512;
3156 else
3157 llvm_unreachable("Unexpected intrinsic");
3158 } else if (Name.starts_with(Prefix: "dbpsadbw.")) {
3159 if (VecWidth == 128)
3160 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3161 else if (VecWidth == 256)
3162 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3163 else if (VecWidth == 512)
3164 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3165 else
3166 llvm_unreachable("Unexpected intrinsic");
3167 } else if (Name.starts_with(Prefix: "pmultishift.qb.")) {
3168 if (VecWidth == 128)
3169 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3170 else if (VecWidth == 256)
3171 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3172 else if (VecWidth == 512)
3173 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3174 else
3175 llvm_unreachable("Unexpected intrinsic");
3176 } else if (Name.starts_with(Prefix: "conflict.")) {
3177 if (Name[9] == 'd' && VecWidth == 128)
3178 IID = Intrinsic::x86_avx512_conflict_d_128;
3179 else if (Name[9] == 'd' && VecWidth == 256)
3180 IID = Intrinsic::x86_avx512_conflict_d_256;
3181 else if (Name[9] == 'd' && VecWidth == 512)
3182 IID = Intrinsic::x86_avx512_conflict_d_512;
3183 else if (Name[9] == 'q' && VecWidth == 128)
3184 IID = Intrinsic::x86_avx512_conflict_q_128;
3185 else if (Name[9] == 'q' && VecWidth == 256)
3186 IID = Intrinsic::x86_avx512_conflict_q_256;
3187 else if (Name[9] == 'q' && VecWidth == 512)
3188 IID = Intrinsic::x86_avx512_conflict_q_512;
3189 else
3190 llvm_unreachable("Unexpected intrinsic");
3191 } else if (Name.starts_with(Prefix: "pavg.")) {
3192 if (Name[5] == 'b' && VecWidth == 128)
3193 IID = Intrinsic::x86_sse2_pavg_b;
3194 else if (Name[5] == 'b' && VecWidth == 256)
3195 IID = Intrinsic::x86_avx2_pavg_b;
3196 else if (Name[5] == 'b' && VecWidth == 512)
3197 IID = Intrinsic::x86_avx512_pavg_b_512;
3198 else if (Name[5] == 'w' && VecWidth == 128)
3199 IID = Intrinsic::x86_sse2_pavg_w;
3200 else if (Name[5] == 'w' && VecWidth == 256)
3201 IID = Intrinsic::x86_avx2_pavg_w;
3202 else if (Name[5] == 'w' && VecWidth == 512)
3203 IID = Intrinsic::x86_avx512_pavg_w_512;
3204 else
3205 llvm_unreachable("Unexpected intrinsic");
3206 } else
3207 return false;
3208
3209 SmallVector<Value *, 4> Args(CI.args());
3210 Args.pop_back();
3211 Args.pop_back();
3212 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3213 unsigned NumArgs = CI.arg_size();
3214 Rep = emitX86Select(Builder, Mask: CI.getArgOperand(i: NumArgs - 1), Op0: Rep,
3215 Op1: CI.getArgOperand(i: NumArgs - 2));
3216 return true;
3217}
3218
3219/// Upgrade comment in call to inline asm that represents an objc retain release
3220/// marker.
3221void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3222 size_t Pos;
3223 if (AsmStr->find(s: "mov\tfp") == 0 &&
3224 AsmStr->find(s: "objc_retainAutoreleaseReturnValue") != std::string::npos &&
3225 (Pos = AsmStr->find(s: "# marker")) != std::string::npos) {
3226 AsmStr->replace(pos: Pos, n1: 1, s: ";");
3227 }
3228}
3229
3230static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3231 Function *F, IRBuilder<> &Builder) {
3232 Value *Rep = nullptr;
3233
3234 if (Name == "abs.i" || Name == "abs.ll") {
3235 Value *Arg = CI->getArgOperand(i: 0);
3236 Rep = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: {Arg->getType()},
3237 Args: {Arg, Builder.getTrue()},
3238 /*FMFSource=*/nullptr, Name: "abs");
3239 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3240 Type *Ty = (Name == "abs.bf16")
3241 ? Builder.getBFloatTy()
3242 : FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2);
3243 Value *Arg = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Ty);
3244 Value *Abs = Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs, Op: Arg);
3245 Rep = Builder.CreateBitCast(V: Abs, DestTy: CI->getType());
3246 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3247 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3248 : Intrinsic::nvvm_fabs;
3249 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3250 } else if (Name.consume_front(Prefix: "add.")) {
3251 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3252 auto FAdd = getNVVMFAddUpgrade(Name);
3253 assert(FAdd && "unsupported nvvm.add.* intrinsic");
3254 auto [IID, RoundingMode] = *FAdd;
3255 Value *A = CI->getArgOperand(i: 0);
3256 Rep = Builder.CreateIntrinsic(
3257 RetTy: A->getType(), ID: IID,
3258 Args: {A, CI->getArgOperand(i: 1),
3259 Builder.getInt32(C: static_cast<int>(RoundingMode))});
3260 } else if (Name.consume_front(Prefix: "ex2.approx.")) {
3261 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3262 Intrinsic::ID IID = Name.starts_with(Prefix: "ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3263 : Intrinsic::nvvm_ex2_approx;
3264 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3265 } else if (Name.starts_with(Prefix: "atomic.load.add.f32.p") ||
3266 Name.starts_with(Prefix: "atomic.load.add.f64.p")) {
3267 Value *Ptr = CI->getArgOperand(i: 0);
3268 Value *Val = CI->getArgOperand(i: 1);
3269 Rep = Builder.CreateAtomicRMW(
3270 Op: AtomicRMWInst::FAdd, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3271 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3272 // The default scope for atomic.load.* intrinsics is device
3273 // (= gpu scope in ptx), but the default LLVM atomic scope is
3274 // "system"
3275 } else if (Name.starts_with(Prefix: "atomic.load.inc.32.p") ||
3276 Name.starts_with(Prefix: "atomic.load.dec.32.p")) {
3277 Value *Ptr = CI->getArgOperand(i: 0);
3278 Value *Val = CI->getArgOperand(i: 1);
3279 auto Op = Name.starts_with(Prefix: "atomic.load.inc") ? AtomicRMWInst::UIncWrap
3280 : AtomicRMWInst::UDecWrap;
3281 Rep = Builder.CreateAtomicRMW(
3282 Op, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3283 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3284 // See comment above.
3285 } else if (Name.starts_with(Prefix: "atomic.") && Name.contains(Other: ".gen.")) {
3286 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3287 StringRef Op = Name.substr(Start: StringRef("atomic.").size());
3288 Value *Ptr = CI->getArgOperand(i: 0);
3289 Value *Val = CI->getArgOperand(i: 1);
3290 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3291 SSN: Op.contains(Other: ".cta.") ? "block" : "");
3292 if (Op.starts_with(Prefix: "cas.")) {
3293 Value *New = CI->getArgOperand(i: 2);
3294 Value *Pair = Builder.CreateAtomicCmpXchg(
3295 Ptr, Cmp: Val, New, Align: MaybeAlign(), SuccessOrdering: AtomicOrdering::Monotonic,
3296 FailureOrdering: AtomicOrdering::Monotonic, SSID);
3297 Rep = Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
3298 } else {
3299 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3300 // because we were actually missing those intrinsics!
3301 AtomicRMWInst::BinOp BinOp =
3302 StringSwitch<AtomicRMWInst::BinOp>(Op)
3303 .StartsWith(S: "add.gen.f", Value: AtomicRMWInst::FAdd)
3304 .StartsWith(S: "add.gen.i", Value: AtomicRMWInst::Add)
3305 .StartsWith(S: "exch.", Value: AtomicRMWInst::Xchg)
3306 .StartsWith(S: "max.", Value: AtomicRMWInst::Max)
3307 .StartsWith(S: "min.", Value: AtomicRMWInst::Min)
3308 .StartsWith(S: "inc.", Value: AtomicRMWInst::UIncWrap)
3309 .StartsWith(S: "dec.", Value: AtomicRMWInst::UDecWrap)
3310 .StartsWith(S: "and.", Value: AtomicRMWInst::And)
3311 .StartsWith(S: "or.", Value: AtomicRMWInst::Or)
3312 .StartsWith(S: "xor.", Value: AtomicRMWInst::Xor)
3313 .Default(Value: AtomicRMWInst::BAD_BINOP);
3314 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3315 "unexpected nvvm scoped atomic intrinsic");
3316 Rep = Builder.CreateAtomicRMW(Op: BinOp, Ptr, Val, Align: MaybeAlign(),
3317 Ordering: AtomicOrdering::Monotonic, SSID);
3318 }
3319 } else if (Name == "clz.ll") {
3320 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3321 Value *Arg = CI->getArgOperand(i: 0);
3322 Value *Ctlz = Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {Arg->getType()},
3323 Args: {Arg, Builder.getFalse()},
3324 /*FMFSource=*/nullptr, Name: "ctlz");
3325 Rep = Builder.CreateTrunc(V: Ctlz, DestTy: Builder.getInt32Ty(), Name: "ctlz.trunc");
3326 } else if (Name == "popc.ll") {
3327 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3328 // i64.
3329 Value *Arg = CI->getArgOperand(i: 0);
3330 Value *Popc = Builder.CreateIntrinsic(ID: Intrinsic::ctpop, OverloadTypes: {Arg->getType()},
3331 Args: Arg, /*FMFSource=*/nullptr, Name: "ctpop");
3332 Rep = Builder.CreateTrunc(V: Popc, DestTy: Builder.getInt32Ty(), Name: "ctpop.trunc");
3333 } else if (Name == "h2f") {
3334 Value *Cast =
3335 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
3336 Rep = Builder.CreateFPExt(V: Cast, DestTy: Builder.getFloatTy());
3337 } else if (Name.consume_front(Prefix: "bitcast.") &&
3338 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3339 Name == "d2ll")) {
3340 Rep = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3341 } else if (Name == "rotate.b32") {
3342 Value *Arg = CI->getOperand(i_nocapture: 0);
3343 Value *ShiftAmt = CI->getOperand(i_nocapture: 1);
3344 Rep = Builder.CreateIntrinsic(RetTy: Builder.getInt32Ty(), ID: Intrinsic::fshl,
3345 Args: {Arg, Arg, ShiftAmt});
3346 } else if (Name == "rotate.b64") {
3347 Type *Int64Ty = Builder.getInt64Ty();
3348 Value *Arg = CI->getOperand(i_nocapture: 0);
3349 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3350 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3351 Args: {Arg, Arg, ZExtShiftAmt});
3352 } else if (Name == "rotate.right.b64") {
3353 Type *Int64Ty = Builder.getInt64Ty();
3354 Value *Arg = CI->getOperand(i_nocapture: 0);
3355 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3356 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshr,
3357 Args: {Arg, Arg, ZExtShiftAmt});
3358 } else if (Name == "swap.lo.hi.b64") {
3359 Type *Int64Ty = Builder.getInt64Ty();
3360 Value *Arg = CI->getOperand(i_nocapture: 0);
3361 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3362 Args: {Arg, Arg, Builder.getInt64(C: 32)});
3363 } else if ((Name.consume_front(Prefix: "ptr.gen.to.") &&
3364 consumeNVVMPtrAddrSpace(Name)) ||
3365 (Name.consume_front(Prefix: "ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3366 Name.starts_with(Prefix: ".to.gen"))) {
3367 Rep = Builder.CreateAddrSpaceCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3368 } else if (Name.consume_front(Prefix: "ldg.global")) {
3369 Value *Ptr = CI->getArgOperand(i: 0);
3370 Align PtrAlign = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getAlignValue();
3371 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3372 Value *ASC = Builder.CreateAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 1));
3373 Instruction *LD = Builder.CreateAlignedLoad(Ty: CI->getType(), Ptr: ASC, Align: PtrAlign);
3374 MDNode *MD = MDNode::get(Context&: Builder.getContext(), MDs: {});
3375 LD->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MD);
3376 return LD;
3377 } else if (Name == "tanh.approx.f32") {
3378 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3379 FastMathFlags FMF;
3380 FMF.setApproxFunc();
3381 Rep = Builder.CreateUnaryIntrinsic(ID: Intrinsic::tanh, Op: CI->getArgOperand(i: 0),
3382 FMFSource: FMF);
3383 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3384 Value *Arg =
3385 Name.ends_with(Suffix: '0') ? Builder.getInt32(C: 0) : CI->getArgOperand(i: 0);
3386 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3387 OverloadTypes: {}, Args: {Arg});
3388 } else if (Name == "barrier") {
3389 Rep = Builder.CreateIntrinsic(
3390 ID: Intrinsic::nvvm_barrier_cta_sync_aligned_count, OverloadTypes: {},
3391 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3392 } else if (Name == "barrier.sync") {
3393 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_all, OverloadTypes: {},
3394 Args: {CI->getArgOperand(i: 0)});
3395 } else if (Name == "barrier.sync.cnt") {
3396 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_count, OverloadTypes: {},
3397 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3398 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3399 Name == "barrier0.or") {
3400 Value *C = CI->getArgOperand(i: 0);
3401 C = Builder.CreateICmpNE(LHS: C, RHS: Builder.getInt32(C: 0));
3402
3403 Intrinsic::ID IID =
3404 StringSwitch<Intrinsic::ID>(Name)
3405 .Case(S: "barrier0.popc",
3406 Value: Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3407 .Case(S: "barrier0.and",
3408 Value: Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3409 .Case(S: "barrier0.or",
3410 Value: Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3411 Value *Bar = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {}, Args: {Builder.getInt32(C: 0), C});
3412 Rep = Builder.CreateZExt(V: Bar, DestTy: CI->getType());
3413 } else {
3414 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3415 if (IID != Intrinsic::not_intrinsic &&
3416 isLegacyNVPTXBF16IntSignature(F, IID)) {
3417 rename(GV: F);
3418 Function *NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
3419 SmallVector<Value *, 2> Args;
3420 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3421 Value *Arg = CI->getArgOperand(i: I);
3422 Type *OldType = Arg->getType();
3423 Type *NewType = NewFn->getArg(i: I)->getType();
3424 Args.push_back(
3425 Elt: (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3426 ? Builder.CreateBitCast(V: Arg, DestTy: NewType)
3427 : Arg);
3428 }
3429 Rep = Builder.CreateCall(Callee: NewFn, Args);
3430 if (F->getReturnType()->isIntegerTy())
3431 Rep = Builder.CreateBitCast(V: Rep, DestTy: F->getReturnType());
3432 }
3433 }
3434
3435 return Rep;
3436}
3437
3438static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3439 IRBuilder<> &Builder) {
3440 LLVMContext &C = F->getContext();
3441 Value *Rep = nullptr;
3442
3443 if (Name.starts_with(Prefix: "sse4a.movnt.")) {
3444 SmallVector<Metadata *, 1> Elts;
3445 Elts.push_back(
3446 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3447 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3448
3449 Value *Arg0 = CI->getArgOperand(i: 0);
3450 Value *Arg1 = CI->getArgOperand(i: 1);
3451
3452 // Nontemporal (unaligned) store of the 0'th element of the float/double
3453 // vector.
3454 Value *Extract =
3455 Builder.CreateExtractElement(Vec: Arg1, Idx: (uint64_t)0, Name: "extractelement");
3456
3457 StoreInst *SI = Builder.CreateAlignedStore(Val: Extract, Ptr: Arg0, Align: Align(1));
3458 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3459 } else if (Name.starts_with(Prefix: "avx.movnt.") ||
3460 Name.starts_with(Prefix: "avx512.storent.")) {
3461 SmallVector<Metadata *, 1> Elts;
3462 Elts.push_back(
3463 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3464 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3465
3466 Value *Arg0 = CI->getArgOperand(i: 0);
3467 Value *Arg1 = CI->getArgOperand(i: 1);
3468
3469 StoreInst *SI = Builder.CreateAlignedStore(
3470 Val: Arg1, Ptr: Arg0,
3471 Align: Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3472 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3473 } else if (Name == "sse2.storel.dq") {
3474 Value *Arg0 = CI->getArgOperand(i: 0);
3475 Value *Arg1 = CI->getArgOperand(i: 1);
3476
3477 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
3478 Value *BC0 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
3479 Value *Elt = Builder.CreateExtractElement(Vec: BC0, Idx: (uint64_t)0);
3480 Builder.CreateAlignedStore(Val: Elt, Ptr: Arg0, Align: Align(1));
3481 } else if (Name.starts_with(Prefix: "sse.storeu.") ||
3482 Name.starts_with(Prefix: "sse2.storeu.") ||
3483 Name.starts_with(Prefix: "avx.storeu.")) {
3484 Value *Arg0 = CI->getArgOperand(i: 0);
3485 Value *Arg1 = CI->getArgOperand(i: 1);
3486 Builder.CreateAlignedStore(Val: Arg1, Ptr: Arg0, Align: Align(1));
3487 } else if (Name == "avx512.mask.store.ss") {
3488 Value *Mask = Builder.CreateAnd(LHS: CI->getArgOperand(i: 2), RHS: Builder.getInt8(C: 1));
3489 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3490 Mask, Aligned: false);
3491 } else if (Name.starts_with(Prefix: "avx512.mask.store")) {
3492 // "avx512.mask.storeu." or "avx512.mask.store."
3493 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3494 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3495 Mask: CI->getArgOperand(i: 2), Aligned);
3496 } else if (Name.starts_with(Prefix: "sse2.pcmp") || Name.starts_with(Prefix: "avx2.pcmp")) {
3497 // Upgrade packed integer vector compare intrinsics to compare instructions.
3498 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3499 bool CmpEq = Name[9] == 'e';
3500 Rep = Builder.CreateICmp(P: CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3501 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
3502 Rep = Builder.CreateSExt(V: Rep, DestTy: CI->getType(), Name: "");
3503 } else if (Name.starts_with(Prefix: "avx512.broadcastm")) {
3504 Type *ExtTy = Type::getInt32Ty(C);
3505 if (CI->getOperand(i_nocapture: 0)->getType()->isIntegerTy(BitWidth: 8))
3506 ExtTy = Type::getInt64Ty(C);
3507 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3508 ExtTy->getPrimitiveSizeInBits();
3509 Rep = Builder.CreateZExt(V: CI->getArgOperand(i: 0), DestTy: ExtTy);
3510 Rep = Builder.CreateVectorSplat(NumElts, V: Rep);
3511 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3512 Value *Vec = CI->getArgOperand(i: 0);
3513 Value *Elt0 = Builder.CreateExtractElement(Vec, Idx: (uint64_t)0);
3514 Elt0 = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: Elt0->getType(), Args: Elt0);
3515 Rep = Builder.CreateInsertElement(Vec, NewElt: Elt0, Idx: (uint64_t)0);
3516 } else if (Name.starts_with(Prefix: "avx.sqrt.p") ||
3517 Name.starts_with(Prefix: "sse2.sqrt.p") ||
3518 Name.starts_with(Prefix: "sse.sqrt.p")) {
3519 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3520 Args: {CI->getArgOperand(i: 0)});
3521 } else if (Name.starts_with(Prefix: "avx512.mask.sqrt.p")) {
3522 if (CI->arg_size() == 4 &&
3523 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3524 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3525 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3526 : Intrinsic::x86_avx512_sqrt_pd_512;
3527
3528 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 3)};
3529 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3530 } else {
3531 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3532 Args: {CI->getArgOperand(i: 0)});
3533 }
3534 Rep =
3535 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3536 } else if (Name.starts_with(Prefix: "avx512.ptestm") ||
3537 Name.starts_with(Prefix: "avx512.ptestnm")) {
3538 Value *Op0 = CI->getArgOperand(i: 0);
3539 Value *Op1 = CI->getArgOperand(i: 1);
3540 Value *Mask = CI->getArgOperand(i: 2);
3541 Rep = Builder.CreateAnd(LHS: Op0, RHS: Op1);
3542 llvm::Type *Ty = Op0->getType();
3543 Value *Zero = llvm::Constant::getNullValue(Ty);
3544 ICmpInst::Predicate Pred = Name.starts_with(Prefix: "avx512.ptestm")
3545 ? ICmpInst::ICMP_NE
3546 : ICmpInst::ICMP_EQ;
3547 Rep = Builder.CreateICmp(P: Pred, LHS: Rep, RHS: Zero);
3548 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask);
3549 } else if (Name.starts_with(Prefix: "avx512.mask.pbroadcast")) {
3550 unsigned NumElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 1)->getType())
3551 ->getNumElements();
3552 Rep = Builder.CreateVectorSplat(NumElts, V: CI->getArgOperand(i: 0));
3553 Rep =
3554 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3555 } else if (Name.starts_with(Prefix: "avx512.kunpck")) {
3556 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3557 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts);
3558 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts);
3559 int Indices[64];
3560 for (unsigned i = 0; i != NumElts; ++i)
3561 Indices[i] = i;
3562
3563 // First extract half of each vector. This gives better codegen than
3564 // doing it in a single shuffle.
3565 LHS = Builder.CreateShuffleVector(V1: LHS, V2: LHS, Mask: ArrayRef(Indices, NumElts / 2));
3566 RHS = Builder.CreateShuffleVector(V1: RHS, V2: RHS, Mask: ArrayRef(Indices, NumElts / 2));
3567 // Concat the vectors.
3568 // NOTE: Operands have to be swapped to match intrinsic definition.
3569 Rep = Builder.CreateShuffleVector(V1: RHS, V2: LHS, Mask: ArrayRef(Indices, NumElts));
3570 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3571 } else if (Name == "avx512.kand.w") {
3572 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3573 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3574 Rep = Builder.CreateAnd(LHS, RHS);
3575 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3576 } else if (Name == "avx512.kandn.w") {
3577 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3578 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3579 LHS = Builder.CreateNot(V: LHS);
3580 Rep = Builder.CreateAnd(LHS, RHS);
3581 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3582 } else if (Name == "avx512.kor.w") {
3583 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3584 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3585 Rep = Builder.CreateOr(LHS, RHS);
3586 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3587 } else if (Name == "avx512.kxor.w") {
3588 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3589 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3590 Rep = Builder.CreateXor(LHS, RHS);
3591 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3592 } else if (Name == "avx512.kxnor.w") {
3593 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3594 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3595 LHS = Builder.CreateNot(V: LHS);
3596 Rep = Builder.CreateXor(LHS, RHS);
3597 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3598 } else if (Name == "avx512.knot.w") {
3599 Rep = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3600 Rep = Builder.CreateNot(V: Rep);
3601 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3602 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3603 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3604 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3605 Rep = Builder.CreateOr(LHS, RHS);
3606 Rep = Builder.CreateBitCast(V: Rep, DestTy: Builder.getInt16Ty());
3607 Value *C;
3608 if (Name[14] == 'c')
3609 C = ConstantInt::getAllOnesValue(Ty: Builder.getInt16Ty());
3610 else
3611 C = ConstantInt::getNullValue(Ty: Builder.getInt16Ty());
3612 Rep = Builder.CreateICmpEQ(LHS: Rep, RHS: C);
3613 Rep = Builder.CreateZExt(V: Rep, DestTy: Builder.getInt32Ty());
3614 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3615 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3616 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3617 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3618 Type *I32Ty = Type::getInt32Ty(C);
3619 Value *Elt0 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 0),
3620 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3621 Value *Elt1 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1),
3622 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3623 Value *EltOp;
3624 if (Name.contains(Other: ".add."))
3625 EltOp = Builder.CreateFAdd(L: Elt0, R: Elt1);
3626 else if (Name.contains(Other: ".sub."))
3627 EltOp = Builder.CreateFSub(L: Elt0, R: Elt1);
3628 else if (Name.contains(Other: ".mul."))
3629 EltOp = Builder.CreateFMul(L: Elt0, R: Elt1);
3630 else
3631 EltOp = Builder.CreateFDiv(L: Elt0, R: Elt1);
3632 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: EltOp,
3633 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3634 } else if (Name.starts_with(Prefix: "avx512.mask.pcmp")) {
3635 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3636 bool CmpEq = Name[16] == 'e';
3637 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: CmpEq ? 0 : 6, Signed: true);
3638 } else if (Name.starts_with(Prefix: "avx512.mask.vpshufbitqmb.")) {
3639 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3640 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3641 Intrinsic::ID IID;
3642 switch (VecWidth) {
3643 default:
3644 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3645 break;
3646 case 128:
3647 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3648 break;
3649 case 256:
3650 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3651 break;
3652 case 512:
3653 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3654 break;
3655 }
3656
3657 Rep =
3658 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3659 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3660 } else if (Name.starts_with(Prefix: "avx512.mask.fpclass.p")) {
3661 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3662 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3663 unsigned EltWidth = OpTy->getScalarSizeInBits();
3664 Intrinsic::ID IID;
3665 if (VecWidth == 128 && EltWidth == 32)
3666 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3667 else if (VecWidth == 256 && EltWidth == 32)
3668 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3669 else if (VecWidth == 512 && EltWidth == 32)
3670 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3671 else if (VecWidth == 128 && EltWidth == 64)
3672 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3673 else if (VecWidth == 256 && EltWidth == 64)
3674 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3675 else if (VecWidth == 512 && EltWidth == 64)
3676 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3677 else
3678 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3679
3680 Rep =
3681 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3682 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3683 } else if (Name.starts_with(Prefix: "avx512.cmp.p")) {
3684 SmallVector<Value *, 4> Args(CI->args());
3685 Type *OpTy = Args[0]->getType();
3686 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3687 unsigned EltWidth = OpTy->getScalarSizeInBits();
3688 Intrinsic::ID IID;
3689 if (VecWidth == 128 && EltWidth == 32)
3690 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3691 else if (VecWidth == 256 && EltWidth == 32)
3692 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3693 else if (VecWidth == 512 && EltWidth == 32)
3694 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3695 else if (VecWidth == 128 && EltWidth == 64)
3696 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3697 else if (VecWidth == 256 && EltWidth == 64)
3698 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3699 else if (VecWidth == 512 && EltWidth == 64)
3700 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3701 else
3702 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3703
3704 Value *Mask = Constant::getAllOnesValue(Ty: CI->getType());
3705 if (VecWidth == 512)
3706 std::swap(a&: Mask, b&: Args.back());
3707 Args.push_back(Elt: Mask);
3708
3709 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3710 } else if (Name.starts_with(Prefix: "avx512.mask.cmp.")) {
3711 // Integer compare intrinsics.
3712 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3713 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: true);
3714 } else if (Name.starts_with(Prefix: "avx512.mask.ucmp.")) {
3715 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3716 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: false);
3717 } else if (Name.starts_with(Prefix: "avx512.cvtb2mask.") ||
3718 Name.starts_with(Prefix: "avx512.cvtw2mask.") ||
3719 Name.starts_with(Prefix: "avx512.cvtd2mask.") ||
3720 Name.starts_with(Prefix: "avx512.cvtq2mask.")) {
3721 Value *Op = CI->getArgOperand(i: 0);
3722 Value *Zero = llvm::Constant::getNullValue(Ty: Op->getType());
3723 Rep = Builder.CreateICmp(P: ICmpInst::ICMP_SLT, LHS: Op, RHS: Zero);
3724 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: nullptr);
3725 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3726 Name == "ssse3.pabs.d.128" || Name.starts_with(Prefix: "avx2.pabs") ||
3727 Name.starts_with(Prefix: "avx512.mask.pabs")) {
3728 Rep = upgradeAbs(Builder, CI&: *CI);
3729 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3730 Name == "sse41.pmaxsd" || Name.starts_with(Prefix: "avx2.pmaxs") ||
3731 Name.starts_with(Prefix: "avx512.mask.pmaxs")) {
3732 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smax);
3733 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3734 Name == "sse41.pmaxud" || Name.starts_with(Prefix: "avx2.pmaxu") ||
3735 Name.starts_with(Prefix: "avx512.mask.pmaxu")) {
3736 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umax);
3737 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3738 Name == "sse41.pminsd" || Name.starts_with(Prefix: "avx2.pmins") ||
3739 Name.starts_with(Prefix: "avx512.mask.pmins")) {
3740 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smin);
3741 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3742 Name == "sse41.pminud" || Name.starts_with(Prefix: "avx2.pminu") ||
3743 Name.starts_with(Prefix: "avx512.mask.pminu")) {
3744 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umin);
3745 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3746 Name == "avx512.pmulu.dq.512" ||
3747 Name.starts_with(Prefix: "avx512.mask.pmulu.dq.")) {
3748 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: false);
3749 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3750 Name == "avx512.pmul.dq.512" ||
3751 Name.starts_with(Prefix: "avx512.mask.pmul.dq.")) {
3752 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: true);
3753 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3754 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3755 Rep =
3756 Builder.CreateSIToFP(V: CI->getArgOperand(i: 1),
3757 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3758 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3759 } else if (Name == "avx512.cvtusi2sd") {
3760 Rep =
3761 Builder.CreateUIToFP(V: CI->getArgOperand(i: 1),
3762 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3763 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3764 } else if (Name == "sse2.cvtss2sd") {
3765 Rep = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1), Idx: (uint64_t)0);
3766 Rep = Builder.CreateFPExt(
3767 V: Rep, DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3768 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3769 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3770 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3771 Name.starts_with(Prefix: "avx512.mask.cvtdq2pd.") ||
3772 Name.starts_with(Prefix: "avx512.mask.cvtudq2pd.") ||
3773 Name.starts_with(Prefix: "avx512.mask.cvtdq2ps.") ||
3774 Name.starts_with(Prefix: "avx512.mask.cvtudq2ps.") ||
3775 Name.starts_with(Prefix: "avx512.mask.cvtqq2pd.") ||
3776 Name.starts_with(Prefix: "avx512.mask.cvtuqq2pd.") ||
3777 Name == "avx512.mask.cvtqq2ps.256" ||
3778 Name == "avx512.mask.cvtqq2ps.512" ||
3779 Name == "avx512.mask.cvtuqq2ps.256" ||
3780 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3781 Name == "avx.cvt.ps2.pd.256" ||
3782 Name == "avx512.mask.cvtps2pd.128" ||
3783 Name == "avx512.mask.cvtps2pd.256") {
3784 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3785 Rep = CI->getArgOperand(i: 0);
3786 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3787
3788 unsigned NumDstElts = DstTy->getNumElements();
3789 if (NumDstElts < SrcTy->getNumElements()) {
3790 assert(NumDstElts == 2 && "Unexpected vector size");
3791 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1});
3792 }
3793
3794 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3795 bool IsUnsigned = Name.contains(Other: "cvtu");
3796 if (IsPS2PD)
3797 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtps2pd");
3798 else if (CI->arg_size() == 4 &&
3799 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3800 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3801 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3802 : Intrinsic::x86_avx512_sitofp_round;
3803 Rep = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {DstTy, SrcTy},
3804 Args: {Rep, CI->getArgOperand(i: 3)});
3805 } else {
3806 Rep = IsUnsigned ? Builder.CreateUIToFP(V: Rep, DestTy: DstTy, Name: "cvt")
3807 : Builder.CreateSIToFP(V: Rep, DestTy: DstTy, Name: "cvt");
3808 }
3809
3810 if (CI->arg_size() >= 3)
3811 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3812 Op1: CI->getArgOperand(i: 1));
3813 } else if (Name.starts_with(Prefix: "avx512.mask.vcvtph2ps.") ||
3814 Name.starts_with(Prefix: "vcvtph2ps.")) {
3815 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3816 Rep = CI->getArgOperand(i: 0);
3817 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3818 unsigned NumDstElts = DstTy->getNumElements();
3819 if (NumDstElts != SrcTy->getNumElements()) {
3820 assert(NumDstElts == 4 && "Unexpected vector size");
3821 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1, 2, 3});
3822 }
3823 Rep = Builder.CreateBitCast(
3824 V: Rep, DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C), NumElts: NumDstElts));
3825 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtph2ps");
3826 if (CI->arg_size() >= 3)
3827 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3828 Op1: CI->getArgOperand(i: 1));
3829 } else if (Name.starts_with(Prefix: "avx512.mask.load")) {
3830 // "avx512.mask.loadu." or "avx512.mask.load."
3831 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3832 Rep = upgradeMaskedLoad(Builder, Ptr: CI->getArgOperand(i: 0), Passthru: CI->getArgOperand(i: 1),
3833 Mask: CI->getArgOperand(i: 2), Aligned);
3834 } else if (Name.starts_with(Prefix: "avx512.mask.expand.load.")) {
3835 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3836 auto *PtrTy = CI->getOperand(i_nocapture: 0)->getType();
3837 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3838 NumElts: ResultTy->getNumElements());
3839 Rep = Builder.CreateIntrinsic(
3840 ID: Intrinsic::masked_expandload, OverloadTypes: {ResultTy, PtrTy},
3841 Args: {CI->getOperand(i_nocapture: 0), MaskVec, CI->getOperand(i_nocapture: 1)});
3842 } else if (Name.starts_with(Prefix: "avx512.mask.compress.store.")) {
3843 auto *ResultTy = cast<VectorType>(Val: CI->getArgOperand(i: 1)->getType());
3844 auto *PtrTy = CI->getArgOperand(i: 0)->getType();
3845 Value *MaskVec =
3846 getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3847 NumElts: cast<FixedVectorType>(Val: ResultTy)->getNumElements());
3848 Rep = Builder.CreateIntrinsic(
3849 ID: Intrinsic::masked_compressstore, OverloadTypes: {ResultTy, PtrTy},
3850 Args: {CI->getArgOperand(i: 1), CI->getArgOperand(i: 0), MaskVec});
3851 } else if (Name.starts_with(Prefix: "avx512.mask.compress.") ||
3852 Name.starts_with(Prefix: "avx512.mask.expand.")) {
3853 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3854
3855 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3856 NumElts: ResultTy->getNumElements());
3857
3858 bool IsCompress = Name[12] == 'c';
3859 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3860 : Intrinsic::x86_avx512_mask_expand;
3861 Rep = Builder.CreateIntrinsic(
3862 ID: IID, OverloadTypes: ResultTy, Args: {CI->getOperand(i_nocapture: 0), CI->getOperand(i_nocapture: 1), MaskVec});
3863 } else if (Name.starts_with(Prefix: "xop.vpcom")) {
3864 bool IsSigned;
3865 if (Name.ends_with(Suffix: "ub") || Name.ends_with(Suffix: "uw") || Name.ends_with(Suffix: "ud") ||
3866 Name.ends_with(Suffix: "uq"))
3867 IsSigned = false;
3868 else if (Name.ends_with(Suffix: "b") || Name.ends_with(Suffix: "w") ||
3869 Name.ends_with(Suffix: "d") || Name.ends_with(Suffix: "q"))
3870 IsSigned = true;
3871 else
3872 reportFatalUsageErrorWithCI(reason: "Intrinsic has unknown suffix", CI);
3873
3874 unsigned Imm;
3875 if (CI->arg_size() == 3) {
3876 Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3877 } else {
3878 Name = Name.substr(Start: 9); // strip off "xop.vpcom"
3879 if (Name.starts_with(Prefix: "lt"))
3880 Imm = 0;
3881 else if (Name.starts_with(Prefix: "le"))
3882 Imm = 1;
3883 else if (Name.starts_with(Prefix: "gt"))
3884 Imm = 2;
3885 else if (Name.starts_with(Prefix: "ge"))
3886 Imm = 3;
3887 else if (Name.starts_with(Prefix: "eq"))
3888 Imm = 4;
3889 else if (Name.starts_with(Prefix: "ne"))
3890 Imm = 5;
3891 else if (Name.starts_with(Prefix: "false"))
3892 Imm = 6;
3893 else if (Name.starts_with(Prefix: "true"))
3894 Imm = 7;
3895 else
3896 llvm_unreachable("Unknown condition");
3897 }
3898
3899 Rep = upgradeX86vpcom(Builder, CI&: *CI, Imm, IsSigned);
3900 } else if (Name.starts_with(Prefix: "xop.vpcmov")) {
3901 Value *Sel = CI->getArgOperand(i: 2);
3902 Value *NotSel = Builder.CreateNot(V: Sel);
3903 Value *Sel0 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 0), RHS: Sel);
3904 Value *Sel1 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 1), RHS: NotSel);
3905 Rep = Builder.CreateOr(LHS: Sel0, RHS: Sel1);
3906 } else if (Name.starts_with(Prefix: "xop.vprot") || Name.starts_with(Prefix: "avx512.prol") ||
3907 Name.starts_with(Prefix: "avx512.mask.prol")) {
3908 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: false);
3909 } else if (Name.starts_with(Prefix: "avx512.pror") ||
3910 Name.starts_with(Prefix: "avx512.mask.pror")) {
3911 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: true);
3912 } else if (Name.starts_with(Prefix: "avx512.vpshld.") ||
3913 Name.starts_with(Prefix: "avx512.mask.vpshld") ||
3914 Name.starts_with(Prefix: "avx512.maskz.vpshld")) {
3915 bool ZeroMask = Name[11] == 'z';
3916 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: false, ZeroMask);
3917 } else if (Name.starts_with(Prefix: "avx512.vpshrd.") ||
3918 Name.starts_with(Prefix: "avx512.mask.vpshrd") ||
3919 Name.starts_with(Prefix: "avx512.maskz.vpshrd")) {
3920 bool ZeroMask = Name[11] == 'z';
3921 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: true, ZeroMask);
3922 } else if (Name == "sse42.crc32.64.8") {
3923 Value *Trunc0 =
3924 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Type::getInt32Ty(C));
3925 Rep = Builder.CreateIntrinsic(ID: Intrinsic::x86_sse42_crc32_32_8,
3926 Args: {Trunc0, CI->getArgOperand(i: 1)});
3927 Rep = Builder.CreateZExt(V: Rep, DestTy: CI->getType(), Name: "");
3928 } else if (Name.starts_with(Prefix: "avx.vbroadcast.s") ||
3929 Name.starts_with(Prefix: "avx512.vbroadcast.s")) {
3930 // Replace broadcasts with a series of insertelements.
3931 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3932 Type *EltTy = VecTy->getElementType();
3933 unsigned EltNum = VecTy->getNumElements();
3934 Value *Load = Builder.CreateLoad(Ty: EltTy, Ptr: CI->getArgOperand(i: 0));
3935 Type *I32Ty = Type::getInt32Ty(C);
3936 Rep = PoisonValue::get(T: VecTy);
3937 for (unsigned I = 0; I < EltNum; ++I)
3938 Rep = Builder.CreateInsertElement(Vec: Rep, NewElt: Load, Idx: ConstantInt::get(Ty: I32Ty, V: I));
3939 } else if (Name.starts_with(Prefix: "sse41.pmovsx") ||
3940 Name.starts_with(Prefix: "sse41.pmovzx") ||
3941 Name.starts_with(Prefix: "avx2.pmovsx") ||
3942 Name.starts_with(Prefix: "avx2.pmovzx") ||
3943 Name.starts_with(Prefix: "avx512.mask.pmovsx") ||
3944 Name.starts_with(Prefix: "avx512.mask.pmovzx")) {
3945 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3946 unsigned NumDstElts = DstTy->getNumElements();
3947
3948 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3949 SmallVector<int, 8> ShuffleMask(NumDstElts);
3950 for (unsigned i = 0; i != NumDstElts; ++i)
3951 ShuffleMask[i] = i;
3952
3953 Value *SV = Builder.CreateShuffleVector(V: CI->getArgOperand(i: 0), Mask: ShuffleMask);
3954
3955 bool DoSext = Name.contains(Other: "pmovsx");
3956 Rep =
3957 DoSext ? Builder.CreateSExt(V: SV, DestTy: DstTy) : Builder.CreateZExt(V: SV, DestTy: DstTy);
3958 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3959 if (CI->arg_size() == 3)
3960 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3961 Op1: CI->getArgOperand(i: 1));
3962 } else if (Name == "avx512.mask.pmov.qd.256" ||
3963 Name == "avx512.mask.pmov.qd.512" ||
3964 Name == "avx512.mask.pmov.wb.256" ||
3965 Name == "avx512.mask.pmov.wb.512") {
3966 Type *Ty = CI->getArgOperand(i: 1)->getType();
3967 Rep = Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Ty);
3968 Rep =
3969 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3970 } else if (Name.starts_with(Prefix: "avx.vbroadcastf128") ||
3971 Name == "avx2.vbroadcasti128") {
3972 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3973 Type *EltTy = cast<VectorType>(Val: CI->getType())->getElementType();
3974 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3975 auto *VT = FixedVectorType::get(ElementType: EltTy, NumElts: NumSrcElts);
3976 Value *Load = Builder.CreateAlignedLoad(Ty: VT, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
3977 if (NumSrcElts == 2)
3978 Rep = Builder.CreateShuffleVector(V: Load, Mask: ArrayRef<int>{0, 1, 0, 1});
3979 else
3980 Rep = Builder.CreateShuffleVector(V: Load,
3981 Mask: ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3982 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.i") ||
3983 Name.starts_with(Prefix: "avx512.mask.shuf.f")) {
3984 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3985 Type *VT = CI->getType();
3986 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3987 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3988 unsigned ControlBitsMask = NumLanes - 1;
3989 unsigned NumControlBits = NumLanes / 2;
3990 SmallVector<int, 8> ShuffleMask(0);
3991
3992 for (unsigned l = 0; l != NumLanes; ++l) {
3993 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3994 // We actually need the other source.
3995 if (l >= NumLanes / 2)
3996 LaneMask += NumLanes;
3997 for (unsigned i = 0; i != NumElementsInLane; ++i)
3998 ShuffleMask.push_back(Elt: LaneMask * NumElementsInLane + i);
3999 }
4000 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
4001 V2: CI->getArgOperand(i: 1), Mask: ShuffleMask);
4002 Rep =
4003 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
4004 } else if (Name.starts_with(Prefix: "avx512.mask.broadcastf") ||
4005 Name.starts_with(Prefix: "avx512.mask.broadcasti")) {
4006 unsigned NumSrcElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 0)->getType())
4007 ->getNumElements();
4008 unsigned NumDstElts =
4009 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4010
4011 SmallVector<int, 8> ShuffleMask(NumDstElts);
4012 for (unsigned i = 0; i != NumDstElts; ++i)
4013 ShuffleMask[i] = i % NumSrcElts;
4014
4015 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
4016 V2: CI->getArgOperand(i: 0), Mask: ShuffleMask);
4017 Rep =
4018 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4019 } else if (Name.starts_with(Prefix: "avx2.pbroadcast") ||
4020 Name.starts_with(Prefix: "avx2.vbroadcast") ||
4021 Name.starts_with(Prefix: "avx512.pbroadcast") ||
4022 Name.starts_with(Prefix: "avx512.mask.broadcast.s")) {
4023 // Replace vp?broadcasts with a vector shuffle.
4024 Value *Op = CI->getArgOperand(i: 0);
4025 ElementCount EC = cast<VectorType>(Val: CI->getType())->getElementCount();
4026 Type *MaskTy = VectorType::get(ElementType: Type::getInt32Ty(C), EC);
4027 SmallVector<int, 8> M;
4028 ShuffleVectorInst::getShuffleMask(Mask: Constant::getNullValue(Ty: MaskTy), Result&: M);
4029 Rep = Builder.CreateShuffleVector(V: Op, Mask: M);
4030
4031 if (CI->arg_size() == 3)
4032 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
4033 Op1: CI->getArgOperand(i: 1));
4034 } else if (Name.starts_with(Prefix: "sse2.padds.") ||
4035 Name.starts_with(Prefix: "avx2.padds.") ||
4036 Name.starts_with(Prefix: "avx512.padds.") ||
4037 Name.starts_with(Prefix: "avx512.mask.padds.")) {
4038 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::sadd_sat);
4039 } else if (Name.starts_with(Prefix: "sse2.psubs.") ||
4040 Name.starts_with(Prefix: "avx2.psubs.") ||
4041 Name.starts_with(Prefix: "avx512.psubs.") ||
4042 Name.starts_with(Prefix: "avx512.mask.psubs.")) {
4043 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::ssub_sat);
4044 } else if (Name.starts_with(Prefix: "sse2.paddus.") ||
4045 Name.starts_with(Prefix: "avx2.paddus.") ||
4046 Name.starts_with(Prefix: "avx512.mask.paddus.")) {
4047 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::uadd_sat);
4048 } else if (Name.starts_with(Prefix: "sse2.psubus.") ||
4049 Name.starts_with(Prefix: "avx2.psubus.") ||
4050 Name.starts_with(Prefix: "avx512.mask.psubus.")) {
4051 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::usub_sat);
4052 } else if (Name.starts_with(Prefix: "avx512.mask.palignr.")) {
4053 Rep = upgradeX86ALIGNIntrinsics(Builder, Op0: CI->getArgOperand(i: 0),
4054 Op1: CI->getArgOperand(i: 1), Shift: CI->getArgOperand(i: 2),
4055 Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4),
4056 IsVALIGN: false);
4057 } else if (Name.starts_with(Prefix: "avx512.mask.valign.")) {
4058 Rep = upgradeX86ALIGNIntrinsics(
4059 Builder, Op0: CI->getArgOperand(i: 0), Op1: CI->getArgOperand(i: 1),
4060 Shift: CI->getArgOperand(i: 2), Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4), IsVALIGN: true);
4061 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4062 // 128/256-bit shift left specified in bits.
4063 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4064 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
4065 Shift: Shift / 8); // Shift is in bits.
4066 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4067 // 128/256-bit shift right specified in bits.
4068 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4069 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
4070 Shift: Shift / 8); // Shift is in bits.
4071 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4072 Name == "avx512.psll.dq.512") {
4073 // 128/256/512-bit shift left specified in bytes.
4074 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4075 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
4076 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4077 Name == "avx512.psrl.dq.512") {
4078 // 128/256/512-bit shift right specified in bytes.
4079 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4080 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
4081 } else if (Name == "sse41.pblendw" || Name.starts_with(Prefix: "sse41.blendp") ||
4082 Name.starts_with(Prefix: "avx.blend.p") || Name == "avx2.pblendw" ||
4083 Name.starts_with(Prefix: "avx2.pblendd.")) {
4084 Value *Op0 = CI->getArgOperand(i: 0);
4085 Value *Op1 = CI->getArgOperand(i: 1);
4086 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4087 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4088 unsigned NumElts = VecTy->getNumElements();
4089
4090 SmallVector<int, 16> Idxs(NumElts);
4091 for (unsigned i = 0; i != NumElts; ++i)
4092 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4093
4094 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4095 } else if (Name.starts_with(Prefix: "avx.vinsertf128.") ||
4096 Name == "avx2.vinserti128" ||
4097 Name.starts_with(Prefix: "avx512.mask.insert")) {
4098 Value *Op0 = CI->getArgOperand(i: 0);
4099 Value *Op1 = CI->getArgOperand(i: 1);
4100 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4101 unsigned DstNumElts =
4102 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4103 unsigned SrcNumElts =
4104 cast<FixedVectorType>(Val: Op1->getType())->getNumElements();
4105 unsigned Scale = DstNumElts / SrcNumElts;
4106
4107 // Mask off the high bits of the immediate value; hardware ignores those.
4108 Imm = Imm % Scale;
4109
4110 // Extend the second operand into a vector the size of the destination.
4111 SmallVector<int, 8> Idxs(DstNumElts);
4112 for (unsigned i = 0; i != SrcNumElts; ++i)
4113 Idxs[i] = i;
4114 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4115 Idxs[i] = SrcNumElts;
4116 Rep = Builder.CreateShuffleVector(V: Op1, Mask: Idxs);
4117
4118 // Insert the second operand into the first operand.
4119
4120 // Note that there is no guarantee that instruction lowering will actually
4121 // produce a vinsertf128 instruction for the created shuffles. In
4122 // particular, the 0 immediate case involves no lane changes, so it can
4123 // be handled as a blend.
4124
4125 // Example of shuffle mask for 32-bit elements:
4126 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4127 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4128
4129 // First fill with identify mask.
4130 for (unsigned i = 0; i != DstNumElts; ++i)
4131 Idxs[i] = i;
4132 // Then replace the elements where we need to insert.
4133 for (unsigned i = 0; i != SrcNumElts; ++i)
4134 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4135 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Rep, Mask: Idxs);
4136
4137 // If the intrinsic has a mask operand, handle that.
4138 if (CI->arg_size() == 5)
4139 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep,
4140 Op1: CI->getArgOperand(i: 3));
4141 } else if (Name.starts_with(Prefix: "avx.vextractf128.") ||
4142 Name == "avx2.vextracti128" ||
4143 Name.starts_with(Prefix: "avx512.mask.vextract")) {
4144 Value *Op0 = CI->getArgOperand(i: 0);
4145 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4146 unsigned DstNumElts =
4147 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4148 unsigned SrcNumElts =
4149 cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
4150 unsigned Scale = SrcNumElts / DstNumElts;
4151
4152 // Mask off the high bits of the immediate value; hardware ignores those.
4153 Imm = Imm % Scale;
4154
4155 // Get indexes for the subvector of the input vector.
4156 SmallVector<int, 8> Idxs(DstNumElts);
4157 for (unsigned i = 0; i != DstNumElts; ++i) {
4158 Idxs[i] = i + (Imm * DstNumElts);
4159 }
4160 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4161
4162 // If the intrinsic has a mask operand, handle that.
4163 if (CI->arg_size() == 4)
4164 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4165 Op1: CI->getArgOperand(i: 2));
4166 } else if (Name.starts_with(Prefix: "avx512.mask.perm.df.") ||
4167 Name.starts_with(Prefix: "avx512.mask.perm.di.")) {
4168 Value *Op0 = CI->getArgOperand(i: 0);
4169 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4170 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4171 unsigned NumElts = VecTy->getNumElements();
4172
4173 SmallVector<int, 8> Idxs(NumElts);
4174 for (unsigned i = 0; i != NumElts; ++i)
4175 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4176
4177 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4178
4179 if (CI->arg_size() == 4)
4180 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4181 Op1: CI->getArgOperand(i: 2));
4182 } else if (Name.starts_with(Prefix: "avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4183 // The immediate permute control byte looks like this:
4184 // [1:0] - select 128 bits from sources for low half of destination
4185 // [2] - ignore
4186 // [3] - zero low half of destination
4187 // [5:4] - select 128 bits from sources for high half of destination
4188 // [6] - ignore
4189 // [7] - zero high half of destination
4190
4191 uint8_t Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4192
4193 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4194 unsigned HalfSize = NumElts / 2;
4195 SmallVector<int, 8> ShuffleMask(NumElts);
4196
4197 // Determine which operand(s) are actually in use for this instruction.
4198 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4199 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4200
4201 // If needed, replace operands based on zero mask.
4202 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(Ty: CI->getType()) : V0;
4203 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(Ty: CI->getType()) : V1;
4204
4205 // Permute low half of result.
4206 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4207 for (unsigned i = 0; i < HalfSize; ++i)
4208 ShuffleMask[i] = StartIndex + i;
4209
4210 // Permute high half of result.
4211 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4212 for (unsigned i = 0; i < HalfSize; ++i)
4213 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4214
4215 Rep = Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
4216
4217 } else if (Name.starts_with(Prefix: "avx.vpermil.") || Name == "sse2.pshuf.d" ||
4218 Name.starts_with(Prefix: "avx512.mask.vpermil.p") ||
4219 Name.starts_with(Prefix: "avx512.mask.pshuf.d.")) {
4220 Value *Op0 = CI->getArgOperand(i: 0);
4221 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4222 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4223 unsigned NumElts = VecTy->getNumElements();
4224 // Calculate the size of each index in the immediate.
4225 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4226 unsigned IdxMask = ((1 << IdxSize) - 1);
4227
4228 SmallVector<int, 8> Idxs(NumElts);
4229 // Lookup the bits for this element, wrapping around the immediate every
4230 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4231 // to offset by the first index of each group.
4232 for (unsigned i = 0; i != NumElts; ++i)
4233 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4234
4235 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4236
4237 if (CI->arg_size() == 4)
4238 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4239 Op1: CI->getArgOperand(i: 2));
4240 } else if (Name == "sse2.pshufl.w" ||
4241 Name.starts_with(Prefix: "avx512.mask.pshufl.w.")) {
4242 Value *Op0 = CI->getArgOperand(i: 0);
4243 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4244 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4245
4246 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4247 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4248
4249 SmallVector<int, 16> Idxs(NumElts);
4250 for (unsigned l = 0; l != NumElts; l += 8) {
4251 for (unsigned i = 0; i != 4; ++i)
4252 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4253 for (unsigned i = 4; i != 8; ++i)
4254 Idxs[i + l] = i + l;
4255 }
4256
4257 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4258
4259 if (CI->arg_size() == 4)
4260 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4261 Op1: CI->getArgOperand(i: 2));
4262 } else if (Name == "sse2.pshufh.w" ||
4263 Name.starts_with(Prefix: "avx512.mask.pshufh.w.")) {
4264 Value *Op0 = CI->getArgOperand(i: 0);
4265 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4266 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4267
4268 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4269 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4270
4271 SmallVector<int, 16> Idxs(NumElts);
4272 for (unsigned l = 0; l != NumElts; l += 8) {
4273 for (unsigned i = 0; i != 4; ++i)
4274 Idxs[i + l] = i + l;
4275 for (unsigned i = 0; i != 4; ++i)
4276 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4277 }
4278
4279 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4280
4281 if (CI->arg_size() == 4)
4282 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4283 Op1: CI->getArgOperand(i: 2));
4284 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.p")) {
4285 Value *Op0 = CI->getArgOperand(i: 0);
4286 Value *Op1 = CI->getArgOperand(i: 1);
4287 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4288 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4289
4290 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4291 unsigned HalfLaneElts = NumLaneElts / 2;
4292
4293 SmallVector<int, 16> Idxs(NumElts);
4294 for (unsigned i = 0; i != NumElts; ++i) {
4295 // Base index is the starting element of the lane.
4296 Idxs[i] = i - (i % NumLaneElts);
4297 // If we are half way through the lane switch to the other source.
4298 if ((i % NumLaneElts) >= HalfLaneElts)
4299 Idxs[i] += NumElts;
4300 // Now select the specific element. By adding HalfLaneElts bits from
4301 // the immediate. Wrapping around the immediate every 8-bits.
4302 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4303 }
4304
4305 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4306
4307 Rep =
4308 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
4309 } else if (Name.starts_with(Prefix: "avx512.mask.movddup") ||
4310 Name.starts_with(Prefix: "avx512.mask.movshdup") ||
4311 Name.starts_with(Prefix: "avx512.mask.movsldup")) {
4312 Value *Op0 = CI->getArgOperand(i: 0);
4313 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4314 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4315
4316 unsigned Offset = 0;
4317 if (Name.starts_with(Prefix: "avx512.mask.movshdup."))
4318 Offset = 1;
4319
4320 SmallVector<int, 16> Idxs(NumElts);
4321 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4322 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4323 Idxs[i + l + 0] = i + l + Offset;
4324 Idxs[i + l + 1] = i + l + Offset;
4325 }
4326
4327 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4328
4329 Rep =
4330 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4331 } else if (Name.starts_with(Prefix: "avx512.mask.punpckl") ||
4332 Name.starts_with(Prefix: "avx512.mask.unpckl.")) {
4333 Value *Op0 = CI->getArgOperand(i: 0);
4334 Value *Op1 = CI->getArgOperand(i: 1);
4335 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4336 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4337
4338 SmallVector<int, 64> Idxs(NumElts);
4339 for (int l = 0; l != NumElts; l += NumLaneElts)
4340 for (int i = 0; i != NumLaneElts; ++i)
4341 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4342
4343 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4344
4345 Rep =
4346 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4347 } else if (Name.starts_with(Prefix: "avx512.mask.punpckh") ||
4348 Name.starts_with(Prefix: "avx512.mask.unpckh.")) {
4349 Value *Op0 = CI->getArgOperand(i: 0);
4350 Value *Op1 = CI->getArgOperand(i: 1);
4351 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4352 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4353
4354 SmallVector<int, 64> Idxs(NumElts);
4355 for (int l = 0; l != NumElts; l += NumLaneElts)
4356 for (int i = 0; i != NumLaneElts; ++i)
4357 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4358
4359 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4360
4361 Rep =
4362 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4363 } else if (Name.starts_with(Prefix: "avx512.mask.and.") ||
4364 Name.starts_with(Prefix: "avx512.mask.pand.")) {
4365 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4366 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4367 Rep = Builder.CreateAnd(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4368 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4369 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4370 Rep =
4371 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4372 } else if (Name.starts_with(Prefix: "avx512.mask.andn.") ||
4373 Name.starts_with(Prefix: "avx512.mask.pandn.")) {
4374 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4375 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4376 Rep = Builder.CreateNot(V: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy));
4377 Rep = Builder.CreateAnd(LHS: Rep,
4378 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4379 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4380 Rep =
4381 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4382 } else if (Name.starts_with(Prefix: "avx512.mask.or.") ||
4383 Name.starts_with(Prefix: "avx512.mask.por.")) {
4384 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4385 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4386 Rep = Builder.CreateOr(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4387 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4388 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4389 Rep =
4390 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4391 } else if (Name.starts_with(Prefix: "avx512.mask.xor.") ||
4392 Name.starts_with(Prefix: "avx512.mask.pxor.")) {
4393 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4394 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4395 Rep = Builder.CreateXor(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4396 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4397 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4398 Rep =
4399 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4400 } else if (Name.starts_with(Prefix: "avx512.mask.padd.")) {
4401 Rep = Builder.CreateAdd(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4402 Rep =
4403 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4404 } else if (Name.starts_with(Prefix: "avx512.mask.psub.")) {
4405 Rep = Builder.CreateSub(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4406 Rep =
4407 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4408 } else if (Name.starts_with(Prefix: "avx512.mask.pmull.")) {
4409 Rep = Builder.CreateMul(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4410 Rep =
4411 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4412 } else if (Name.starts_with(Prefix: "avx512.mask.add.p")) {
4413 if (Name.ends_with(Suffix: ".512")) {
4414 Intrinsic::ID IID;
4415 if (Name[17] == 's')
4416 IID = Intrinsic::x86_avx512_add_ps_512;
4417 else
4418 IID = Intrinsic::x86_avx512_add_pd_512;
4419
4420 Rep = Builder.CreateIntrinsic(
4421 ID: IID,
4422 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4423 } else {
4424 Rep = Builder.CreateFAdd(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4425 }
4426 Rep =
4427 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4428 } else if (Name.starts_with(Prefix: "avx512.mask.div.p")) {
4429 if (Name.ends_with(Suffix: ".512")) {
4430 Intrinsic::ID IID;
4431 if (Name[17] == 's')
4432 IID = Intrinsic::x86_avx512_div_ps_512;
4433 else
4434 IID = Intrinsic::x86_avx512_div_pd_512;
4435
4436 Rep = Builder.CreateIntrinsic(
4437 ID: IID,
4438 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4439 } else {
4440 Rep = Builder.CreateFDiv(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4441 }
4442 Rep =
4443 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4444 } else if (Name.starts_with(Prefix: "avx512.mask.mul.p")) {
4445 if (Name.ends_with(Suffix: ".512")) {
4446 Intrinsic::ID IID;
4447 if (Name[17] == 's')
4448 IID = Intrinsic::x86_avx512_mul_ps_512;
4449 else
4450 IID = Intrinsic::x86_avx512_mul_pd_512;
4451
4452 Rep = Builder.CreateIntrinsic(
4453 ID: IID,
4454 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4455 } else {
4456 Rep = Builder.CreateFMul(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4457 }
4458 Rep =
4459 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4460 } else if (Name.starts_with(Prefix: "avx512.mask.sub.p")) {
4461 if (Name.ends_with(Suffix: ".512")) {
4462 Intrinsic::ID IID;
4463 if (Name[17] == 's')
4464 IID = Intrinsic::x86_avx512_sub_ps_512;
4465 else
4466 IID = Intrinsic::x86_avx512_sub_pd_512;
4467
4468 Rep = Builder.CreateIntrinsic(
4469 ID: IID,
4470 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4471 } else {
4472 Rep = Builder.CreateFSub(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4473 }
4474 Rep =
4475 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4476 } else if ((Name.starts_with(Prefix: "avx512.mask.max.p") ||
4477 Name.starts_with(Prefix: "avx512.mask.min.p")) &&
4478 Name.drop_front(N: 18) == ".512") {
4479 bool IsDouble = Name[17] == 'd';
4480 bool IsMin = Name[13] == 'i';
4481 static const Intrinsic::ID MinMaxTbl[2][2] = {
4482 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4483 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4484 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4485
4486 Rep = Builder.CreateIntrinsic(
4487 ID: IID,
4488 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4489 Rep =
4490 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4491 } else if (Name.starts_with(Prefix: "avx512.mask.lzcnt.")) {
4492 Rep =
4493 Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: CI->getType(),
4494 Args: {CI->getArgOperand(i: 0), Builder.getInt1(V: false)});
4495 Rep =
4496 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4497 } else if (Name.starts_with(Prefix: "avx512.mask.psll")) {
4498 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4499 bool IsVariable = Name[16] == 'v';
4500 char Size = Name[16] == '.' ? Name[17]
4501 : Name[17] == '.' ? Name[18]
4502 : Name[18] == '.' ? Name[19]
4503 : Name[20];
4504
4505 Intrinsic::ID IID;
4506 if (IsVariable && Name[17] != '.') {
4507 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4508 IID = Intrinsic::x86_avx2_psllv_q;
4509 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4510 IID = Intrinsic::x86_avx2_psllv_q_256;
4511 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4512 IID = Intrinsic::x86_avx2_psllv_d;
4513 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4514 IID = Intrinsic::x86_avx2_psllv_d_256;
4515 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4516 IID = Intrinsic::x86_avx512_psllv_w_128;
4517 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4518 IID = Intrinsic::x86_avx512_psllv_w_256;
4519 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4520 IID = Intrinsic::x86_avx512_psllv_w_512;
4521 else
4522 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4523 } else if (Name.ends_with(Suffix: ".128")) {
4524 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4525 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4526 : Intrinsic::x86_sse2_psll_d;
4527 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4528 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4529 : Intrinsic::x86_sse2_psll_q;
4530 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4531 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4532 : Intrinsic::x86_sse2_psll_w;
4533 else
4534 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4535 } else if (Name.ends_with(Suffix: ".256")) {
4536 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4537 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4538 : Intrinsic::x86_avx2_psll_d;
4539 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4540 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4541 : Intrinsic::x86_avx2_psll_q;
4542 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4543 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4544 : Intrinsic::x86_avx2_psll_w;
4545 else
4546 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4547 } else {
4548 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4549 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4550 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4551 : Intrinsic::x86_avx512_psll_d_512;
4552 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4553 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4554 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4555 : Intrinsic::x86_avx512_psll_q_512;
4556 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4557 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4558 : Intrinsic::x86_avx512_psll_w_512;
4559 else
4560 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4561 }
4562
4563 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4564 } else if (Name.starts_with(Prefix: "avx512.mask.psrl")) {
4565 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4566 bool IsVariable = Name[16] == 'v';
4567 char Size = Name[16] == '.' ? Name[17]
4568 : Name[17] == '.' ? Name[18]
4569 : Name[18] == '.' ? Name[19]
4570 : Name[20];
4571
4572 Intrinsic::ID IID;
4573 if (IsVariable && Name[17] != '.') {
4574 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4575 IID = Intrinsic::x86_avx2_psrlv_q;
4576 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4577 IID = Intrinsic::x86_avx2_psrlv_q_256;
4578 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4579 IID = Intrinsic::x86_avx2_psrlv_d;
4580 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4581 IID = Intrinsic::x86_avx2_psrlv_d_256;
4582 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4583 IID = Intrinsic::x86_avx512_psrlv_w_128;
4584 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4585 IID = Intrinsic::x86_avx512_psrlv_w_256;
4586 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4587 IID = Intrinsic::x86_avx512_psrlv_w_512;
4588 else
4589 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4590 } else if (Name.ends_with(Suffix: ".128")) {
4591 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4592 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4593 : Intrinsic::x86_sse2_psrl_d;
4594 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4595 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4596 : Intrinsic::x86_sse2_psrl_q;
4597 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4598 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4599 : Intrinsic::x86_sse2_psrl_w;
4600 else
4601 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4602 } else if (Name.ends_with(Suffix: ".256")) {
4603 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4604 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4605 : Intrinsic::x86_avx2_psrl_d;
4606 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4607 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4608 : Intrinsic::x86_avx2_psrl_q;
4609 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4610 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4611 : Intrinsic::x86_avx2_psrl_w;
4612 else
4613 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4614 } else {
4615 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4616 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4617 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4618 : Intrinsic::x86_avx512_psrl_d_512;
4619 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4620 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4621 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4622 : Intrinsic::x86_avx512_psrl_q_512;
4623 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4624 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4625 : Intrinsic::x86_avx512_psrl_w_512;
4626 else
4627 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4628 }
4629
4630 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4631 } else if (Name.starts_with(Prefix: "avx512.mask.psra")) {
4632 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4633 bool IsVariable = Name[16] == 'v';
4634 char Size = Name[16] == '.' ? Name[17]
4635 : Name[17] == '.' ? Name[18]
4636 : Name[18] == '.' ? Name[19]
4637 : Name[20];
4638
4639 Intrinsic::ID IID;
4640 if (IsVariable && Name[17] != '.') {
4641 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4642 IID = Intrinsic::x86_avx2_psrav_d;
4643 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4644 IID = Intrinsic::x86_avx2_psrav_d_256;
4645 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4646 IID = Intrinsic::x86_avx512_psrav_w_128;
4647 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4648 IID = Intrinsic::x86_avx512_psrav_w_256;
4649 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4650 IID = Intrinsic::x86_avx512_psrav_w_512;
4651 else
4652 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4653 } else if (Name.ends_with(Suffix: ".128")) {
4654 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4655 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4656 : Intrinsic::x86_sse2_psra_d;
4657 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4658 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4659 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4660 : Intrinsic::x86_avx512_psra_q_128;
4661 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4662 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4663 : Intrinsic::x86_sse2_psra_w;
4664 else
4665 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4666 } else if (Name.ends_with(Suffix: ".256")) {
4667 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4668 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4669 : Intrinsic::x86_avx2_psra_d;
4670 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4671 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4672 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4673 : Intrinsic::x86_avx512_psra_q_256;
4674 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4675 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4676 : Intrinsic::x86_avx2_psra_w;
4677 else
4678 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4679 } else {
4680 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4681 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4682 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4683 : Intrinsic::x86_avx512_psra_d_512;
4684 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4685 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4686 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4687 : Intrinsic::x86_avx512_psra_q_512;
4688 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4689 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4690 : Intrinsic::x86_avx512_psra_w_512;
4691 else
4692 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4693 }
4694
4695 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4696 } else if (Name.starts_with(Prefix: "avx512.mask.move.s")) {
4697 Rep = upgradeMaskedMove(Builder, CI&: *CI);
4698 } else if (Name.starts_with(Prefix: "avx512.cvtmask2")) {
4699 Rep = upgradeMaskToInt(Builder, CI&: *CI);
4700 } else if (Name.ends_with(Suffix: ".movntdqa")) {
4701 MDNode *Node = MDNode::get(
4702 Context&: C, MDs: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
4703
4704 LoadInst *LI = Builder.CreateAlignedLoad(
4705 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0),
4706 Align: Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4707 LI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
4708 Rep = LI;
4709 } else if (Name.starts_with(Prefix: "fma.vfmadd.") ||
4710 Name.starts_with(Prefix: "fma.vfmsub.") ||
4711 Name.starts_with(Prefix: "fma.vfnmadd.") ||
4712 Name.starts_with(Prefix: "fma.vfnmsub.")) {
4713 bool NegMul = Name[6] == 'n';
4714 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4715 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4716
4717 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4718 CI->getArgOperand(i: 2)};
4719
4720 if (IsScalar) {
4721 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4722 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4723 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4724 }
4725
4726 if (NegMul && !IsScalar)
4727 Ops[0] = Builder.CreateFNeg(V: Ops[0]);
4728 if (NegMul && IsScalar)
4729 Ops[1] = Builder.CreateFNeg(V: Ops[1]);
4730 if (NegAcc)
4731 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4732
4733 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4734
4735 if (IsScalar)
4736 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
4737 } else if (Name.starts_with(Prefix: "fma4.vfmadd.s")) {
4738 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4739 CI->getArgOperand(i: 2)};
4740
4741 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4742 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4743 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4744
4745 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4746
4747 Rep = Builder.CreateInsertElement(Vec: Constant::getNullValue(Ty: CI->getType()),
4748 NewElt: Rep, Idx: (uint64_t)0);
4749 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.s") ||
4750 Name.starts_with(Prefix: "avx512.maskz.vfmadd.s") ||
4751 Name.starts_with(Prefix: "avx512.mask3.vfmadd.s") ||
4752 Name.starts_with(Prefix: "avx512.mask3.vfmsub.s") ||
4753 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.s")) {
4754 bool IsMask3 = Name[11] == '3';
4755 bool IsMaskZ = Name[11] == 'z';
4756 // Drop the "avx512.mask." to make it easier.
4757 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4758 bool NegMul = Name[2] == 'n';
4759 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4760
4761 Value *A = CI->getArgOperand(i: 0);
4762 Value *B = CI->getArgOperand(i: 1);
4763 Value *C = CI->getArgOperand(i: 2);
4764
4765 if (NegMul && (IsMask3 || IsMaskZ))
4766 A = Builder.CreateFNeg(V: A);
4767 if (NegMul && !(IsMask3 || IsMaskZ))
4768 B = Builder.CreateFNeg(V: B);
4769 if (NegAcc)
4770 C = Builder.CreateFNeg(V: C);
4771
4772 A = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)0);
4773 B = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
4774 C = Builder.CreateExtractElement(Vec: C, Idx: (uint64_t)0);
4775
4776 if (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4777 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4) {
4778 Value *Ops[] = {A, B, C, CI->getArgOperand(i: 4)};
4779
4780 Intrinsic::ID IID;
4781 if (Name.back() == 'd')
4782 IID = Intrinsic::x86_avx512_vfmadd_f64;
4783 else
4784 IID = Intrinsic::x86_avx512_vfmadd_f32;
4785 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4786 } else {
4787 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4788 }
4789
4790 Value *PassThru = IsMaskZ ? Constant::getNullValue(Ty: Rep->getType())
4791 : IsMask3 ? C
4792 : A;
4793
4794 // For Mask3 with NegAcc, we need to create a new extractelement that
4795 // avoids the negation above.
4796 if (NegAcc && IsMask3)
4797 PassThru =
4798 Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 2), Idx: (uint64_t)0);
4799
4800 Rep = emitX86ScalarSelect(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4801 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: IsMask3 ? 2 : 0), NewElt: Rep,
4802 Idx: (uint64_t)0);
4803 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.p") ||
4804 Name.starts_with(Prefix: "avx512.mask.vfnmadd.p") ||
4805 Name.starts_with(Prefix: "avx512.mask.vfnmsub.p") ||
4806 Name.starts_with(Prefix: "avx512.mask3.vfmadd.p") ||
4807 Name.starts_with(Prefix: "avx512.mask3.vfmsub.p") ||
4808 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.p") ||
4809 Name.starts_with(Prefix: "avx512.maskz.vfmadd.p")) {
4810 bool IsMask3 = Name[11] == '3';
4811 bool IsMaskZ = Name[11] == 'z';
4812 // Drop the "avx512.mask." to make it easier.
4813 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4814 bool NegMul = Name[2] == 'n';
4815 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4816
4817 Value *A = CI->getArgOperand(i: 0);
4818 Value *B = CI->getArgOperand(i: 1);
4819 Value *C = CI->getArgOperand(i: 2);
4820
4821 if (NegMul && (IsMask3 || IsMaskZ))
4822 A = Builder.CreateFNeg(V: A);
4823 if (NegMul && !(IsMask3 || IsMaskZ))
4824 B = Builder.CreateFNeg(V: B);
4825 if (NegAcc)
4826 C = Builder.CreateFNeg(V: C);
4827
4828 if (CI->arg_size() == 5 &&
4829 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4830 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4)) {
4831 Intrinsic::ID IID;
4832 // Check the character before ".512" in string.
4833 if (Name[Name.size() - 5] == 's')
4834 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4835 else
4836 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4837
4838 Rep = Builder.CreateIntrinsic(ID: IID, Args: {A, B, C, CI->getArgOperand(i: 4)});
4839 } else {
4840 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4841 }
4842
4843 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4844 : IsMask3 ? CI->getArgOperand(i: 2)
4845 : CI->getArgOperand(i: 0);
4846
4847 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4848 } else if (Name.starts_with(Prefix: "fma.vfmsubadd.p")) {
4849 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4850 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4851 Intrinsic::ID IID;
4852 if (VecWidth == 128 && EltWidth == 32)
4853 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4854 else if (VecWidth == 256 && EltWidth == 32)
4855 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4856 else if (VecWidth == 128 && EltWidth == 64)
4857 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4858 else if (VecWidth == 256 && EltWidth == 64)
4859 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4860 else
4861 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4862
4863 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4864 CI->getArgOperand(i: 2)};
4865 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4866 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4867 } else if (Name.starts_with(Prefix: "avx512.mask.vfmaddsub.p") ||
4868 Name.starts_with(Prefix: "avx512.mask3.vfmaddsub.p") ||
4869 Name.starts_with(Prefix: "avx512.maskz.vfmaddsub.p") ||
4870 Name.starts_with(Prefix: "avx512.mask3.vfmsubadd.p")) {
4871 bool IsMask3 = Name[11] == '3';
4872 bool IsMaskZ = Name[11] == 'z';
4873 // Drop the "avx512.mask." to make it easier.
4874 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4875 bool IsSubAdd = Name[3] == 's';
4876 if (CI->arg_size() == 5) {
4877 Intrinsic::ID IID;
4878 // Check the character before ".512" in string.
4879 if (Name[Name.size() - 5] == 's')
4880 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4881 else
4882 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4883
4884 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4885 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
4886 if (IsSubAdd)
4887 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4888
4889 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4890 } else {
4891 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4892
4893 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4894 CI->getArgOperand(i: 2)};
4895
4896 Function *FMA = Intrinsic::getOrInsertDeclaration(
4897 M: CI->getModule(), id: Intrinsic::fma, OverloadTys: Ops[0]->getType());
4898 Value *Odd = Builder.CreateCall(Callee: FMA, Args: Ops);
4899 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4900 Value *Even = Builder.CreateCall(Callee: FMA, Args: Ops);
4901
4902 if (IsSubAdd)
4903 std::swap(a&: Even, b&: Odd);
4904
4905 SmallVector<int, 32> Idxs(NumElts);
4906 for (int i = 0; i != NumElts; ++i)
4907 Idxs[i] = i + (i % 2) * NumElts;
4908
4909 Rep = Builder.CreateShuffleVector(V1: Even, V2: Odd, Mask: Idxs);
4910 }
4911
4912 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4913 : IsMask3 ? CI->getArgOperand(i: 2)
4914 : CI->getArgOperand(i: 0);
4915
4916 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4917 } else if (Name.starts_with(Prefix: "avx512.mask.pternlog.") ||
4918 Name.starts_with(Prefix: "avx512.maskz.pternlog.")) {
4919 bool ZeroMask = Name[11] == 'z';
4920 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4921 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4922 Intrinsic::ID IID;
4923 if (VecWidth == 128 && EltWidth == 32)
4924 IID = Intrinsic::x86_avx512_pternlog_d_128;
4925 else if (VecWidth == 256 && EltWidth == 32)
4926 IID = Intrinsic::x86_avx512_pternlog_d_256;
4927 else if (VecWidth == 512 && EltWidth == 32)
4928 IID = Intrinsic::x86_avx512_pternlog_d_512;
4929 else if (VecWidth == 128 && EltWidth == 64)
4930 IID = Intrinsic::x86_avx512_pternlog_q_128;
4931 else if (VecWidth == 256 && EltWidth == 64)
4932 IID = Intrinsic::x86_avx512_pternlog_q_256;
4933 else if (VecWidth == 512 && EltWidth == 64)
4934 IID = Intrinsic::x86_avx512_pternlog_q_512;
4935 else
4936 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4937
4938 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4939 CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)};
4940 Rep = Builder.CreateIntrinsic(ID: IID, Args);
4941 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4942 : CI->getArgOperand(i: 0);
4943 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: PassThru);
4944 } else if (Name.starts_with(Prefix: "avx512.mask.vpmadd52") ||
4945 Name.starts_with(Prefix: "avx512.maskz.vpmadd52")) {
4946 bool ZeroMask = Name[11] == 'z';
4947 bool High = Name[20] == 'h' || Name[21] == 'h';
4948 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4949 Intrinsic::ID IID;
4950 if (VecWidth == 128 && !High)
4951 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4952 else if (VecWidth == 256 && !High)
4953 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4954 else if (VecWidth == 512 && !High)
4955 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4956 else if (VecWidth == 128 && High)
4957 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4958 else if (VecWidth == 256 && High)
4959 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4960 else if (VecWidth == 512 && High)
4961 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4962 else
4963 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4964
4965 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4966 CI->getArgOperand(i: 2)};
4967 Rep = Builder.CreateIntrinsic(ID: IID, Args);
4968 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4969 : CI->getArgOperand(i: 0);
4970 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4971 } else if (Name.starts_with(Prefix: "avx512.mask.vpermi2var.") ||
4972 Name.starts_with(Prefix: "avx512.mask.vpermt2var.") ||
4973 Name.starts_with(Prefix: "avx512.maskz.vpermt2var.")) {
4974 bool ZeroMask = Name[11] == 'z';
4975 bool IndexForm = Name[17] == 'i';
4976 Rep = upgradeX86VPERMT2Intrinsics(Builder, CI&: *CI, ZeroMask, IndexForm);
4977 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpbusd.") ||
4978 Name.starts_with(Prefix: "avx512.maskz.vpdpbusd.") ||
4979 Name.starts_with(Prefix: "avx512.mask.vpdpbusds.") ||
4980 Name.starts_with(Prefix: "avx512.maskz.vpdpbusds.")) {
4981 bool ZeroMask = Name[11] == 'z';
4982 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4983 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4984 Intrinsic::ID IID;
4985 if (VecWidth == 128 && !IsSaturating)
4986 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4987 else if (VecWidth == 256 && !IsSaturating)
4988 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4989 else if (VecWidth == 512 && !IsSaturating)
4990 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4991 else if (VecWidth == 128 && IsSaturating)
4992 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4993 else if (VecWidth == 256 && IsSaturating)
4994 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4995 else if (VecWidth == 512 && IsSaturating)
4996 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4997 else
4998 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4999
5000 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5001 CI->getArgOperand(i: 2)};
5002
5003 // Input arguments types were incorrectly set to vectors of i32 before but
5004 // they should be vectors of i8. Insert bit cast when encountering the old
5005 // types
5006 if (Args[1]->getType()->isVectorTy() &&
5007 cast<VectorType>(Val: Args[1]->getType())
5008 ->getElementType()
5009 ->isIntegerTy(BitWidth: 32) &&
5010 Args[2]->getType()->isVectorTy() &&
5011 cast<VectorType>(Val: Args[2]->getType())
5012 ->getElementType()
5013 ->isIntegerTy(BitWidth: 32)) {
5014 Type *NewArgType = nullptr;
5015 if (VecWidth == 128)
5016 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 16, Scalable: false);
5017 else if (VecWidth == 256)
5018 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 32, Scalable: false);
5019 else if (VecWidth == 512)
5020 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 64, Scalable: false);
5021 else
5022 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
5023 CI);
5024
5025 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
5026 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
5027 }
5028
5029 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5030 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5031 : CI->getArgOperand(i: 0);
5032 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5033 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpwssd.") ||
5034 Name.starts_with(Prefix: "avx512.maskz.vpdpwssd.") ||
5035 Name.starts_with(Prefix: "avx512.mask.vpdpwssds.") ||
5036 Name.starts_with(Prefix: "avx512.maskz.vpdpwssds.")) {
5037 bool ZeroMask = Name[11] == 'z';
5038 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5039 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5040 Intrinsic::ID IID;
5041 if (VecWidth == 128 && !IsSaturating)
5042 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5043 else if (VecWidth == 256 && !IsSaturating)
5044 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5045 else if (VecWidth == 512 && !IsSaturating)
5046 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5047 else if (VecWidth == 128 && IsSaturating)
5048 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5049 else if (VecWidth == 256 && IsSaturating)
5050 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5051 else if (VecWidth == 512 && IsSaturating)
5052 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5053 else
5054 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5055
5056 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5057 CI->getArgOperand(i: 2)};
5058
5059 // Input arguments types were incorrectly set to vectors of i32 before but
5060 // they should be vectors of i16. Insert bit cast when encountering the old
5061 // types
5062 if (Args[1]->getType()->isVectorTy() &&
5063 cast<VectorType>(Val: Args[1]->getType())
5064 ->getElementType()
5065 ->isIntegerTy(BitWidth: 32) &&
5066 Args[2]->getType()->isVectorTy() &&
5067 cast<VectorType>(Val: Args[2]->getType())
5068 ->getElementType()
5069 ->isIntegerTy(BitWidth: 32)) {
5070 Type *NewArgType = nullptr;
5071 if (VecWidth == 128)
5072 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 8, Scalable: false);
5073 else if (VecWidth == 256)
5074 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 16, Scalable: false);
5075 else if (VecWidth == 512)
5076 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 32, Scalable: false);
5077 else
5078 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
5079 CI);
5080
5081 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
5082 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
5083 }
5084
5085 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5086 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5087 : CI->getArgOperand(i: 0);
5088 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5089 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5090 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5091 Name == "subborrow.u32" || Name == "subborrow.u64") {
5092 Intrinsic::ID IID;
5093 if (Name[0] == 'a' && Name.back() == '2')
5094 IID = Intrinsic::x86_addcarry_32;
5095 else if (Name[0] == 'a' && Name.back() == '4')
5096 IID = Intrinsic::x86_addcarry_64;
5097 else if (Name[0] == 's' && Name.back() == '2')
5098 IID = Intrinsic::x86_subborrow_32;
5099 else if (Name[0] == 's' && Name.back() == '4')
5100 IID = Intrinsic::x86_subborrow_64;
5101 else
5102 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5103
5104 // Make a call with 3 operands.
5105 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5106 CI->getArgOperand(i: 2)};
5107 Value *NewCall = Builder.CreateIntrinsic(ID: IID, Args);
5108
5109 // Extract the second result and store it.
5110 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
5111 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 3), Align: Align(1));
5112 // Replace the original call result with the first result of the new call.
5113 Value *CF = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
5114
5115 CI->replaceAllUsesWith(V: CF);
5116 Rep = nullptr;
5117 } else if (Name.starts_with(Prefix: "avx512.mask.") &&
5118 upgradeAVX512MaskToSelect(Name, Builder, CI&: *CI, Rep)) {
5119 // Rep will be updated by the call in the condition.
5120 } else if (Name.starts_with(Prefix: "bmi.pdep.")) {
5121 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pdep);
5122 } else if (Name.starts_with(Prefix: "bmi.pext.")) {
5123 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pext);
5124 } else
5125 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5126
5127 return Rep;
5128}
5129
5130static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5131 Function *F, IRBuilder<> &Builder) {
5132 if (Name.starts_with(Prefix: "neon.bfcvt")) {
5133 if (Name.starts_with(Prefix: "neon.bfcvtn2")) {
5134 SmallVector<int, 32> LoMask(4);
5135 std::iota(first: LoMask.begin(), last: LoMask.end(), value: 0);
5136 SmallVector<int, 32> ConcatMask(8);
5137 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
5138 Value *Inactive = Builder.CreateShuffleVector(V: CI->getOperand(i_nocapture: 0), Mask: LoMask);
5139 Value *Trunc =
5140 Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 1), DestTy: Inactive->getType());
5141 return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
5142 } else if (Name.starts_with(Prefix: "neon.bfcvtn")) {
5143 SmallVector<int, 32> ConcatMask(8);
5144 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
5145 Type *V4BF16 =
5146 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: F->getContext()), NumElts: 4);
5147 Value *Trunc = Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0), DestTy: V4BF16);
5148 dbgs() << "Trunc: " << *Trunc << "\n";
5149 return Builder.CreateShuffleVector(
5150 V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
5151 } else {
5152 return Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0),
5153 DestTy: Type::getBFloatTy(C&: F->getContext()));
5154 }
5155 } else if (Name.starts_with(Prefix: "sve.fcvt")) {
5156 Intrinsic::ID NewID =
5157 StringSwitch<Intrinsic::ID>(Name)
5158 .Case(S: "sve.fcvt.bf16f32", Value: Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5159 .Case(S: "sve.fcvtnt.bf16f32",
5160 Value: Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5161 .Default(Value: Intrinsic::not_intrinsic);
5162 if (NewID == Intrinsic::not_intrinsic)
5163 llvm_unreachable("Unhandled Intrinsic!");
5164
5165 SmallVector<Value *, 3> Args(CI->args());
5166
5167 // The original intrinsics incorrectly used a predicate based on the
5168 // smallest element type rather than the largest.
5169 Type *BadPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
5170 Type *GoodPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
5171
5172 if (Args[1]->getType() != BadPredTy)
5173 llvm_unreachable("Unexpected predicate type!");
5174
5175 Args[1] = Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
5176 OverloadTypes: BadPredTy, Args: Args[1]);
5177 Args[1] = Builder.CreateIntrinsic(
5178 ID: Intrinsic::aarch64_sve_convert_from_svbool, OverloadTypes: GoodPredTy, Args: Args[1]);
5179
5180 return Builder.CreateIntrinsic(ID: NewID, Args, /*FMFSource=*/nullptr,
5181 Name: CI->getName());
5182 }
5183
5184 if (Name == "neon.vcvtfp2hf")
5185 return Builder.CreateBitCast(
5186 V: Builder.CreateFPTrunc(
5187 V: CI->getOperand(i_nocapture: 0),
5188 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
5189 DestTy: FixedVectorType::get(ElementType: Type::getInt16Ty(C&: F->getContext()), NumElts: 4));
5190 if (Name == "neon.vcvthf2fp")
5191 return Builder.CreateFPExt(
5192 V: Builder.CreateBitCast(
5193 V: CI->getOperand(i_nocapture: 0),
5194 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
5195 DestTy: FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4));
5196
5197 llvm_unreachable("Unhandled Intrinsic!");
5198}
5199
5200static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5201 IRBuilder<> &Builder) {
5202 if (Name == "mve.vctp64.old") {
5203 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5204 // correct type.
5205 Value *VCTP = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vctp64, OverloadTypes: {},
5206 Args: CI->getArgOperand(i: 0),
5207 /*FMFSource=*/nullptr, Name: CI->getName());
5208 Value *C1 = Builder.CreateIntrinsic(
5209 ID: Intrinsic::arm_mve_pred_v2i,
5210 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 2, Scalable: false)}, Args: VCTP);
5211 return Builder.CreateIntrinsic(
5212 ID: Intrinsic::arm_mve_pred_i2v,
5213 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: C1);
5214 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5215 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5216 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5217 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5218 Name ==
5219 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5220 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5221 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5222 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5223 Name ==
5224 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5225 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5226 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5227 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5228 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5229 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5230 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5231 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5232 std::vector<Type *> Tys;
5233 unsigned ID = CI->getIntrinsicID();
5234 Type *V2I1Ty = FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts: 2);
5235 switch (ID) {
5236 case Intrinsic::arm_mve_mull_int_predicated:
5237 case Intrinsic::arm_mve_vqdmull_predicated:
5238 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5239 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(), V2I1Ty};
5240 break;
5241 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5242 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5243 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5244 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5245 V2I1Ty};
5246 break;
5247 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5248 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5249 CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5250 break;
5251 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5252 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 1)->getType(),
5253 CI->getOperand(i_nocapture: 2)->getType(), V2I1Ty};
5254 break;
5255 case Intrinsic::arm_cde_vcx1q_predicated:
5256 case Intrinsic::arm_cde_vcx1qa_predicated:
5257 case Intrinsic::arm_cde_vcx2q_predicated:
5258 case Intrinsic::arm_cde_vcx2qa_predicated:
5259 case Intrinsic::arm_cde_vcx3q_predicated:
5260 case Intrinsic::arm_cde_vcx3qa_predicated:
5261 Tys = {CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5262 break;
5263 default:
5264 llvm_unreachable("Unhandled Intrinsic!");
5265 }
5266
5267 std::vector<Value *> Ops;
5268 for (Value *Op : CI->args()) {
5269 Type *Ty = Op->getType();
5270 if (Ty->getScalarSizeInBits() == 1) {
5271 Value *C1 = Builder.CreateIntrinsic(
5272 ID: Intrinsic::arm_mve_pred_v2i,
5273 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: Op);
5274 Op = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_pred_i2v, OverloadTypes: {V2I1Ty}, Args: C1);
5275 }
5276 Ops.push_back(x: Op);
5277 }
5278
5279 return Builder.CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops, /*FMFSource=*/nullptr,
5280 Name: CI->getName());
5281 }
5282 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5283}
5284
5285// These are expected to have the arguments:
5286// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5287//
5288// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5289//
5290static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5291 Function *F, IRBuilder<> &Builder) {
5292 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5293 // for compatibility.
5294 auto UpgradeLegacyWMMAIUIntrinsicCall =
5295 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5296 ArrayRef<Type *> OverloadTys) -> Value * {
5297 // Prepare arguments, append clamp=0 for compatibility
5298 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5299 Args.push_back(Elt: Builder.getFalse());
5300
5301 // Insert the declaration for the right overload types
5302 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5303 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys);
5304
5305 // Copy operand bundles if any
5306 SmallVector<OperandBundleDef, 1> Bundles;
5307 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5308
5309 // Create the new call and copy calling properties
5310 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5311 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5312 NewCall->setCallingConv(CI->getCallingConv());
5313 NewCall->setAttributes(CI->getAttributes());
5314 NewCall->copyMetadata(SrcInst: *CI);
5315 return NewCall;
5316 };
5317
5318 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5319 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5320 "intrinsic should have 7 arguments");
5321 Type *T1 = CI->getArgOperand(i: 4)->getType();
5322 Type *T2 = CI->getArgOperand(i: 1)->getType();
5323 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5324 }
5325 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5326 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5327 "intrinsic should have 8 arguments");
5328 Type *T1 = CI->getArgOperand(i: 4)->getType();
5329 Type *T2 = CI->getArgOperand(i: 1)->getType();
5330 Type *T3 = CI->getArgOperand(i: 3)->getType();
5331 Type *T4 = CI->getArgOperand(i: 5)->getType();
5332 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5333 }
5334
5335 switch (F->getIntrinsicID()) {
5336 default:
5337 break;
5338 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5339 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5340 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5341 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5342 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5343 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5344 // Drop src0 and src1 modifiers.
5345 const Value *Op0 = CI->getArgOperand(i: 0);
5346 const Value *Op2 = CI->getArgOperand(i: 2);
5347 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5348 const ConstantInt *ModA = dyn_cast<ConstantInt>(Val: Op0);
5349 const ConstantInt *ModB = dyn_cast<ConstantInt>(Val: Op2);
5350 if (!ModA->isZero() || !ModB->isZero())
5351 reportFatalUsageError(reason: Name + " matrix A and B modifiers shall be zero");
5352
5353 SmallVector<Value *, 8> Args{CI->getArgOperand(i: 1), CI->getArgOperand(i: 3)};
5354 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5355 Args.push_back(Elt: CI->getArgOperand(i: I));
5356
5357 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5358 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5359 Overloads.push_back(Elt: Args[3]->getType());
5360 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5361 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys: Overloads);
5362
5363 SmallVector<OperandBundleDef, 1> Bundles;
5364 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5365
5366 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5367 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5368 NewCall->setCallingConv(CI->getCallingConv());
5369 NewCall->setAttributes(CI->getAttributes());
5370 NewCall->copyMetadata(SrcInst: *CI);
5371 NewCall->takeName(V: CI);
5372 return NewCall;
5373 }
5374 }
5375
5376 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
5377 Value *LHS = CI->getArgOperand(i: 0);
5378 Value *RHS = CI->getArgOperand(i: 1);
5379 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5380 cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue());
5381 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5382 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5383 RetTy: CI->getType(), ID: Intrinsic::amdgcn_ballot, Args: Cmp);
5384 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5385 NewCall->setCallingConv(CI->getCallingConv());
5386 NewCall->copyMetadata(SrcInst: *CI);
5387 NewCall->takeName(V: CI);
5388 return NewCall;
5389 }
5390
5391 if (Name.starts_with(Prefix: "addrspacecast.nonnull")) {
5392 if (CI->getNumOperands() < 2) // Malformed bitcode.
5393 return nullptr;
5394 Value *ASC = Builder.CreateAddrSpaceCast(
5395 V: CI->getArgOperand(i: 0), DestTy: CI->getType(), Name: "", /*IsNonNull=*/true);
5396 ASC->takeName(V: CI);
5397 return ASC;
5398 }
5399
5400 AtomicRMWInst::BinOp RMWOp =
5401 StringSwitch<AtomicRMWInst::BinOp>(Name)
5402 .StartsWith(S: "ds.fadd", Value: AtomicRMWInst::FAdd)
5403 .StartsWith(S: "ds.fmin", Value: AtomicRMWInst::FMin)
5404 .StartsWith(S: "ds.fmax", Value: AtomicRMWInst::FMax)
5405 .StartsWith(S: "atomic.inc.", Value: AtomicRMWInst::UIncWrap)
5406 .StartsWith(S: "atomic.dec.", Value: AtomicRMWInst::UDecWrap)
5407 .StartsWith(S: "global.atomic.fadd", Value: AtomicRMWInst::FAdd)
5408 .StartsWith(S: "flat.atomic.fadd", Value: AtomicRMWInst::FAdd)
5409 .StartsWith(S: "global.atomic.fmin", Value: AtomicRMWInst::FMin)
5410 .StartsWith(S: "flat.atomic.fmin", Value: AtomicRMWInst::FMin)
5411 .StartsWith(S: "global.atomic.fmax", Value: AtomicRMWInst::FMax)
5412 .StartsWith(S: "flat.atomic.fmax", Value: AtomicRMWInst::FMax)
5413 .StartsWith(S: "atomic.cond.sub", Value: AtomicRMWInst::USubCond)
5414 .StartsWith(S: "atomic.csub", Value: AtomicRMWInst::USubSat);
5415
5416 unsigned NumOperands = CI->getNumOperands();
5417 if (NumOperands < 3) // Malformed bitcode.
5418 return nullptr;
5419
5420 Value *Ptr = CI->getArgOperand(i: 0);
5421 PointerType *PtrTy = dyn_cast<PointerType>(Val: Ptr->getType());
5422 if (!PtrTy) // Malformed.
5423 return nullptr;
5424
5425 Value *Val = CI->getArgOperand(i: 1);
5426 if (Val->getType() != CI->getType()) // Malformed.
5427 return nullptr;
5428
5429 ConstantInt *OrderArg = nullptr;
5430 bool IsVolatile = false;
5431
5432 // These should have 5 arguments (plus the callee). A separate version of the
5433 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5434 if (NumOperands > 3)
5435 OrderArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5436
5437 // Ignore scope argument at 3
5438
5439 if (NumOperands > 5) {
5440 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 4));
5441 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5442 }
5443
5444 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5445 if (OrderArg && isValidAtomicOrdering(I: OrderArg->getZExtValue()))
5446 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5447 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5448 Order = AtomicOrdering::SequentiallyConsistent;
5449
5450 LLVMContext &Ctx = F->getContext();
5451
5452 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5453 Type *RetTy = CI->getType();
5454 if (VectorType *VT = dyn_cast<VectorType>(Val: RetTy)) {
5455 if (VT->getElementType()->isIntegerTy(BitWidth: 16)) {
5456 VectorType *AsBF16 =
5457 VectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), EC: VT->getElementCount());
5458 Val = Builder.CreateBitCast(V: Val, DestTy: AsBF16);
5459 }
5460 }
5461
5462 // The scope argument never really worked correctly. Use agent as the most
5463 // conservative option which should still always produce the instruction.
5464 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID(SSN: "agent");
5465 AtomicRMWInst *RMW =
5466 Builder.CreateAtomicRMW(Op: RMWOp, Ptr, Val, Align: std::nullopt, Ordering: Order, SSID);
5467
5468 unsigned AddrSpace = PtrTy->getAddressSpace();
5469 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5470 MDNode *EmptyMD = MDNode::get(Context&: F->getContext(), MDs: {});
5471 RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: EmptyMD);
5472 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5473 RMW->setMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode, Node: EmptyMD);
5474 }
5475
5476 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5477 MDBuilder MDB(F->getContext());
5478 MDNode *RangeNotPrivate =
5479 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5480 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5481 RMW->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: RangeNotPrivate);
5482 }
5483
5484 if (IsVolatile)
5485 RMW->setVolatile(true);
5486
5487 return Builder.CreateBitCast(V: RMW, DestTy: RetTy);
5488}
5489
5490/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5491/// plain MDNode, as it's the verifier's job to check these are the correct
5492/// types later.
5493static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5494 if (Op < CI->arg_size()) {
5495 if (MetadataAsValue *MAV =
5496 dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op))) {
5497 Metadata *MD = MAV->getMetadata();
5498 return dyn_cast_if_present<MDNode>(Val: MD);
5499 }
5500 }
5501 return nullptr;
5502}
5503
5504/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5505static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5506 if (Op < CI->arg_size())
5507 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op)))
5508 return MAV->getMetadata();
5509 return nullptr;
5510}
5511
5512/// Convert debug intrinsic calls to non-instruction debug records.
5513/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5514/// \p CI - The debug intrinsic call.
5515static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5516 DbgRecord *DR = nullptr;
5517 if (Name == "label") {
5518 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(Label: unwrapMAVOp(CI, Op: 0));
5519 } else if (Name == "assign") {
5520 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5521 Type: DbgVariableRecord::LocationType::Assign, Val: unwrapMAVMetadataOp(CI, Op: 0),
5522 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: unwrapMAVOp(CI, Op: 3),
5523 Address: unwrapMAVMetadataOp(CI, Op: 4),
5524 /*The address is a Value ref, it will be stored as a Metadata */
5525 AddressExpression: unwrapMAVOp(CI, Op: 5));
5526 } else if (Name == "declare") {
5527 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5528 Type: DbgVariableRecord::LocationType::Declare, Val: unwrapMAVMetadataOp(CI, Op: 0),
5529 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5530 } else if (Name == "addr") {
5531 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5532 MDNode *ExprNode = unwrapMAVOp(CI, Op: 2);
5533 // Don't try to add something to the expression if it's not an expression.
5534 // Instead, allow the verifier to fail later.
5535 if (DIExpression *Expr = dyn_cast<DIExpression>(Val: ExprNode)) {
5536 ExprNode = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
5537 }
5538 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5539 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5540 Variable: unwrapMAVOp(CI, Op: 1), Expression: ExprNode, AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5541 } else if (Name == "value") {
5542 // An old version of dbg.value had an extra offset argument.
5543 unsigned VarOp = 1;
5544 unsigned ExprOp = 2;
5545 if (CI->arg_size() == 4) {
5546 auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1));
5547 // Nonzero offset dbg.values get dropped without a replacement.
5548 if (!Offset || !Offset->isNullValue())
5549 return;
5550 VarOp = 2;
5551 ExprOp = 3;
5552 }
5553 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5554 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5555 Variable: unwrapMAVOp(CI, Op: VarOp), Expression: unwrapMAVOp(CI, Op: ExprOp), AssignID: nullptr, Address: nullptr,
5556 AddressExpression: nullptr);
5557 }
5558 DR->setDebugLoc(CI->getDebugLoc());
5559 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5560 CI->getParent()->insertDbgRecordBefore(DR, Here: CI->getIterator());
5561}
5562
5563static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5564 auto *Offset = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5565 if (!Offset)
5566 reportFatalUsageError(reason: "Invalid llvm.vector.splice offset argument");
5567 int64_t OffsetVal = Offset->getSExtValue();
5568 return Builder.CreateIntrinsic(ID: OffsetVal >= 0
5569 ? Intrinsic::vector_splice_left
5570 : Intrinsic::vector_splice_right,
5571 OverloadTypes: CI->getType(),
5572 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5573 Builder.getInt32(C: std::abs(i: OffsetVal))});
5574}
5575
5576static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5577 Function *F, IRBuilder<> &Builder) {
5578 if (Name.starts_with(Prefix: "to.fp16")) {
5579 Value *Cast =
5580 Builder.CreateFPTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5581 return Builder.CreateBitCast(V: Cast, DestTy: CI->getType());
5582 }
5583
5584 if (Name.starts_with(Prefix: "from.fp16")) {
5585 Value *Cast =
5586 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5587 return Builder.CreateFPExt(V: Cast, DestTy: CI->getType());
5588 }
5589
5590 return nullptr;
5591}
5592
5593static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5594 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5595 if (!MD || !isa<MDString>(Val: MD))
5596 return ICmpInst::BAD_ICMP_PREDICATE;
5597 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5598 .Case(S: "eq", Value: ICmpInst::ICMP_EQ)
5599 .Case(S: "ne", Value: ICmpInst::ICMP_NE)
5600 .Case(S: "ugt", Value: ICmpInst::ICMP_UGT)
5601 .Case(S: "uge", Value: ICmpInst::ICMP_UGE)
5602 .Case(S: "ult", Value: ICmpInst::ICMP_ULT)
5603 .Case(S: "ule", Value: ICmpInst::ICMP_ULE)
5604 .Case(S: "sgt", Value: ICmpInst::ICMP_SGT)
5605 .Case(S: "sge", Value: ICmpInst::ICMP_SGE)
5606 .Case(S: "slt", Value: ICmpInst::ICMP_SLT)
5607 .Case(S: "sle", Value: ICmpInst::ICMP_SLE)
5608 .Default(Value: ICmpInst::BAD_ICMP_PREDICATE);
5609}
5610
5611static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5612 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5613 if (!MD || !isa<MDString>(Val: MD))
5614 return FCmpInst::BAD_FCMP_PREDICATE;
5615 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5616 .Case(S: "oeq", Value: FCmpInst::FCMP_OEQ)
5617 .Case(S: "ogt", Value: FCmpInst::FCMP_OGT)
5618 .Case(S: "oge", Value: FCmpInst::FCMP_OGE)
5619 .Case(S: "olt", Value: FCmpInst::FCMP_OLT)
5620 .Case(S: "ole", Value: FCmpInst::FCMP_OLE)
5621 .Case(S: "one", Value: FCmpInst::FCMP_ONE)
5622 .Case(S: "ord", Value: FCmpInst::FCMP_ORD)
5623 .Case(S: "uno", Value: FCmpInst::FCMP_UNO)
5624 .Case(S: "ueq", Value: FCmpInst::FCMP_UEQ)
5625 .Case(S: "ugt", Value: FCmpInst::FCMP_UGT)
5626 .Case(S: "uge", Value: FCmpInst::FCMP_UGE)
5627 .Case(S: "ult", Value: FCmpInst::FCMP_ULT)
5628 .Case(S: "ule", Value: FCmpInst::FCMP_ULE)
5629 .Case(S: "une", Value: FCmpInst::FCMP_UNE)
5630 .Default(Value: FCmpInst::BAD_FCMP_PREDICATE);
5631}
5632
5633static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5634 IRBuilder<> &Builder) {
5635 Value *Rep;
5636 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5637 if (Opcode && Instruction::isUnaryOp(Opcode))
5638 Rep =
5639 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)Opcode, V: CI->getArgOperand(i: 0));
5640 else if (Opcode && Instruction::isBinaryOp(Opcode))
5641 Rep = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opcode,
5642 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5643 else if (Opcode && Instruction::isCast(Opcode))
5644 Rep = Builder.CreateCast(Op: (Instruction::CastOps)Opcode, V: CI->getArgOperand(i: 0),
5645 DestTy: CI->getType());
5646 else if (Opcode == Instruction::ICmp)
5647 Rep = Builder.CreateICmp(P: getVPIntPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5648 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5649 else if (Opcode == Instruction::FCmp)
5650 Rep = Builder.CreateFCmp(P: getVPFPPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5651 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5652 else if (Opcode == Instruction::Select)
5653 Rep = Builder.CreateSelect(C: CI->getArgOperand(i: 0), True: CI->getArgOperand(i: 1),
5654 False: CI->getArgOperand(i: 2));
5655 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5656 SmallVector<Value *, 2> Args(drop_end(RangeOrContainer: CI->args(), N: 2));
5657 Rep = Builder.CreateIntrinsic(RetTy: CI->getType(), ID: IntrinsicID, Args, FMFSource: {});
5658 } else
5659 llvm_unreachable("Unexpected vp intrinsic");
5660 Rep->takeName(V: CI);
5661 return Rep;
5662}
5663
5664static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5665 IRBuilder<> &Builder) {
5666 Intrinsic::ID IID = NewFn->getIntrinsicID();
5667
5668 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5669 if (Defaults.empty())
5670 return false;
5671
5672 unsigned OldArgCount = CI->arg_size();
5673 unsigned NewArgCount = NewFn->arg_size();
5674
5675 if (OldArgCount < FirstDefault)
5676 return false;
5677
5678 // More arguments than the new intrinsic accepts, cannot upgrade.
5679 if (OldArgCount > NewArgCount)
5680 return false;
5681
5682 // The call already passes the defaulted arguments explicitly; only the
5683 // callee is still the old, shorter declaration, so retarget it.
5684 if (OldArgCount == NewArgCount) {
5685 if (CI->getFunctionType() != NewFn->getFunctionType())
5686 return false;
5687 CI->setCalledFunction(NewFn);
5688 return true;
5689 }
5690
5691 // OldArgCount < NewArgCount: Fill in each missing trailing default
5692 // argument from the table.
5693 SmallVector<Value *, 8> NewArgs(CI->args());
5694
5695 FunctionType *NewFT = NewFn->getFunctionType();
5696 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5697 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5698 "missing argument outside the default range");
5699 Type *ParamTy = NewFT->getParamType(i: Idx);
5700
5701 // Only integer types are supported (i1, i8, i16, i32, i64).
5702 if (!ParamTy->isIntegerTy())
5703 return false;
5704 NewArgs.push_back(Elt: ConstantInt::get(Ty: ParamTy, V: Defaults[Idx - FirstDefault]));
5705 }
5706
5707 // Preserve operand bundles by creating the call with them.
5708 SmallVector<OperandBundleDef, 1> OpBundles;
5709 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
5710 CallInst *NewCall = Builder.CreateCall(Callee: NewFn, Args: NewArgs, OpBundles);
5711
5712 NewCall->takeName(V: CI);
5713 NewCall->setCallingConv(CI->getCallingConv());
5714 NewCall->copyMetadata(SrcInst: *CI);
5715 if (auto *OldCI = dyn_cast<CallInst>(Val: CI))
5716 NewCall->setTailCallKind(OldCI->getTailCallKind());
5717
5718 CI->replaceAllUsesWith(V: NewCall);
5719 CI->eraseFromParent();
5720 return true;
5721}
5722
5723/// Upgrade a call to an old intrinsic. All argument and return casting must be
5724/// provided to seamlessly integrate with existing context.
5725void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5726 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5727 // checks the callee's function type matches. It's likely we need to handle
5728 // type changes here.
5729 Function *F = dyn_cast<Function>(Val: CI->getCalledOperand());
5730 if (!F)
5731 return;
5732
5733 LLVMContext &C = CI->getContext();
5734 IRBuilder<> Builder(C);
5735 if (isa<FPMathOperator>(Val: CI))
5736 Builder.setFastMathFlags(CI->getFastMathFlags());
5737 Builder.SetInsertPoint(TheBB: CI->getParent(), IP: CI->getIterator());
5738
5739 if (!NewFn) {
5740 // Get the Function's name.
5741 StringRef Name = F->getName();
5742 if (!Name.consume_front(Prefix: "llvm."))
5743 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5744
5745 bool IsX86 = Name.consume_front(Prefix: "x86.");
5746 bool IsNVVM = Name.consume_front(Prefix: "nvvm.");
5747 bool IsAArch64 = Name.consume_front(Prefix: "aarch64.");
5748 bool IsARM = Name.consume_front(Prefix: "arm.");
5749 bool IsAMDGCN = Name.consume_front(Prefix: "amdgcn.");
5750 bool IsDbg = Name.consume_front(Prefix: "dbg.");
5751 bool IsOldSplice =
5752 (Name.consume_front(Prefix: "experimental.vector.splice") ||
5753 Name.consume_front(Prefix: "vector.splice")) &&
5754 !(Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"));
5755 Value *Rep = nullptr;
5756
5757 if (!IsX86 && Name == "stackprotectorcheck") {
5758 Rep = nullptr;
5759 } else if (IsNVVM) {
5760 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5761 } else if (IsX86) {
5762 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5763 } else if (IsAArch64) {
5764 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5765 } else if (IsARM) {
5766 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5767 } else if (IsAMDGCN) {
5768 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5769 } else if (IsDbg) {
5770 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5771 } else if (IsOldSplice) {
5772 Rep = upgradeVectorSplice(CI, Builder);
5773 } else if (Name.consume_front(Prefix: "convert.")) {
5774 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5775 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5776 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5777 Rep = nullptr;
5778 } else if (shouldUpgradeVPIntrinsic(Name)) {
5779 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5780 } else {
5781 llvm_unreachable("Unknown function for CallBase upgrade.");
5782 }
5783
5784 if (Rep)
5785 CI->replaceAllUsesWith(V: Rep);
5786 CI->eraseFromParent();
5787 return;
5788 }
5789
5790 const auto &DefaultCase = [&]() -> void {
5791 if (F == NewFn)
5792 return;
5793
5794 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5795 // Handle generic mangling change.
5796 assert(
5797 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5798 "Unknown function for CallBase upgrade and isn't just a name change");
5799 CI->setCalledFunction(NewFn);
5800 return;
5801 }
5802
5803 // This must be an upgrade from a named to a literal struct.
5804 if (auto *OldST = dyn_cast<StructType>(Val: CI->getType())) {
5805 assert(OldST != NewFn->getReturnType() &&
5806 "Return type must have changed");
5807 assert(OldST->getNumElements() ==
5808 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5809 "Must have same number of elements");
5810
5811 SmallVector<Value *> Args(CI->args());
5812 CallInst *NewCI = Builder.CreateCall(Callee: NewFn, Args);
5813 NewCI->setAttributes(CI->getAttributes());
5814 Value *Res = PoisonValue::get(T: OldST);
5815 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5816 Value *Elem = Builder.CreateExtractValue(Agg: NewCI, Idxs: Idx);
5817 Res = Builder.CreateInsertValue(Agg: Res, Val: Elem, Idxs: Idx);
5818 }
5819 CI->replaceAllUsesWith(V: Res);
5820 CI->eraseFromParent();
5821 return;
5822 }
5823
5824 // We're probably about to produce something invalid. Let the verifier catch
5825 // it instead of dying here.
5826 CI->setCalledOperand(
5827 ConstantExpr::getPointerCast(C: NewFn, Ty: CI->getCalledOperand()->getType()));
5828 return;
5829 };
5830 CallInst *NewCall = nullptr;
5831 switch (NewFn->getIntrinsicID()) {
5832 default: {
5833 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5834 return;
5835 DefaultCase();
5836 return;
5837 }
5838 case Intrinsic::arm_neon_vst1:
5839 case Intrinsic::arm_neon_vst2:
5840 case Intrinsic::arm_neon_vst3:
5841 case Intrinsic::arm_neon_vst4:
5842 case Intrinsic::arm_neon_vst2lane:
5843 case Intrinsic::arm_neon_vst3lane:
5844 case Intrinsic::arm_neon_vst4lane: {
5845 SmallVector<Value *, 4> Args(CI->args());
5846 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5847 break;
5848 }
5849 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5850 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5851 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5852 LLVMContext &Ctx = F->getParent()->getContext();
5853 SmallVector<Value *, 4> Args(CI->args());
5854 Args[3] = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx),
5855 V: cast<ConstantInt>(Val: Args[3])->getZExtValue());
5856 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5857 break;
5858 }
5859 case Intrinsic::aarch64_sve_ld3_sret:
5860 case Intrinsic::aarch64_sve_ld4_sret:
5861 case Intrinsic::aarch64_sve_ld2_sret: {
5862 // Is this a trivial remangle of the name to support ptr address spaces?
5863 if (isa<StructType>(Val: F->getReturnType())) {
5864 DefaultCase();
5865 return;
5866 }
5867
5868 StringRef Name = F->getName();
5869 Name = Name.substr(Start: 5);
5870 unsigned N = StringSwitch<unsigned>(Name)
5871 .StartsWith(S: "aarch64.sve.ld2", Value: 2)
5872 .StartsWith(S: "aarch64.sve.ld3", Value: 3)
5873 .StartsWith(S: "aarch64.sve.ld4", Value: 4)
5874 .Default(Value: 0);
5875 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5876 unsigned MinElts = RetTy->getMinNumElements() / N;
5877 SmallVector<Value *, 2> Args(CI->args());
5878 Value *NewLdCall = Builder.CreateCall(Callee: NewFn, Args);
5879 Value *Ret = llvm::PoisonValue::get(T: RetTy);
5880 for (unsigned I = 0; I < N; I++) {
5881 Value *SRet = Builder.CreateExtractValue(Agg: NewLdCall, Idxs: I);
5882 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: SRet, Idx: I * MinElts);
5883 }
5884 NewCall = dyn_cast<CallInst>(Val: Ret);
5885 break;
5886 }
5887
5888 case Intrinsic::coro_end_async:
5889 case Intrinsic::coro_end: {
5890 SmallVector<Value *, 3> Args(CI->args());
5891 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5892 Args.push_back(Elt: ConstantTokenNone::get(Context&: CI->getContext()));
5893 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5894
5895 if (!CI->getType()->isVoidTy()) {
5896 if (!CI->use_empty()) {
5897 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5898 M: CI->getModule(), id: Intrinsic::coro_is_in_ramp);
5899 Value *InRamp = Builder.CreateCall(Callee: IsInRamp);
5900 CI->replaceAllUsesWith(V: Builder.CreateNot(V: InRamp));
5901 }
5902 CI->eraseFromParent();
5903 return;
5904 }
5905
5906 break;
5907 }
5908
5909 case Intrinsic::vector_extract: {
5910 StringRef Name = F->getName();
5911 Name = Name.substr(Start: 5); // Strip llvm
5912 if (!Name.starts_with(Prefix: "aarch64.sve.tuple.get")) {
5913 DefaultCase();
5914 return;
5915 }
5916 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5917 unsigned MinElts = RetTy->getMinNumElements();
5918 unsigned I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
5919 Value *NewIdx = ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * MinElts);
5920 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), NewIdx});
5921 break;
5922 }
5923
5924 case Intrinsic::vector_insert: {
5925 StringRef Name = F->getName();
5926 Name = Name.substr(Start: 5);
5927 if (!Name.starts_with(Prefix: "aarch64.sve.tuple")) {
5928 DefaultCase();
5929 return;
5930 }
5931 if (Name.starts_with(Prefix: "aarch64.sve.tuple.set")) {
5932 unsigned I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
5933 auto *Ty = cast<ScalableVectorType>(Val: CI->getArgOperand(i: 2)->getType());
5934 Value *NewIdx =
5935 ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * Ty->getMinNumElements());
5936 NewCall = Builder.CreateCall(
5937 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), NewIdx});
5938 break;
5939 }
5940 if (Name.starts_with(Prefix: "aarch64.sve.tuple.create")) {
5941 unsigned N = StringSwitch<unsigned>(Name)
5942 .StartsWith(S: "aarch64.sve.tuple.create2", Value: 2)
5943 .StartsWith(S: "aarch64.sve.tuple.create3", Value: 3)
5944 .StartsWith(S: "aarch64.sve.tuple.create4", Value: 4)
5945 .Default(Value: 0);
5946 assert(N > 1 && "Create is expected to be between 2-4");
5947 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5948 Value *Ret = llvm::PoisonValue::get(T: RetTy);
5949 unsigned MinElts = RetTy->getMinNumElements() / N;
5950 for (unsigned I = 0; I < N; I++) {
5951 Value *V = CI->getArgOperand(i: I);
5952 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: V, Idx: I * MinElts);
5953 }
5954 NewCall = dyn_cast<CallInst>(Val: Ret);
5955 }
5956 break;
5957 }
5958
5959 case Intrinsic::arm_neon_bfdot:
5960 case Intrinsic::arm_neon_bfmmla:
5961 case Intrinsic::arm_neon_bfmlalb:
5962 case Intrinsic::arm_neon_bfmlalt:
5963 case Intrinsic::aarch64_neon_bfdot:
5964 case Intrinsic::aarch64_neon_bfmmla:
5965 case Intrinsic::aarch64_neon_bfmlalb:
5966 case Intrinsic::aarch64_neon_bfmlalt: {
5967 SmallVector<Value *, 3> Args;
5968 assert(CI->arg_size() == 3 &&
5969 "Mismatch between function args and call args");
5970 size_t OperandWidth =
5971 CI->getArgOperand(i: 1)->getType()->getPrimitiveSizeInBits();
5972 assert((OperandWidth == 64 || OperandWidth == 128) &&
5973 "Unexpected operand width");
5974 Type *NewTy = FixedVectorType::get(ElementType: Type::getBFloatTy(C), NumElts: OperandWidth / 16);
5975 auto Iter = CI->args().begin();
5976 Args.push_back(Elt: *Iter++);
5977 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
5978 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
5979 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5980 break;
5981 }
5982
5983 case Intrinsic::bitreverse:
5984 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
5985 break;
5986
5987 case Intrinsic::ctlz:
5988 case Intrinsic::cttz: {
5989 if (CI->arg_size() != 1) {
5990 DefaultCase();
5991 return;
5992 }
5993
5994 NewCall =
5995 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), Builder.getFalse()});
5996 break;
5997 }
5998
5999 case Intrinsic::objectsize: {
6000 Value *NullIsUnknownSize =
6001 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(i: 2);
6002 Value *Dynamic =
6003 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(i: 3);
6004 NewCall = Builder.CreateCall(
6005 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), NullIsUnknownSize, Dynamic});
6006 break;
6007 }
6008
6009 case Intrinsic::ctpop:
6010 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
6011 break;
6012 case Intrinsic::dbg_value: {
6013 StringRef Name = F->getName();
6014 Name = Name.substr(Start: 5); // Strip llvm.
6015 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6016 if (Name.starts_with(Prefix: "dbg.addr")) {
6017 DIExpression *Expr = cast<DIExpression>(
6018 Val: cast<MetadataAsValue>(Val: CI->getArgOperand(i: 2))->getMetadata());
6019 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
6020 NewCall =
6021 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6022 MetadataAsValue::get(Context&: C, MD: Expr)});
6023 break;
6024 }
6025
6026 // Upgrade from the old version that had an extra offset argument.
6027 assert(CI->arg_size() == 4);
6028 // Drop nonzero offsets instead of attempting to upgrade them.
6029 if (auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1)))
6030 if (Offset->isNullValue()) {
6031 NewCall = Builder.CreateCall(
6032 Callee: NewFn,
6033 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)});
6034 break;
6035 }
6036 CI->eraseFromParent();
6037 return;
6038 }
6039
6040 case Intrinsic::ptr_annotation:
6041 // Upgrade from versions that lacked the annotation attribute argument.
6042 if (CI->arg_size() != 4) {
6043 DefaultCase();
6044 return;
6045 }
6046
6047 // Create a new call with an added null annotation attribute argument.
6048 NewCall = Builder.CreateCall(
6049 Callee: NewFn,
6050 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
6051 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
6052 NewCall->takeName(V: CI);
6053 CI->replaceAllUsesWith(V: NewCall);
6054 CI->eraseFromParent();
6055 return;
6056
6057 case Intrinsic::var_annotation:
6058 // Upgrade from versions that lacked the annotation attribute argument.
6059 if (CI->arg_size() != 4) {
6060 DefaultCase();
6061 return;
6062 }
6063 // Create a new call with an added null annotation attribute argument.
6064 NewCall = Builder.CreateCall(
6065 Callee: NewFn,
6066 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
6067 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
6068 NewCall->takeName(V: CI);
6069 CI->replaceAllUsesWith(V: NewCall);
6070 CI->eraseFromParent();
6071 return;
6072
6073 case Intrinsic::riscv_aes32dsi:
6074 case Intrinsic::riscv_aes32dsmi:
6075 case Intrinsic::riscv_aes32esi:
6076 case Intrinsic::riscv_aes32esmi:
6077 case Intrinsic::riscv_sm4ks:
6078 case Intrinsic::riscv_sm4ed: {
6079 // The last argument to these intrinsics used to be i8 and changed to i32.
6080 // The type overload for sm4ks and sm4ed was removed.
6081 Value *Arg2 = CI->getArgOperand(i: 2);
6082 if (Arg2->getType()->isIntegerTy(BitWidth: 32) && !CI->getType()->isIntegerTy(BitWidth: 64))
6083 return;
6084
6085 Value *Arg0 = CI->getArgOperand(i: 0);
6086 Value *Arg1 = CI->getArgOperand(i: 1);
6087 if (CI->getType()->isIntegerTy(BitWidth: 64)) {
6088 Arg0 = Builder.CreateTrunc(V: Arg0, DestTy: Builder.getInt32Ty());
6089 Arg1 = Builder.CreateTrunc(V: Arg1, DestTy: Builder.getInt32Ty());
6090 }
6091
6092 Arg2 = ConstantInt::get(Ty: Type::getInt32Ty(C),
6093 V: cast<ConstantInt>(Val: Arg2)->getZExtValue());
6094
6095 NewCall = Builder.CreateCall(Callee: NewFn, Args: {Arg0, Arg1, Arg2});
6096 Value *Res = NewCall;
6097 if (Res->getType() != CI->getType())
6098 Res = Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
6099 NewCall->takeName(V: CI);
6100 CI->replaceAllUsesWith(V: Res);
6101 CI->eraseFromParent();
6102 return;
6103 }
6104 case Intrinsic::nvvm_mapa_shared_cluster: {
6105 // Create a new call with the correct address space.
6106 NewCall =
6107 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
6108 Value *Res = NewCall;
6109 Res = Builder.CreateAddrSpaceCast(
6110 V: Res, DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED));
6111 NewCall->takeName(V: CI);
6112 CI->replaceAllUsesWith(V: Res);
6113 CI->eraseFromParent();
6114 return;
6115 }
6116 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6117 SmallVector<Value *, 4> Args(CI->args());
6118 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6119 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6120 Args[0] = Builder.CreateAddrSpaceCast(
6121 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6122
6123 // Append the missing trailing flag_valid_pattern (0 = disabled).
6124 Args.push_back(Elt: Builder.getInt32(C: 0));
6125
6126 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6127 NewCall->takeName(V: CI);
6128 CI->replaceAllUsesWith(V: NewCall);
6129 CI->eraseFromParent();
6130 return;
6131 }
6132 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6133 // (dst, mbar, src, size, ch, flag_ch)
6134 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6135 // i32 0 /* flag_valid_pattern=disabled */)
6136 SmallVector<Value *, 10> Args;
6137 for (unsigned I = 0; I < 4; ++I)
6138 Args.push_back(Elt: CI->getArgOperand(i: I));
6139 Args.push_back(Elt: Builder.getInt32(C: 0)); // ignore_bytes_left
6140 Args.push_back(Elt: Builder.getInt32(C: 0)); // ignore_bytes_right
6141 Args.push_back(Elt: CI->getArgOperand(i: 4)); // cache_hint
6142 Args.push_back(Elt: CI->getArgOperand(i: 5)); // flag_ch
6143 Args.push_back(Elt: Builder.getInt1(V: false)); // flag_oob
6144 Args.push_back(Elt: Builder.getInt32(C: 0)); // flag_valid_pattern
6145
6146 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6147 NewCall->takeName(V: CI);
6148 CI->replaceAllUsesWith(V: NewCall);
6149 CI->eraseFromParent();
6150 return;
6151 }
6152 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6153 // Create a new call with the correct address space.
6154 SmallVector<Value *, 4> Args(CI->args());
6155 Args[0] = Builder.CreateAddrSpaceCast(
6156 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6157
6158 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6159 NewCall->takeName(V: CI);
6160 CI->replaceAllUsesWith(V: NewCall);
6161 CI->eraseFromParent();
6162 return;
6163 }
6164 // clang-format off
6165#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6166 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6167 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6168#undef G2S_CLUSTER_CASE
6169 {
6170 SmallVector<Value *, 16> Args(CI->args());
6171 unsigned AS = CI->getArgOperand(i: 0)->getType()->getPointerAddressSpace();
6172 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6173 Args[0] = Builder.CreateAddrSpaceCast(
6174 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6175
6176 // Append the missing trailing arguments with default values (cta_group,
6177 // flag_valid_pattern).
6178 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6179 Args.push_back(Elt: Builder.getInt32(C: 0));
6180
6181 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6182 NewCall->takeName(V: CI);
6183 CI->replaceAllUsesWith(V: NewCall);
6184 CI->eraseFromParent();
6185 return;
6186 }
6187
6188#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6189 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6190 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6191#undef G2S_CTA_CASE
6192 {
6193 SmallVector<Value *, 16> Args(CI->args());
6194 // Append the missing trailing flag_valid_pattern argument with default
6195 // value 0.
6196 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6197 "expected only the trailing flag_valid_pattern to be missing");
6198 Args.push_back(Elt: Builder.getInt32(C: 0));
6199
6200 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6201 NewCall->takeName(V: CI);
6202 CI->replaceAllUsesWith(V: NewCall);
6203 CI->eraseFromParent();
6204 return;
6205 }
6206#undef NVVM_TMA_G2S_MODES
6207 // clang-format on
6208
6209 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6210 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6211 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6212 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6213 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6214 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6215 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6216 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6217 StringRef Name = F->getName();
6218 Name.consume_front(Prefix: "llvm.nvvm.cp.async.bulk.tensor.reduce.");
6219 auto RedOp = getNVPTXTMAReductionOp(Name: Name.split(Separator: '.').first);
6220
6221 SmallVector<Value *, 16> Args(CI->args());
6222 Args.insert(I: Args.end() - 1, Elt: Builder.getInt32(C: *RedOp));
6223 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6224 break;
6225 }
6226 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6227 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6228 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6229 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6230 NewCall =
6231 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6232 Builder.getFalse()});
6233 break;
6234 case Intrinsic::nvvm_mbarrier_init: {
6235 SmallVector<Value *, 3> Args(CI->args());
6236 // The .shared variant folded into the overloaded form without gaining an
6237 // operand, so only the pre-layout two-argument form needs one appended.
6238 if (Args.size() == 2)
6239 Args.push_back(Elt: Builder.getInt32(C: 0)); // layout = default(0)
6240 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6241 break;
6242 }
6243 case Intrinsic::riscv_sha256sig0:
6244 case Intrinsic::riscv_sha256sig1:
6245 case Intrinsic::riscv_sha256sum0:
6246 case Intrinsic::riscv_sha256sum1:
6247 case Intrinsic::riscv_sm3p0:
6248 case Intrinsic::riscv_sm3p1: {
6249 // The last argument to these intrinsics used to be i8 and changed to i32.
6250 // The type overload for sm4ks and sm4ed was removed.
6251 if (!CI->getType()->isIntegerTy(BitWidth: 64))
6252 return;
6253
6254 Value *Arg =
6255 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getInt32Ty());
6256
6257 NewCall = Builder.CreateCall(Callee: NewFn, Args: Arg);
6258 Value *Res =
6259 Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
6260 NewCall->takeName(V: CI);
6261 CI->replaceAllUsesWith(V: Res);
6262 CI->eraseFromParent();
6263 return;
6264 }
6265
6266 case Intrinsic::x86_xop_vfrcz_ss:
6267 case Intrinsic::x86_xop_vfrcz_sd:
6268 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 1)});
6269 break;
6270
6271 case Intrinsic::x86_xop_vpermil2pd:
6272 case Intrinsic::x86_xop_vpermil2ps:
6273 case Intrinsic::x86_xop_vpermil2pd_256:
6274 case Intrinsic::x86_xop_vpermil2ps_256: {
6275 SmallVector<Value *, 4> Args(CI->args());
6276 VectorType *FltIdxTy = cast<VectorType>(Val: Args[2]->getType());
6277 VectorType *IntIdxTy = VectorType::getInteger(VTy: FltIdxTy);
6278 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: IntIdxTy);
6279 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6280 break;
6281 }
6282
6283 case Intrinsic::x86_sse41_ptestc:
6284 case Intrinsic::x86_sse41_ptestz:
6285 case Intrinsic::x86_sse41_ptestnzc: {
6286 // The arguments for these intrinsics used to be v4f32, and changed
6287 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6288 // So, the only thing required is a bitcast for both arguments.
6289 // First, check the arguments have the old type.
6290 Value *Arg0 = CI->getArgOperand(i: 0);
6291 if (Arg0->getType() != FixedVectorType::get(ElementType: Type::getFloatTy(C), NumElts: 4))
6292 return;
6293
6294 // Old intrinsic, add bitcasts
6295 Value *Arg1 = CI->getArgOperand(i: 1);
6296
6297 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
6298
6299 Value *BC0 = Builder.CreateBitCast(V: Arg0, DestTy: NewVecTy, Name: "cast");
6300 Value *BC1 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
6301
6302 NewCall = Builder.CreateCall(Callee: NewFn, Args: {BC0, BC1});
6303 break;
6304 }
6305
6306 case Intrinsic::x86_rdtscp: {
6307 // This used to take 1 arguments. If we have no arguments, it is already
6308 // upgraded.
6309 if (CI->getNumOperands() == 0)
6310 return;
6311
6312 NewCall = Builder.CreateCall(Callee: NewFn);
6313 // Extract the second result and store it.
6314 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
6315 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
6316 // Replace the original call result with the first result of the new call.
6317 Value *TSC = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
6318
6319 NewCall->takeName(V: CI);
6320 CI->replaceAllUsesWith(V: TSC);
6321 CI->eraseFromParent();
6322 return;
6323 }
6324
6325 case Intrinsic::x86_sse41_insertps:
6326 case Intrinsic::x86_sse41_dppd:
6327 case Intrinsic::x86_sse41_dpps:
6328 case Intrinsic::x86_sse41_mpsadbw:
6329 case Intrinsic::x86_avx_dp_ps_256:
6330 case Intrinsic::x86_avx2_mpsadbw: {
6331 // Need to truncate the last argument from i32 to i8 -- this argument models
6332 // an inherently 8-bit immediate operand to these x86 instructions.
6333 SmallVector<Value *, 4> Args(CI->args());
6334
6335 // Replace the last argument with a trunc.
6336 Args.back() = Builder.CreateTrunc(V: Args.back(), DestTy: Type::getInt8Ty(C), Name: "trunc");
6337 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6338 break;
6339 }
6340
6341 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6342 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6343 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6344 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6345 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6346 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6347 SmallVector<Value *, 4> Args(CI->args());
6348 unsigned NumElts =
6349 cast<FixedVectorType>(Val: Args[0]->getType())->getNumElements();
6350 Args[3] = getX86MaskVec(Builder, Mask: Args[3], NumElts);
6351
6352 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6353 Value *Res = applyX86MaskOn1BitsVec(Builder, Vec: NewCall, Mask: nullptr);
6354
6355 NewCall->takeName(V: CI);
6356 CI->replaceAllUsesWith(V: Res);
6357 CI->eraseFromParent();
6358 return;
6359 }
6360
6361 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6362 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6363 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6364 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6365 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6366 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6367 SmallVector<Value *, 4> Args(CI->args());
6368 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
6369 if (NewFn->getIntrinsicID() ==
6370 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6371 Args[1] = Builder.CreateBitCast(
6372 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6373
6374 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6375 Value *Res = Builder.CreateBitCast(
6376 V: NewCall, DestTy: FixedVectorType::get(ElementType: Builder.getInt16Ty(), NumElts));
6377
6378 NewCall->takeName(V: CI);
6379 CI->replaceAllUsesWith(V: Res);
6380 CI->eraseFromParent();
6381 return;
6382 }
6383 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6384 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6385 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6386 SmallVector<Value *, 4> Args(CI->args());
6387 unsigned NumElts =
6388 cast<FixedVectorType>(Val: CI->getType())->getNumElements() * 2;
6389 Args[1] = Builder.CreateBitCast(
6390 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6391 Args[2] = Builder.CreateBitCast(
6392 V: Args[2], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6393
6394 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6395 break;
6396 }
6397
6398 case Intrinsic::thread_pointer: {
6399 NewCall = Builder.CreateCall(Callee: NewFn, Args: {});
6400 break;
6401 }
6402
6403 case Intrinsic::memcpy:
6404 case Intrinsic::memmove:
6405 case Intrinsic::memset: {
6406 // We have to make sure that the call signature is what we're expecting.
6407 // We only want to change the old signatures by removing the alignment arg:
6408 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6409 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6410 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6411 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6412 // Note: i8*'s in the above can be any pointer type
6413 if (CI->arg_size() != 5) {
6414 DefaultCase();
6415 return;
6416 }
6417 // Remove alignment argument (3), and add alignment attributes to the
6418 // dest/src pointers.
6419 Value *Args[4] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6420 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
6421 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6422 AttributeList OldAttrs = CI->getAttributes();
6423 AttributeList NewAttrs = AttributeList::get(
6424 C, FnAttrs: OldAttrs.getFnAttrs(), RetAttrs: OldAttrs.getRetAttrs(),
6425 ArgAttrs: {OldAttrs.getParamAttrs(ArgNo: 0), OldAttrs.getParamAttrs(ArgNo: 1),
6426 OldAttrs.getParamAttrs(ArgNo: 2), OldAttrs.getParamAttrs(ArgNo: 4)});
6427 NewCall->setAttributes(NewAttrs);
6428 auto *MemCI = cast<MemIntrinsic>(Val: NewCall);
6429 // All mem intrinsics support dest alignment.
6430 const ConstantInt *Align = cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
6431 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6432 // Memcpy/Memmove also support source alignment.
6433 if (auto *MTI = dyn_cast<MemTransferInst>(Val: MemCI))
6434 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6435 break;
6436 }
6437
6438 case Intrinsic::masked_load:
6439 case Intrinsic::masked_gather:
6440 case Intrinsic::masked_store:
6441 case Intrinsic::masked_scatter: {
6442 if (CI->arg_size() != 4) {
6443 DefaultCase();
6444 return;
6445 }
6446
6447 auto GetMaybeAlign = [](Value *Op) {
6448 if (auto *CI = dyn_cast<ConstantInt>(Val: Op)) {
6449 uint64_t Val = CI->getZExtValue();
6450 if (Val == 0)
6451 return MaybeAlign();
6452 if (isPowerOf2_64(Value: Val))
6453 return MaybeAlign(Val);
6454 }
6455 reportFatalUsageError(reason: "Invalid alignment argument");
6456 };
6457 auto GetAlign = [&](Value *Op) {
6458 MaybeAlign Align = GetMaybeAlign(Op);
6459 if (Align)
6460 return *Align;
6461 reportFatalUsageError(reason: "Invalid zero alignment argument");
6462 };
6463
6464 const DataLayout &DL = CI->getDataLayout();
6465 switch (NewFn->getIntrinsicID()) {
6466 case Intrinsic::masked_load:
6467 NewCall = Builder.CreateMaskedLoad(
6468 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0), Alignment: GetAlign(CI->getArgOperand(i: 1)),
6469 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6470 break;
6471 case Intrinsic::masked_gather:
6472 NewCall = Builder.CreateMaskedGather(
6473 Ty: CI->getType(), Ptrs: CI->getArgOperand(i: 0),
6474 Alignment: DL.getValueOrABITypeAlignment(Alignment: GetMaybeAlign(CI->getArgOperand(i: 1)),
6475 Ty: CI->getType()->getScalarType()),
6476 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6477 break;
6478 case Intrinsic::masked_store:
6479 NewCall = Builder.CreateMaskedStore(
6480 Val: CI->getArgOperand(i: 0), Ptr: CI->getArgOperand(i: 1),
6481 Alignment: GetAlign(CI->getArgOperand(i: 2)), Mask: CI->getArgOperand(i: 3));
6482 break;
6483 case Intrinsic::masked_scatter:
6484 NewCall = Builder.CreateMaskedScatter(
6485 Val: CI->getArgOperand(i: 0), Ptrs: CI->getArgOperand(i: 1),
6486 Alignment: DL.getValueOrABITypeAlignment(
6487 Alignment: GetMaybeAlign(CI->getArgOperand(i: 2)),
6488 Ty: CI->getArgOperand(i: 0)->getType()->getScalarType()),
6489 Mask: CI->getArgOperand(i: 3));
6490 break;
6491 default:
6492 llvm_unreachable("Unexpected intrinsic ID");
6493 }
6494 // Previous metadata is still valid.
6495 NewCall->copyMetadata(SrcInst: *CI);
6496 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6497 break;
6498 }
6499
6500 case Intrinsic::lifetime_start:
6501 case Intrinsic::lifetime_end: {
6502 if (CI->arg_size() != 2) {
6503 DefaultCase();
6504 return;
6505 }
6506
6507 Value *Ptr = CI->getArgOperand(i: 1);
6508 // Try to strip pointer casts, such that the lifetime works on an alloca.
6509 Ptr = Ptr->stripPointerCasts();
6510 if (isa<AllocaInst>(Val: Ptr)) {
6511 // Don't use NewFn, as we might have looked through an addrspacecast.
6512 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6513 NewCall = Builder.CreateLifetimeStart(Ptr);
6514 else
6515 NewCall = Builder.CreateLifetimeEnd(Ptr);
6516 break;
6517 }
6518
6519 // Otherwise remove the lifetime marker.
6520 CI->eraseFromParent();
6521 return;
6522 }
6523
6524 case Intrinsic::x86_avx512_vpdpbusd_128:
6525 case Intrinsic::x86_avx512_vpdpbusd_256:
6526 case Intrinsic::x86_avx512_vpdpbusd_512:
6527 case Intrinsic::x86_avx512_vpdpbusds_128:
6528 case Intrinsic::x86_avx512_vpdpbusds_256:
6529 case Intrinsic::x86_avx512_vpdpbusds_512:
6530 case Intrinsic::x86_avx2_vpdpbssd_128:
6531 case Intrinsic::x86_avx2_vpdpbssd_256:
6532 case Intrinsic::x86_avx10_vpdpbssd_512:
6533 case Intrinsic::x86_avx2_vpdpbssds_128:
6534 case Intrinsic::x86_avx2_vpdpbssds_256:
6535 case Intrinsic::x86_avx10_vpdpbssds_512:
6536 case Intrinsic::x86_avx2_vpdpbsud_128:
6537 case Intrinsic::x86_avx2_vpdpbsud_256:
6538 case Intrinsic::x86_avx10_vpdpbsud_512:
6539 case Intrinsic::x86_avx2_vpdpbsuds_128:
6540 case Intrinsic::x86_avx2_vpdpbsuds_256:
6541 case Intrinsic::x86_avx10_vpdpbsuds_512:
6542 case Intrinsic::x86_avx2_vpdpbuud_128:
6543 case Intrinsic::x86_avx2_vpdpbuud_256:
6544 case Intrinsic::x86_avx10_vpdpbuud_512:
6545 case Intrinsic::x86_avx2_vpdpbuuds_128:
6546 case Intrinsic::x86_avx2_vpdpbuuds_256:
6547 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6548 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6549 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6550 CI->getArgOperand(i: 2)};
6551 Type *NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: NumElts, Scalable: false);
6552 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6553 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6554
6555 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6556 break;
6557 }
6558 case Intrinsic::x86_avx512_vpdpwssd_128:
6559 case Intrinsic::x86_avx512_vpdpwssd_256:
6560 case Intrinsic::x86_avx512_vpdpwssd_512:
6561 case Intrinsic::x86_avx512_vpdpwssds_128:
6562 case Intrinsic::x86_avx512_vpdpwssds_256:
6563 case Intrinsic::x86_avx512_vpdpwssds_512:
6564 case Intrinsic::x86_avx2_vpdpwsud_128:
6565 case Intrinsic::x86_avx2_vpdpwsud_256:
6566 case Intrinsic::x86_avx10_vpdpwsud_512:
6567 case Intrinsic::x86_avx2_vpdpwsuds_128:
6568 case Intrinsic::x86_avx2_vpdpwsuds_256:
6569 case Intrinsic::x86_avx10_vpdpwsuds_512:
6570 case Intrinsic::x86_avx2_vpdpwusd_128:
6571 case Intrinsic::x86_avx2_vpdpwusd_256:
6572 case Intrinsic::x86_avx10_vpdpwusd_512:
6573 case Intrinsic::x86_avx2_vpdpwusds_128:
6574 case Intrinsic::x86_avx2_vpdpwusds_256:
6575 case Intrinsic::x86_avx10_vpdpwusds_512:
6576 case Intrinsic::x86_avx2_vpdpwuud_128:
6577 case Intrinsic::x86_avx2_vpdpwuud_256:
6578 case Intrinsic::x86_avx10_vpdpwuud_512:
6579 case Intrinsic::x86_avx2_vpdpwuuds_128:
6580 case Intrinsic::x86_avx2_vpdpwuuds_256:
6581 case Intrinsic::x86_avx10_vpdpwuuds_512:
6582 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6583 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6584 CI->getArgOperand(i: 2)};
6585 Type *NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: NumElts, Scalable: false);
6586 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6587 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6588
6589 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6590 break;
6591 }
6592 assert(NewCall && "Should have either set this variable or returned through "
6593 "the default case");
6594 NewCall->takeName(V: CI);
6595 CI->replaceAllUsesWith(V: NewCall);
6596 CI->eraseFromParent();
6597}
6598
6599void llvm::UpgradeCallsToIntrinsic(Function *F) {
6600 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6601
6602 // Check if this function should be upgraded and get the replacement function
6603 // if there is one.
6604 Function *NewFn;
6605 if (UpgradeIntrinsicFunction(F, NewFn)) {
6606 // Replace all users of the old function with the new function or new
6607 // instructions. This is not a range loop because the call is deleted.
6608 for (User *U : make_early_inc_range(Range: F->users()))
6609 if (CallBase *CB = dyn_cast<CallBase>(Val: U))
6610 UpgradeIntrinsicCall(CI: CB, NewFn);
6611
6612 // Remove old function, no longer used, from the module.
6613 if (F != NewFn)
6614 F->eraseFromParent();
6615 }
6616}
6617
6618MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6619 const unsigned NumOperands = MD.getNumOperands();
6620 if (NumOperands == 0)
6621 return &MD; // Invalid, punt to a verifier error.
6622
6623 // Check if the tag uses struct-path aware TBAA format.
6624 if (isa<MDNode>(Val: MD.getOperand(I: 0)) && NumOperands >= 3)
6625 return &MD;
6626
6627 auto &Context = MD.getContext();
6628 if (NumOperands == 3) {
6629 Metadata *Elts[] = {MD.getOperand(I: 0), MD.getOperand(I: 1)};
6630 MDNode *ScalarType = MDNode::get(Context, MDs: Elts);
6631 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6632 Metadata *Elts2[] = {ScalarType, ScalarType,
6633 ConstantAsMetadata::get(
6634 C: Constant::getNullValue(Ty: Type::getInt64Ty(C&: Context))),
6635 MD.getOperand(I: 2)};
6636 return MDNode::get(Context, MDs: Elts2);
6637 }
6638 // Create a MDNode <MD, MD, offset 0>
6639 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(C: Constant::getNullValue(
6640 Ty: Type::getInt64Ty(C&: Context)))};
6641 return MDNode::get(Context, MDs: Elts);
6642}
6643
6644Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6645 Instruction *&Temp) {
6646 if (Opc != Instruction::BitCast)
6647 return nullptr;
6648
6649 Temp = nullptr;
6650 Type *SrcTy = V->getType();
6651 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6652 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6653 LLVMContext &Context = V->getContext();
6654
6655 // We have no information about target data layout, so we assume that
6656 // the maximum pointer size is 64bit.
6657 Type *MidTy = Type::getInt64Ty(C&: Context);
6658 Temp = CastInst::Create(Instruction::PtrToInt, S: V, Ty: MidTy);
6659
6660 return CastInst::Create(Instruction::IntToPtr, S: Temp, Ty: DestTy);
6661 }
6662
6663 return nullptr;
6664}
6665
6666Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6667 if (Opc != Instruction::BitCast)
6668 return nullptr;
6669
6670 Type *SrcTy = C->getType();
6671 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6672 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6673 LLVMContext &Context = C->getContext();
6674
6675 // We have no information about target data layout, so we assume that
6676 // the maximum pointer size is 64bit.
6677 Type *MidTy = Type::getInt64Ty(C&: Context);
6678
6679 return ConstantExpr::getIntToPtr(C: ConstantExpr::getPtrToInt(C, Ty: MidTy),
6680 Ty: DestTy);
6681 }
6682
6683 return nullptr;
6684}
6685
6686static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6687 if (Flag.getNumOperands() < 3)
6688 return std::nullopt;
6689 if (MDString *Name = dyn_cast_or_null<MDString>(Val: Flag.getOperand(I: 1)))
6690 return Name->getString();
6691 return std::nullopt;
6692}
6693
6694/// Check the debug info version number, if it is out-dated, drop the debug
6695/// info. Return true if module is modified.
6696bool llvm::UpgradeDebugInfo(Module &M) {
6697 if (DisableAutoUpgradeDebugInfo)
6698 return false;
6699
6700 llvm::TimeTraceScope timeScope("Upgrade debug info");
6701 // We need to get metadata before the module is verified (i.e., getModuleFlag
6702 // makes assumptions that we haven't verified yet). Carefully extract the flag
6703 // from the metadata.
6704 unsigned Version = 0;
6705 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6706 auto OpIt = find_if(Range: ModFlags->operands(), P: [](const MDNode *Flag) {
6707 if (auto Name = getModuleFlagNameSafely(Flag: *Flag))
6708 return *Name == "Debug Info Version";
6709 return false;
6710 });
6711 if (OpIt != ModFlags->op_end()) {
6712 const MDOperand &ValOp = (*OpIt)->getOperand(I: 2);
6713 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(MD: ValOp))
6714 Version = CI->getZExtValue();
6715 }
6716 }
6717
6718 if (Version == DEBUG_METADATA_VERSION) {
6719 bool BrokenDebugInfo = false;
6720 if (verifyModule(M, OS: &llvm::errs(), BrokenDebugInfo: &BrokenDebugInfo))
6721 report_fatal_error(reason: "Broken module found, compilation aborted!");
6722 if (!BrokenDebugInfo)
6723 // Everything is ok.
6724 return false;
6725 else {
6726 // Diagnose malformed debug info.
6727 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6728 M.getContext().diagnose(DI: Diag);
6729 }
6730 }
6731 bool Modified = StripDebugInfo(M);
6732 if (Modified && Version != DEBUG_METADATA_VERSION) {
6733 // Diagnose a version mismatch.
6734 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6735 M.getContext().diagnose(DI: DiagVersion);
6736 }
6737 return Modified;
6738}
6739
6740static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6741 GlobalValue *GV, const Metadata *V) {
6742 Function *F = cast<Function>(Val: GV);
6743
6744 constexpr StringLiteral DefaultValue = "1";
6745 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6746 unsigned Length = 0;
6747
6748 if (F->hasFnAttribute(Kind: Attr)) {
6749 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6750 // parse these elements placing them into Vect3
6751 StringRef S = F->getFnAttribute(Kind: Attr).getValueAsString();
6752 for (; Length < 3 && !S.empty(); Length++) {
6753 auto [Part, Rest] = S.split(Separator: ',');
6754 Vect3[Length] = Part.trim();
6755 S = Rest;
6756 }
6757 }
6758
6759 const unsigned Dim = DimC - 'x';
6760 assert(Dim < 3 && "Unexpected dim char");
6761
6762 const uint64_t VInt = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6763
6764 // local variable required for StringRef in Vect3 to point to.
6765 const std::string VStr = llvm::utostr(X: VInt);
6766 Vect3[Dim] = VStr;
6767 Length = std::max(a: Length, b: Dim + 1);
6768
6769 const std::string NewAttr = llvm::join(R: ArrayRef(Vect3, Length), Separator: ",");
6770 F->addFnAttr(Kind: Attr, Val: NewAttr);
6771}
6772
6773static inline bool isXYZ(StringRef S) {
6774 return S == "x" || S == "y" || S == "z";
6775}
6776
6777bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6778 const Metadata *V) {
6779 if (K == "kernel") {
6780 if (!mdconst::extract<ConstantInt>(MD&: V)->isZero())
6781 cast<Function>(Val: GV)->setCallingConv(CallingConv::PTX_Kernel);
6782 return true;
6783 }
6784 if (K == "align") {
6785 // V is a bitfeild specifying two 16-bit values. The alignment value is
6786 // specfied in low 16-bits, The index is specified in the high bits. For the
6787 // index, 0 indicates the return value while higher values correspond to
6788 // each parameter (idx = param + 1).
6789 const uint64_t AlignIdxValuePair =
6790 mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6791 const unsigned Idx = (AlignIdxValuePair >> 16);
6792 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6793 cast<Function>(Val: GV)->addAttributeAtIndex(
6794 i: Idx, Attr: Attribute::getWithStackAlignment(Context&: GV->getContext(), Alignment: StackAlign));
6795 return true;
6796 }
6797 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6798 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6799 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: CV));
6800 return true;
6801 }
6802 if (K == "minctasm") {
6803 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6804 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MinCTASm, Val: llvm::utostr(X: CV));
6805 return true;
6806 }
6807 if (K == "maxnreg") {
6808 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6809 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxNReg, Val: llvm::utostr(X: CV));
6810 return true;
6811 }
6812 if (K.consume_front(Prefix: "maxntid") && isXYZ(S: K)) {
6813 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::MaxNTID, DimC: K[0], GV, V);
6814 return true;
6815 }
6816 if (K.consume_front(Prefix: "reqntid") && isXYZ(S: K)) {
6817 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ReqNTID, DimC: K[0], GV, V);
6818 return true;
6819 }
6820 if (K.consume_front(Prefix: "cluster_dim_") && isXYZ(S: K)) {
6821 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ClusterDim, DimC: K[0], GV, V);
6822 return true;
6823 }
6824 if (K == "grid_constant") {
6825 const auto Attr = Attribute::get(Context&: GV->getContext(), Kind: NVVMAttr::GridConstant);
6826 for (const auto &Op : cast<MDNode>(Val: V)->operands()) {
6827 // For some reason, the index is 1-based in the metadata. Good thing we're
6828 // able to auto-upgrade it!
6829 const auto Index = mdconst::extract<ConstantInt>(MD: Op)->getZExtValue() - 1;
6830 cast<Function>(Val: GV)->addParamAttr(ArgNo: Index, Attr);
6831 }
6832 return true;
6833 }
6834
6835 return false;
6836}
6837
6838void llvm::UpgradeNVVMAnnotations(Module &M) {
6839 NamedMDNode *NamedMD = M.getNamedMetadata(Name: "nvvm.annotations");
6840 if (!NamedMD)
6841 return;
6842
6843 SmallVector<MDNode *, 8> NewNodes;
6844 SmallPtrSet<const MDNode *, 8> SeenNodes;
6845 for (MDNode *MD : NamedMD->operands()) {
6846 if (!SeenNodes.insert(Ptr: MD).second)
6847 continue;
6848
6849 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD: MD->getOperand(I: 0));
6850 if (!GV)
6851 continue;
6852
6853 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6854
6855 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(I: 0)};
6856 // Each nvvm.annotations metadata entry will be of the following form:
6857 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6858 // start index = 1, to skip the global variable key
6859 // increment = 2, to skip the value for each property-value pairs
6860 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6861 MDString *K = cast<MDString>(Val: MD->getOperand(I: j));
6862 const MDOperand &V = MD->getOperand(I: j + 1);
6863 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K: K->getString(), V);
6864 if (!Upgraded)
6865 NewOperands.append(IL: {K, V});
6866 }
6867
6868 if (NewOperands.size() > 1)
6869 NewNodes.push_back(Elt: MDNode::get(Context&: M.getContext(), MDs: NewOperands));
6870 }
6871
6872 NamedMD->clearOperands();
6873 for (MDNode *N : NewNodes)
6874 NamedMD->addOperand(M: N);
6875}
6876
6877/// This checks for objc retain release marker which should be upgraded. It
6878/// returns true if module is modified.
6879static bool upgradeRetainReleaseMarker(Module &M) {
6880 bool Changed = false;
6881 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6882 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(Name: MarkerKey);
6883 if (ModRetainReleaseMarker) {
6884 MDNode *Op = ModRetainReleaseMarker->getOperand(i: 0);
6885 if (Op) {
6886 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 0));
6887 if (ID) {
6888 SmallVector<StringRef, 4> ValueComp;
6889 ID->getString().split(A&: ValueComp, Separator: "#");
6890 if (ValueComp.size() == 2) {
6891 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6892 ID = MDString::get(Context&: M.getContext(), Str: NewValue);
6893 }
6894 M.addModuleFlag(Behavior: Module::Error, Key: MarkerKey, Val: ID);
6895 M.eraseNamedMetadata(NMD: ModRetainReleaseMarker);
6896 Changed = true;
6897 }
6898 }
6899 }
6900 return Changed;
6901}
6902
6903void llvm::UpgradeARCRuntime(Module &M) {
6904 // This lambda converts normal function calls to ARC runtime functions to
6905 // intrinsic calls.
6906 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6907 llvm::Intrinsic::ID IntrinsicFunc) {
6908 Function *Fn = M.getFunction(Name: OldFunc);
6909
6910 if (!Fn)
6911 return;
6912
6913 Function *NewFn =
6914 llvm::Intrinsic::getOrInsertDeclaration(M: &M, id: IntrinsicFunc);
6915
6916 for (User *U : make_early_inc_range(Range: Fn->users())) {
6917 CallInst *CI = dyn_cast<CallInst>(Val: U);
6918 if (!CI || CI->getCalledFunction() != Fn)
6919 continue;
6920
6921 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6922 FunctionType *NewFuncTy = NewFn->getFunctionType();
6923 SmallVector<Value *, 2> Args;
6924
6925 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6926 // value to the return type of the old function.
6927 if (NewFuncTy->getReturnType() != CI->getType() &&
6928 !CastInst::castIsValid(op: Instruction::BitCast, S: CI,
6929 DstTy: NewFuncTy->getReturnType()))
6930 continue;
6931
6932 bool InvalidCast = false;
6933
6934 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6935 Value *Arg = CI->getArgOperand(i: I);
6936
6937 // Bitcast argument to the parameter type of the new function if it's
6938 // not a variadic argument.
6939 if (I < NewFuncTy->getNumParams()) {
6940 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6941 // to the parameter type of the new function.
6942 if (!CastInst::castIsValid(op: Instruction::BitCast, S: Arg,
6943 DstTy: NewFuncTy->getParamType(i: I))) {
6944 InvalidCast = true;
6945 break;
6946 }
6947 Arg = Builder.CreateBitCast(V: Arg, DestTy: NewFuncTy->getParamType(i: I));
6948 }
6949 Args.push_back(Elt: Arg);
6950 }
6951
6952 if (InvalidCast)
6953 continue;
6954
6955 // Create a call instruction that calls the new function.
6956 CallInst *NewCall = Builder.CreateCall(FTy: NewFuncTy, Callee: NewFn, Args);
6957 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6958 NewCall->takeName(V: CI);
6959
6960 // Bitcast the return value back to the type of the old call.
6961 Value *NewRetVal = Builder.CreateBitCast(V: NewCall, DestTy: CI->getType());
6962
6963 if (!CI->use_empty())
6964 CI->replaceAllUsesWith(V: NewRetVal);
6965 CI->eraseFromParent();
6966 }
6967
6968 if (Fn->use_empty())
6969 Fn->eraseFromParent();
6970 };
6971
6972 // Unconditionally convert a call to "clang.arc.use" to a call to
6973 // "llvm.objc.clang.arc.use".
6974 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6975
6976 // Upgrade the retain release marker. If there is no need to upgrade
6977 // the marker, that means either the module is already new enough to contain
6978 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6979 if (!upgradeRetainReleaseMarker(M))
6980 return;
6981
6982 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6983 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6984 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6985 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6986 {"objc_autoreleaseReturnValue",
6987 llvm::Intrinsic::objc_autoreleaseReturnValue},
6988 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6989 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6990 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6991 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6992 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6993 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6994 {"objc_release", llvm::Intrinsic::objc_release},
6995 {"objc_retain", llvm::Intrinsic::objc_retain},
6996 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6997 {"objc_retainAutoreleaseReturnValue",
6998 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6999 {"objc_retainAutoreleasedReturnValue",
7000 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7001 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7002 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7003 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7004 {"objc_unsafeClaimAutoreleasedReturnValue",
7005 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7006 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7007 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7008 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7009 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7010 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7011 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7012 {"objc_arc_annotation_topdown_bbstart",
7013 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7014 {"objc_arc_annotation_topdown_bbend",
7015 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7016 {"objc_arc_annotation_bottomup_bbstart",
7017 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7018 {"objc_arc_annotation_bottomup_bbend",
7019 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7020
7021 for (auto &I : RuntimeFuncs)
7022 UpgradeToIntrinsic(I.first, I.second);
7023}
7024
7025// Upgrade the way signing of pointers to init/fini functions is described.
7026//
7027// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7028// constants, if signing was requested. After the upgrade, these arrays contain
7029// plain function pointers and the desired signing schema is described via a
7030// pair of module flags.
7031//
7032// Note that the upgrade is only performed if all elements of *both* arrays
7033// agree on a common signing schema.
7034static bool upgradePtrauthInitFiniArrays(Module &M) {
7035 // As we cannot always decide whether the particular module should have
7036 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7037 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7038 // returns as soon as it spots any non-signed init/fini pointer: either we
7039 // should request non-signed pointers (safe to omit both flags) or there is
7040 // no common schema (and thus we do not modify anything).
7041 //
7042 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7043 // or whether we should request address diversity in addition to the basic
7044 // constant diversity. There is no value representing "decided not to sign"
7045 // for the reasons explained above.
7046 std::optional<bool> UseAddressDisc;
7047
7048 // Do not attempt upgrading if the new module flags already exist.
7049 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7050 for (const MDNode *Flag : ModFlags->operands()) {
7051 std::optional<StringRef> Name = getModuleFlagNameSafely(Flag: *Flag);
7052 if (Name && (*Name == "ptrauth-init-fini" ||
7053 *Name == "ptrauth-init-fini-address-discrimination"))
7054 return false;
7055 }
7056 }
7057
7058 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7059 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7060 constexpr unsigned ExpectedAddressMarker = 1;
7061
7062 auto *CPA = dyn_cast<ConstantPtrAuth>(Val: CV);
7063 if (!CPA || !CPA->getDiscriminator()->equalsInt(V: ExpectedConstDisc))
7064 return nullptr; // Nothing to upgrade or unknown pattern found.
7065
7066 bool HasAddressDisc;
7067 if (!CPA->hasAddressDiscriminator())
7068 HasAddressDisc = false;
7069 else if (CPA->hasSpecialAddressDiscriminator(Value: ExpectedAddressMarker))
7070 HasAddressDisc = true;
7071 else
7072 return nullptr; // Unknown pattern.
7073
7074 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7075 return nullptr; // Disagreement with the decided mode.
7076
7077 UseAddressDisc = HasAddressDisc;
7078 return CPA->getPointer();
7079 };
7080
7081 // Do not apply any changes until we know the upgrade is non-ambiguous.
7082 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7083 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7084
7085 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7086 auto *GV = dyn_cast_if_present<GlobalVariable>(Val: M.getNamedValue(Name));
7087 if (!GV || !GV->hasInitializer())
7088 continue; // Skip, but it is okay to upgrade the other variable.
7089
7090 auto *OldStructorsArray = dyn_cast<ConstantArray>(Val: GV->getInitializer());
7091 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7092 return false;
7093
7094 std::vector<Constant *> NewStructors;
7095 NewStructors.reserve(n: OldStructorsArray->getNumOperands());
7096
7097 for (Use &U : OldStructorsArray->operands()) {
7098 ConstantStruct *Structor = dyn_cast<ConstantStruct>(Val: U.get());
7099 if (!Structor || Structor->getNumOperands() != 3)
7100 return false;
7101
7102 Constant *Prio = Structor->getOperand(i_nocapture: 0);
7103 Constant *Func = Structor->getOperand(i_nocapture: 1);
7104 Constant *Arg = Structor->getOperand(i_nocapture: 2);
7105
7106 Func = UpgradeSinglePointer(Func);
7107 if (!Func)
7108 return false;
7109
7110 NewStructors.push_back(
7111 x: ConstantStruct::get(T: Structor->getType(), V: {Prio, Func, Arg}));
7112 }
7113
7114 Constant *NewInit =
7115 ConstantArray::get(T: OldStructorsArray->getType(), V: NewStructors);
7116 GlobalArraysToUpgrade.emplace_back(Args&: GV, Args&: NewInit);
7117 }
7118
7119 if (GlobalArraysToUpgrade.empty())
7120 return false;
7121 assert(UseAddressDisc.has_value());
7122
7123 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7124 GV->setInitializer(NewInit);
7125
7126 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini", Val: 1);
7127 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini-address-discrimination",
7128 Val: *UseAddressDisc);
7129
7130 return true;
7131}
7132
7133bool llvm::UpgradeModuleFlags(Module &M) {
7134 bool Changed = false;
7135 Changed |= upgradePtrauthInitFiniArrays(M);
7136
7137 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7138 if (!ModFlags)
7139 return Changed;
7140
7141 bool HasObjCFlag = false, HasClassProperties = false;
7142 bool HasSwiftVersionFlag = false;
7143 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7144 uint32_t SwiftABIVersion;
7145 auto Int8Ty = Type::getInt8Ty(C&: M.getContext());
7146 auto Int32Ty = Type::getInt32Ty(C&: M.getContext());
7147
7148 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7149 MDNode *Op = ModFlags->getOperand(i: I);
7150 if (Op->getNumOperands() != 3)
7151 continue;
7152 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
7153 if (!ID)
7154 continue;
7155 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7156 Metadata *Ops[3] = {ConstantAsMetadata::get(C: ConstantInt::get(
7157 Ty: Type::getInt32Ty(C&: M.getContext()), V: B)),
7158 MDString::get(Context&: M.getContext(), Str: ID->getString()),
7159 Op->getOperand(I: 2)};
7160 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7161 Changed = true;
7162 };
7163
7164 if (ID->getString() == "Objective-C Image Info Version")
7165 HasObjCFlag = true;
7166 if (ID->getString() == "Objective-C Class Properties")
7167 HasClassProperties = true;
7168 // Upgrade PIC from Error/Max to Min.
7169 if (ID->getString() == "PIC Level") {
7170 if (auto *Behavior =
7171 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
7172 uint64_t V = Behavior->getLimitedValue();
7173 if (V == Module::Error || V == Module::Max)
7174 SetBehavior(Module::Min);
7175 }
7176 }
7177 // Upgrade "PIE Level" from Error to Max.
7178 if (ID->getString() == "PIE Level")
7179 if (auto *Behavior =
7180 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0)))
7181 if (Behavior->getLimitedValue() == Module::Error)
7182 SetBehavior(Module::Max);
7183
7184 // Upgrade branch protection and return address signing module flags. The
7185 // module flag behavior for these fields were Error and now they are Min.
7186 if (ID->getString() == "branch-target-enforcement" ||
7187 ID->getString().starts_with(Prefix: "sign-return-address")) {
7188 if (auto *Behavior =
7189 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
7190 if (Behavior->getLimitedValue() == Module::Error) {
7191 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
7192 Metadata *Ops[3] = {
7193 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Module::Min)),
7194 Op->getOperand(I: 1), Op->getOperand(I: 2)};
7195 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7196 Changed = true;
7197 }
7198 }
7199 }
7200
7201 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7202 // section name so that llvm-lto will not complain about mismatching
7203 // module flags that is functionally the same.
7204 if (ID->getString() == "Objective-C Image Info Section") {
7205 if (auto *Value = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2))) {
7206 SmallVector<StringRef, 4> ValueComp;
7207 Value->getString().split(A&: ValueComp, Separator: " ");
7208 if (ValueComp.size() != 1) {
7209 std::string NewValue;
7210 for (auto &S : ValueComp)
7211 NewValue += S.str();
7212 Metadata *Ops[3] = {Op->getOperand(I: 0), Op->getOperand(I: 1),
7213 MDString::get(Context&: M.getContext(), Str: NewValue)};
7214 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7215 Changed = true;
7216 }
7217 }
7218 }
7219
7220 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7221 // If the higher bits are set, it adds new module flag for swift info.
7222 if (ID->getString() == "Objective-C Garbage Collection") {
7223 auto Md = dyn_cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2));
7224 if (Md) {
7225 assert(Md->getValue() && "Expected non-empty metadata");
7226 auto Type = Md->getValue()->getType();
7227 if (Type == Int8Ty)
7228 continue;
7229 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7230 if ((Val & 0xff) != Val) {
7231 HasSwiftVersionFlag = true;
7232 SwiftABIVersion = (Val & 0xff00) >> 8;
7233 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7234 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7235 }
7236 Metadata *Ops[3] = {
7237 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty,V: Module::Error)),
7238 Op->getOperand(I: 1),
7239 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int8Ty,V: Val & 0xff))};
7240 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7241 Changed = true;
7242 }
7243 }
7244
7245 if (ID->getString() == "amdgpu_code_object_version") {
7246 Metadata *Ops[3] = {
7247 Op->getOperand(I: 0),
7248 MDString::get(Context&: M.getContext(), Str: "amdhsa_code_object_version"),
7249 Op->getOperand(I: 2)};
7250 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7251 Changed = true;
7252 }
7253
7254 // clang/PowerPC used to use "float-abi" to describe the long double format;
7255 // it has been renamed to "long-double-type", with its values changed to the
7256 // corresponding IR floating-point type names.
7257 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7258 StringRef Format;
7259 if (auto *S = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2)))
7260 Format = S->getString();
7261
7262 // The "float-abi" key is now reserved for the target-independent
7263 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7264 // (including unrecognized ones, which were never valid) to the default.
7265 if (!FloatABI::parseABIType(S: Format)) {
7266 LongDoubleFormat NewFormat =
7267 StringSwitch<LongDoubleFormat>(Format)
7268 .Case(S: "ieeequad", Value: LongDoubleFormat::IEEEquad)
7269 .Case(S: "ieeedouble", Value: LongDoubleFormat::IEEEdouble)
7270 .Default(Value: LongDoubleFormat::PPCDoubleDouble);
7271 Metadata *Ops[3] = {
7272 Op->getOperand(I: 0),
7273 MDString::get(Context&: M.getContext(), Str: "long-double-type"),
7274 MDString::get(Context&: M.getContext(), Str: getLongDoubleFormatName(Format: NewFormat))};
7275 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7276 Changed = true;
7277 }
7278 }
7279 }
7280
7281 // "Objective-C Class Properties" is recently added for Objective-C. We
7282 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7283 // flag of value 0, so we can correclty downgrade this flag when trying to
7284 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7285 // this module flag.
7286 if (HasObjCFlag && !HasClassProperties) {
7287 M.addModuleFlag(Behavior: llvm::Module::Override, Key: "Objective-C Class Properties",
7288 Val: (uint32_t)0);
7289 Changed = true;
7290 }
7291
7292 if (HasSwiftVersionFlag) {
7293 M.addModuleFlag(Behavior: Module::Error, Key: "Swift ABI Version",
7294 Val: SwiftABIVersion);
7295 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Major Version",
7296 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMajorVersion));
7297 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Minor Version",
7298 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMinorVersion));
7299 Changed = true;
7300 }
7301
7302 return Changed;
7303}
7304
7305bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7306 NamedMDNode *CFIConsts = M.getNamedMetadata(Name: "cfi.functions");
7307 // If this metadata has operands, we expect all of them to be either from
7308 // before or from after the format change handled here, so we can bail out
7309 // fast if the first (if any) operands is of the new format.
7310 auto MatchesVersion = [](const MDNode *Op) {
7311 return Op->getNumOperands() >= 3 &&
7312 isa<ConstantAsMetadata>(Val: Op->getOperand(I: 2)) &&
7313 cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2))
7314 ->getType()
7315 ->isIntegerTy(BitWidth: 64);
7316 };
7317
7318 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7319 MatchesVersion(CFIConsts->getOperand(i: 0)))
7320 return false;
7321
7322 bool Changed = false;
7323 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7324 MDNode *Op = CFIConsts->getOperand(i: I);
7325 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7326 assert(Op->getNumOperands() >= 2 &&
7327 "Expected at least 2 operands - name and linkage type");
7328 MDString *NameMD = dyn_cast<MDString>(Val: Op->getOperand(I: 0));
7329 StringRef Name = NameMD->getString();
7330 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7331 GlobalName: GlobalValue::dropLLVMManglingEscape(Name));
7332
7333 SmallVector<Metadata *, 4> Elts;
7334 Elts.push_back(Elt: Op->getOperand(I: 0));
7335 Elts.push_back(Elt: Op->getOperand(I: 1));
7336 Elts.push_back(Elt: ConstantAsMetadata::get(
7337 C: ConstantInt::get(Ty: Type::getInt64Ty(C&: M.getContext()), V: GUID)));
7338
7339 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7340 Elts.push_back(Elt: Op->getOperand(I: J));
7341
7342 CFIConsts->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Elts));
7343 Changed = true;
7344 }
7345
7346 return Changed;
7347}
7348
7349void llvm::UpgradeSectionAttributes(Module &M) {
7350 auto TrimSpaces = [](StringRef Section) -> std::string {
7351 SmallVector<StringRef, 5> Components;
7352 Section.split(A&: Components, Separator: ',');
7353
7354 SmallString<32> Buffer;
7355 raw_svector_ostream OS(Buffer);
7356
7357 for (auto Component : Components)
7358 OS << ',' << Component.trim();
7359
7360 return std::string(OS.str().substr(Start: 1));
7361 };
7362
7363 for (auto &GV : M.globals()) {
7364 if (!GV.hasSection())
7365 continue;
7366
7367 StringRef Section = GV.getSection();
7368
7369 if (!Section.starts_with(Prefix: "__DATA, __objc_catlist"))
7370 continue;
7371
7372 // __DATA, __objc_catlist, regular, no_dead_strip
7373 // __DATA,__objc_catlist,regular,no_dead_strip
7374 GV.setSection(TrimSpaces(Section));
7375 }
7376}
7377
7378namespace {
7379// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7380// callsites within a function that did not also have the strictfp attribute.
7381// Since 10.0, if strict FP semantics are needed within a function, the
7382// function must have the strictfp attribute and all calls within the function
7383// must also have the strictfp attribute. This latter restriction is
7384// necessary to prevent unwanted libcall simplification when a function is
7385// being cloned (such as for inlining).
7386//
7387// The "dangling" strictfp attribute usage was only used to prevent constant
7388// folding and other libcall simplification. The nobuiltin attribute on the
7389// callsite has the same effect.
7390struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7391 StrictFPUpgradeVisitor() = default;
7392
7393 void visitCallBase(CallBase &Call) {
7394 if (!Call.isStrictFP())
7395 return;
7396 if (isa<ConstrainedFPIntrinsic>(Val: &Call))
7397 return;
7398 // If we get here, the caller doesn't have the strictfp attribute
7399 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7400 Call.removeFnAttr(Kind: Attribute::StrictFP);
7401 Call.addFnAttr(Kind: Attribute::NoBuiltin);
7402 }
7403};
7404
7405/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7406struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7407 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7408 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7409
7410 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7411 if (!RMW.isFloatingPointOperation())
7412 return;
7413
7414 MDNode *Empty = MDNode::get(Context&: RMW.getContext(), MDs: {});
7415 RMW.setMetadata(Kind: "amdgpu.no.fine.grained.host.memory", Node: Empty);
7416 RMW.setMetadata(Kind: "amdgpu.no.remote.memory.access", Node: Empty);
7417 RMW.setMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode, Node: Empty);
7418 }
7419};
7420} // namespace
7421
7422void llvm::UpgradeFunctionAttributes(Function &F) {
7423 // If a function definition doesn't have the strictfp attribute,
7424 // convert any callsite strictfp attributes to nobuiltin.
7425 if (!F.isDeclaration() && !F.hasFnAttribute(Kind: Attribute::StrictFP)) {
7426 StrictFPUpgradeVisitor SFPV;
7427 SFPV.visit(F);
7428 }
7429
7430 // Remove all incompatibile attributes from function.
7431 F.removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
7432 Ty: F.getReturnType(), AS: F.getAttributes().getRetAttrs()));
7433 for (auto &Arg : F.args())
7434 Arg.removeAttrs(
7435 AM: AttributeFuncs::typeIncompatible(Ty: Arg.getType(), AS: Arg.getAttributes()));
7436
7437 bool AddingAttrs = false, RemovingAttrs = false;
7438 AttrBuilder AttrsToAdd(F.getContext());
7439 AttributeMask AttrsToRemove;
7440
7441 // Older versions of LLVM treated an "implicit-section-name" attribute
7442 // similarly to directly setting the section on a Function.
7443 if (Attribute A = F.getFnAttribute(Kind: "implicit-section-name");
7444 A.isValid() && A.isStringAttribute()) {
7445 F.setSection(A.getValueAsString());
7446 AttrsToRemove.addAttribute(A: "implicit-section-name");
7447 RemovingAttrs = true;
7448 }
7449
7450 if (Attribute A = F.getFnAttribute(Kind: "nooutline");
7451 A.isValid() && A.isStringAttribute()) {
7452 AttrsToRemove.addAttribute(A: "nooutline");
7453 AttrsToAdd.addAttribute(Val: Attribute::NoOutline);
7454 AddingAttrs = RemovingAttrs = true;
7455 }
7456
7457 if (Attribute A = F.getFnAttribute(Kind: "uniform-work-group-size");
7458 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7459 AttrsToRemove.addAttribute(A: "uniform-work-group-size");
7460 RemovingAttrs = true;
7461 if (A.getValueAsString() == "true") {
7462 AttrsToAdd.addAttribute(A: "uniform-work-group-size");
7463 AddingAttrs = true;
7464 }
7465 }
7466
7467 if (!F.empty()) {
7468 // For some reason this is called twice, and the first time is before any
7469 // instructions are loaded into the body.
7470
7471 if (Attribute A = F.getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics");
7472 A.isValid()) {
7473
7474 if (A.getValueAsBool()) {
7475 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7476 Visitor.visit(F);
7477 }
7478
7479 // We will leave behind dead attribute uses on external declarations, but
7480 // clang never added these to declarations anyway.
7481 AttrsToRemove.addAttribute(A: "amdgpu-unsafe-fp-atomics");
7482 RemovingAttrs = true;
7483 }
7484 }
7485
7486 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7487 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7488
7489 bool HandleDenormalMode = false;
7490
7491 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math"); Attr.isValid()) {
7492 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7493 if (ParsedMode.isValid()) {
7494 DenormalFPMath = ParsedMode;
7495 AttrsToRemove.addAttribute(A: "denormal-fp-math");
7496 AddingAttrs = RemovingAttrs = true;
7497 HandleDenormalMode = true;
7498 }
7499 }
7500
7501 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math-f32");
7502 Attr.isValid()) {
7503 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7504 if (ParsedMode.isValid()) {
7505 DenormalFPMathF32 = ParsedMode;
7506 AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
7507 AddingAttrs = RemovingAttrs = true;
7508 HandleDenormalMode = true;
7509 }
7510 }
7511
7512 if (HandleDenormalMode)
7513 AttrsToAdd.addDenormalFPEnvAttr(
7514 Mode: DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7515
7516 if (RemovingAttrs)
7517 F.removeFnAttrs(Attrs: AttrsToRemove);
7518
7519 if (AddingAttrs)
7520 F.addFnAttrs(Attrs: AttrsToAdd);
7521}
7522
7523// Check if the function attribute is not present and set it.
7524static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7525 StringRef Value) {
7526 if (!F.hasFnAttribute(Kind: FnAttrName))
7527 F.addFnAttr(Kind: FnAttrName, Val: Value);
7528}
7529
7530// Check if the function attribute is not present and set it if needed.
7531// If the attribute is "false" then removes it.
7532// If the attribute is "true" resets it to a valueless attribute.
7533static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7534 if (!F.hasFnAttribute(Kind: FnAttrName)) {
7535 if (Set)
7536 F.addFnAttr(Kind: FnAttrName);
7537 } else {
7538 auto A = F.getFnAttribute(Kind: FnAttrName);
7539 if ("false" == A.getValueAsString())
7540 F.removeFnAttr(Kind: FnAttrName);
7541 else if ("true" == A.getValueAsString()) {
7542 F.removeFnAttr(Kind: FnAttrName);
7543 F.addFnAttr(Kind: FnAttrName);
7544 }
7545 }
7546}
7547
7548void llvm::copyModuleAttrToFunctions(Module &M) {
7549 Triple T(M.getTargetTriple());
7550 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7551 return;
7552
7553 uint64_t BTEValue = 0;
7554 uint64_t BPPLRValue = 0;
7555 uint64_t GCSValue = 0;
7556 uint64_t SRAValue = 0;
7557 uint64_t SRAALLValue = 0;
7558 uint64_t SRABKeyValue = 0;
7559
7560 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7561 if (ModFlags) {
7562 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7563 MDNode *Op = ModFlags->getOperand(i: I);
7564 if (Op->getNumOperands() != 3)
7565 continue;
7566
7567 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
7568 auto *CI = mdconst::dyn_extract<ConstantInt>(MD: Op->getOperand(I: 2));
7569 if (!ID || !CI)
7570 continue;
7571
7572 StringRef IDStr = ID->getString();
7573 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7574 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7575 : IDStr == "guarded-control-stack" ? &GCSValue
7576 : IDStr == "sign-return-address" ? &SRAValue
7577 : IDStr == "sign-return-address-all" ? &SRAALLValue
7578 : IDStr == "sign-return-address-with-bkey"
7579 ? &SRABKeyValue
7580 : nullptr;
7581 if (!ValPtr)
7582 continue;
7583
7584 *ValPtr = CI->getZExtValue();
7585 if (*ValPtr == 2)
7586 return;
7587 }
7588 }
7589
7590 bool BTE = BTEValue == 1;
7591 bool BPPLR = BPPLRValue == 1;
7592 bool GCS = GCSValue == 1;
7593 bool SRA = SRAValue == 1;
7594
7595 StringRef SignTypeValue = "non-leaf";
7596 if (SRA && SRAALLValue == 1)
7597 SignTypeValue = "all";
7598
7599 StringRef SignKeyValue = "a_key";
7600 if (SRA && SRABKeyValue == 1)
7601 SignKeyValue = "b_key";
7602
7603 for (Function &F : M.getFunctionList()) {
7604 if (F.isDeclaration())
7605 continue;
7606
7607 if (SRA) {
7608 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address", Value: SignTypeValue);
7609 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address-key", Value: SignKeyValue);
7610 } else {
7611 if (auto A = F.getFnAttribute(Kind: "sign-return-address");
7612 A.isValid() && "none" == A.getValueAsString()) {
7613 F.removeFnAttr(Kind: "sign-return-address");
7614 F.removeFnAttr(Kind: "sign-return-address-key");
7615 }
7616 }
7617 ConvertFunctionAttr(F, Set: BTE, FnAttrName: "branch-target-enforcement");
7618 ConvertFunctionAttr(F, Set: BPPLR, FnAttrName: "branch-protection-pauth-lr");
7619 ConvertFunctionAttr(F, Set: GCS, FnAttrName: "guarded-control-stack");
7620 }
7621
7622 if (BTE)
7623 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "branch-target-enforcement", Val: 2);
7624 if (BPPLR)
7625 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "branch-protection-pauth-lr", Val: 2);
7626 if (GCS)
7627 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "guarded-control-stack", Val: 2);
7628 if (SRA) {
7629 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address", Val: 2);
7630 if (SRAALLValue == 1)
7631 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-all", Val: 2);
7632 if (SRABKeyValue == 1)
7633 M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-with-bkey", Val: 2);
7634 }
7635}
7636
7637/// Return the replacement tags if \p T still uses a removed two-operand form.
7638static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7639 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
7640 return nullptr;
7641 auto *Tag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7642 return Tag ? findBooleanLoopTags(Name: Tag->getString()) : nullptr;
7643}
7644
7645/// Build the single-operand node that replaces a boolean operand: nonzero
7646/// selects the enable tag, zero the disable tag.
7647static Metadata *makeBooleanLoopNode(LLVMContext &C,
7648 const BooleanLoopTags &Tags,
7649 const MDOperand &Op) {
7650 bool Enable = !mdconst::extract<ConstantInt>(MD: Op)->isZero();
7651 return MDTuple::get(Context&: C,
7652 MDs: {MDString::get(Context&: C, Str: Enable ? Tags.Enable : Tags.Disable)});
7653}
7654
7655static bool isOldLoopArgument(Metadata *MD) {
7656 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
7657 if (!T)
7658 return false;
7659 if (T->getNumOperands() < 1)
7660 return false;
7661 auto *S = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7662 if (!S)
7663 return false;
7664 if (S->getString().starts_with(Prefix: "llvm.vectorizer."))
7665 return true;
7666 return getOldBooleanLoopTags(T) != nullptr;
7667}
7668
7669static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7670 StringRef OldPrefix = "llvm.vectorizer.";
7671 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7672
7673 if (OldTag == "llvm.vectorizer.unroll")
7674 return MDString::get(Context&: C, Str: "llvm.loop.interleave.count");
7675
7676 return MDString::get(
7677 Context&: C, Str: (Twine("llvm.loop.vectorize.") + OldTag.drop_front(N: OldPrefix.size()))
7678 .str());
7679}
7680
7681static Metadata *upgradeLoopArgument(Metadata *MD) {
7682 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
7683 if (!T)
7684 return MD;
7685 if (T->getNumOperands() < 1)
7686 return MD;
7687 auto *OldTag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7688 if (!OldTag)
7689 return MD;
7690
7691 LLVMContext &C = T->getContext();
7692
7693 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7694 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7695 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
7696
7697 if (!OldTag->getString().starts_with(Prefix: "llvm.vectorizer."))
7698 return MD;
7699
7700 // This has an old tag. Upgrade it.
7701 MDString *NewTag = upgradeLoopTag(C, OldTag: OldTag->getString());
7702
7703 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7704 // vectorize.enable/disable pair, not a two-operand enable node.
7705 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
7706 if (const BooleanLoopTags *Tags = findBooleanLoopTags(Name: NewTag->getString()))
7707 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
7708
7709 SmallVector<Metadata *, 8> Ops;
7710 Ops.reserve(N: T->getNumOperands());
7711 Ops.push_back(Elt: NewTag);
7712 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7713 Ops.push_back(Elt: T->getOperand(I));
7714
7715 return MDTuple::get(Context&: C, MDs: Ops);
7716}
7717
7718MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7719 auto *T = dyn_cast<MDTuple>(Val: &N);
7720 if (!T)
7721 return &N;
7722
7723 if (none_of(Range: T->operands(), P: isOldLoopArgument))
7724 return &N;
7725
7726 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7727 // any MDNode carrying those tags with more than one operand, so a leftover
7728 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7729 // In-place mutation is safe on distinct MDNodes.
7730 if (T->isDistinct()) {
7731 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7732 auto *OpT = dyn_cast_or_null<MDTuple>(Val: T->getOperand(I));
7733 if (OpT && getOldBooleanLoopTags(T: OpT))
7734 T->replaceOperandWith(I, New: upgradeLoopArgument(MD: OpT));
7735 }
7736 if (none_of(Range: T->operands(), P: isOldLoopArgument))
7737 return &N;
7738 }
7739
7740 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7741 // attachment; the original distinct loop-ID is kept as the first operand.
7742 SmallVector<Metadata *, 8> Ops;
7743 Ops.reserve(N: T->getNumOperands());
7744 for (Metadata *MD : T->operands())
7745 Ops.push_back(Elt: upgradeLoopArgument(MD));
7746
7747 return MDTuple::get(Context&: T->getContext(), MDs: Ops);
7748}
7749
7750std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7751 Triple T(TT);
7752 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7753 // the address space of globals to 1. This does not apply to SPIRV Logical.
7754 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7755 !DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G")) {
7756 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7757 }
7758
7759 if (T.isLoongArch64() || T.isRISCV64()) {
7760 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7761 auto I = DL.find(Str: "-n64-");
7762 if (I != StringRef::npos)
7763 return (DL.take_front(N: I) + "-n32:64-" + DL.drop_front(N: I + 5)).str();
7764 return DL.str();
7765 }
7766
7767 // AMDGPU data layout upgrades.
7768 std::string Res = DL.str();
7769 if (T.isAMDGPU()) {
7770 // Define address spaces for constants.
7771 if (!DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G"))
7772 Res.append(s: Res.empty() ? "G1" : "-G1");
7773
7774 // AMDGCN data layout upgrades.
7775 if (T.isAMDGCN()) {
7776
7777 // Add missing non-integral declarations.
7778 // This goes before adding new address spaces to prevent incoherent string
7779 // values.
7780 if (!DL.contains(Other: "-ni") && !DL.starts_with(Prefix: "ni"))
7781 Res.append(s: "-ni:7:8:9");
7782 // Update ni:7 to ni:7:8:9.
7783 if (DL.ends_with(Suffix: "ni:7"))
7784 Res.append(s: ":8:9");
7785 if (DL.ends_with(Suffix: "ni:7:8"))
7786 Res.append(s: ":9");
7787
7788 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7789 // resources) An empty data layout has already been upgraded to G1 by now.
7790 if (!DL.contains(Other: "-p7") && !DL.starts_with(Prefix: "p7"))
7791 Res.append(s: "-p7:160:256:256:32");
7792 if (!DL.contains(Other: "-p8") && !DL.starts_with(Prefix: "p8"))
7793 Res.append(s: "-p8:128:128:128:48");
7794 constexpr StringRef OldP8("-p8:128:128-");
7795 if (DL.contains(Other: OldP8))
7796 Res.replace(pos: Res.find(svt: OldP8), n1: OldP8.size(), s: "-p8:128:128:128:48-");
7797 if (!DL.contains(Other: "-p9") && !DL.starts_with(Prefix: "p9"))
7798 Res.append(s: "-p9:192:256:256:32");
7799
7800 // Add sizing for address space 10 through 15.
7801 // AS 10-14 are reserved and defaulted to 32:32
7802 // AS 15 is in use and is 32:32.
7803 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7804 if (!DL.contains(Other: ("-" + AS).str()) && !DL.starts_with(Prefix: AS))
7805 Res.append(str: ("-" + AS + ":32:32").str());
7806 }
7807 }
7808
7809 // Upgrade the ELF mangling mode.
7810 if (!DL.contains(Other: "m:e"))
7811 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7812
7813 return Res;
7814 }
7815
7816 if (T.isSystemZ() && !DL.empty()) {
7817 // Make sure the stack alignment is present.
7818 if (!DL.contains(Other: "-S64"))
7819 return "E-S64" + DL.drop_front(N: 1).str();
7820 return DL.str();
7821 }
7822
7823 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7824 // If the datalayout matches the expected format, add pointer size address
7825 // spaces to the datalayout.
7826 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7827 if (!DL.contains(Other: AddrSpaces)) {
7828 SmallVector<StringRef, 4> Groups;
7829 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7830 if (R.match(String: Res, Matches: &Groups))
7831 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7832 }
7833 };
7834
7835 // AArch64 data layout upgrades.
7836 if (T.isAArch64()) {
7837 // Add "-Fn32"
7838 if (!DL.empty() && !DL.contains(Other: "-Fn32"))
7839 Res.append(s: "-Fn32");
7840 AddPtr32Ptr64AddrSpaces();
7841 return Res;
7842 }
7843
7844 if (T.isSPARC() || (T.isMIPS64() && !DL.contains(Other: "m:m")) || T.isPPC64() ||
7845 T.isWasm()) {
7846 // Mips64 with o32 ABI did not add "-i128:128".
7847 // Add "-i128:128"
7848 std::string I64 = "-i64:64";
7849 std::string I128 = "-i128:128";
7850 if (!StringRef(Res).contains(Other: I128)) {
7851 size_t Pos = Res.find(str: I64);
7852 if (Pos != size_t(-1))
7853 Res.insert(pos1: Pos + I64.size(), str: I128);
7854 }
7855 }
7856
7857 if (T.isPPC() && T.isOSAIX() && !DL.contains(Other: "f64:32:64") && !DL.empty()) {
7858 size_t Pos = Res.find(s: "-S128");
7859 if (Pos == StringRef::npos)
7860 Pos = Res.size();
7861 Res.insert(pos: Pos, s: "-f64:32:64");
7862 }
7863
7864 if (!T.isX86())
7865 return Res;
7866
7867 AddPtr32Ptr64AddrSpaces();
7868
7869 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7870 // for i128 operations prior to this being reflected in the data layout, and
7871 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7872 // boundaries, so although this is a breaking change, the upgrade is expected
7873 // to fix more IR than it breaks.
7874 // Intel MCU is an exception and uses 4-byte-alignment.
7875 if (!T.isOSIAMCU()) {
7876 std::string I128 = "-i128:128";
7877 if (StringRef Ref = Res; !Ref.contains(Other: I128)) {
7878 SmallVector<StringRef, 4> Groups;
7879 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7880 if (R.match(String: Res, Matches: &Groups))
7881 Res = (Groups[1] + I128 + Groups[3]).str();
7882 }
7883 }
7884
7885 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7886 // Raising the alignment is safe because Clang did not produce f80 values in
7887 // the MSVC environment before this upgrade was added.
7888 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7889 StringRef Ref = Res;
7890 auto I = Ref.find(Str: "-f80:32-");
7891 if (I != StringRef::npos)
7892 Res = (Ref.take_front(N: I) + "-f80:128-" + Ref.drop_front(N: I + 8)).str();
7893 }
7894
7895 return Res;
7896}
7897
7898void llvm::UpgradeAttributes(AttrBuilder &B) {
7899 StringRef FramePointer;
7900 Attribute A = B.getAttribute(Kind: "no-frame-pointer-elim");
7901 if (A.isValid()) {
7902 // The value can be "true" or "false".
7903 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7904 B.removeAttribute(A: "no-frame-pointer-elim");
7905 }
7906 if (B.contains(A: "no-frame-pointer-elim-non-leaf")) {
7907 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7908 if (FramePointer != "all")
7909 FramePointer = "non-leaf";
7910 B.removeAttribute(A: "no-frame-pointer-elim-non-leaf");
7911 }
7912 if (!FramePointer.empty())
7913 B.addAttribute(A: "frame-pointer", V: FramePointer);
7914
7915 A = B.getAttribute(Kind: "null-pointer-is-valid");
7916 if (A.isValid()) {
7917 // The value can be "true" or "false".
7918 bool NullPointerIsValid = A.getValueAsString() == "true";
7919 B.removeAttribute(A: "null-pointer-is-valid");
7920 if (NullPointerIsValid)
7921 B.addAttribute(Val: Attribute::NullPointerIsValid);
7922 }
7923
7924 A = B.getAttribute(Kind: "uniform-work-group-size");
7925 if (A.isValid()) {
7926 StringRef Val = A.getValueAsString();
7927 if (!Val.empty()) {
7928 bool IsTrue = Val == "true";
7929 B.removeAttribute(A: "uniform-work-group-size");
7930 if (IsTrue)
7931 B.addAttribute(A: "uniform-work-group-size");
7932 }
7933 }
7934}
7935
7936void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7937 // clang.arc.attachedcall bundles are now required to have an operand.
7938 // If they don't, it's okay to drop them entirely: when there is an operand,
7939 // the "attachedcall" is meaningful and required, but without an operand,
7940 // it's just a marker NOP. Dropping it merely prevents an optimization.
7941 erase_if(C&: Bundles, P: [&](OperandBundleDef &OBD) {
7942 return OBD.getTag() == "clang.arc.attachedcall" &&
7943 OBD.inputs().empty();
7944 });
7945}
7946