1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "IROptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSwitch.h"
21#include "llvm/BinaryFormat/Dwarf.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/Attributes.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfo.h"
27#include "llvm/IR/DebugInfoMetadata.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstIterator.h"
33#include "llvm/IR/InstVisitor.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsAArch64.h"
38#include "llvm/IR/IntrinsicsAMDGPU.h"
39#include "llvm/IR/IntrinsicsARM.h"
40#include "llvm/IR/IntrinsicsNVPTX.h"
41#include "llvm/IR/IntrinsicsRISCV.h"
42#include "llvm/IR/IntrinsicsWebAssembly.h"
43#include "llvm/IR/IntrinsicsX86.h"
44#include "llvm/IR/LLVMContext.h"
45#include "llvm/IR/MDBuilder.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/NVVMIntrinsicUtils.h"
49#include "llvm/IR/Value.h"
50#include "llvm/IR/Verifier.h"
51#include "llvm/Support/AMDGPUAddrSpace.h"
52#include "llvm/Support/CodeGen.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/NVPTXAddrSpace.h"
56#include "llvm/Support/NVVMAttributes.h"
57#include "llvm/Support/Regex.h"
58#include "llvm/Support/TimeProfiler.h"
59#include "llvm/TargetParser/Triple.h"
60#include <cstdint>
61#include <cstring>
62#include <numeric>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "auto-upgrade"
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(O&: llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(i: 0);
86 if (Arg0Type != FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(GV: F);
91 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 i: F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(BitWidth: 32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(GV: F);
107 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(GV: F);
120 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 8))
135 return false;
136
137 rename(GV: F);
138 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 16))
153 return false;
154
155 rename(GV: F);
156 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(GV: F);
166 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(i: 1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(GV: F);
176 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front(Prefix: "avx."))
187 return (Name.starts_with(Prefix: "blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with(Prefix: "movnt.") || // Added in 3.2
192 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
193 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
194 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 3.5
195 Name.starts_with(Prefix: "vbroadcastf128") || // Added in 4.0
196 Name.starts_with(Prefix: "vextractf128.") || // Added in 3.7
197 Name.starts_with(Prefix: "vinsertf128.") || // Added in 3.7
198 Name.starts_with(Prefix: "vperm2f128.") || // Added in 6.0
199 Name.starts_with(Prefix: "vpermil.")); // Added in 3.1
200
201 if (Name.consume_front(Prefix: "avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
204 Name.starts_with(Prefix: "padds.") || // Added in 8.0
205 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
206 Name.starts_with(Prefix: "pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.8
209 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
210 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
211 Name.starts_with(Prefix: "pmax") || // Added in 3.9
212 Name.starts_with(Prefix: "pmin") || // Added in 3.9
213 Name.starts_with(Prefix: "pmovsx") || // Added in 3.9
214 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
215 Name.starts_with(Prefix: "pmulh.w") || // Added in 24.0
216 Name.starts_with(Prefix: "pmulhu.w") || // Added in 24.0
217 Name == "pmul.dq" || // Added in 7.0
218 Name == "pmulu.dq" || // Added in 7.0
219 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
220 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
221 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
222 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
223 Name.starts_with(Prefix: "vbroadcast") || // Added in 3.8
224 Name == "vbroadcasti128" || // Added in 3.7
225 Name == "vextracti128" || // Added in 3.7
226 Name == "vinserti128" || // Added in 3.7
227 Name == "vperm2i128"); // Added in 6.0
228
229 if (Name.consume_front(Prefix: "avx512.")) {
230 if (Name.consume_front(Prefix: "mask."))
231 // 'avx512.mask.*'
232 return (Name.starts_with(Prefix: "add.p") || // Added in 7.0. 128/256 in 4.0
233 Name.starts_with(Prefix: "and.") || // Added in 3.9
234 Name.starts_with(Prefix: "andn.") || // Added in 3.9
235 Name.starts_with(Prefix: "broadcast.s") || // Added in 3.9
236 Name.starts_with(Prefix: "broadcastf32x4.") || // Added in 6.0
237 Name.starts_with(Prefix: "broadcastf32x8.") || // Added in 6.0
238 Name.starts_with(Prefix: "broadcastf64x2.") || // Added in 6.0
239 Name.starts_with(Prefix: "broadcastf64x4.") || // Added in 6.0
240 Name.starts_with(Prefix: "broadcasti32x4.") || // Added in 6.0
241 Name.starts_with(Prefix: "broadcasti32x8.") || // Added in 6.0
242 Name.starts_with(Prefix: "broadcasti64x2.") || // Added in 6.0
243 Name.starts_with(Prefix: "broadcasti64x4.") || // Added in 6.0
244 Name.starts_with(Prefix: "cmp.b") || // Added in 5.0
245 Name.starts_with(Prefix: "cmp.d") || // Added in 5.0
246 Name.starts_with(Prefix: "cmp.q") || // Added in 5.0
247 Name.starts_with(Prefix: "cmp.w") || // Added in 5.0
248 Name.starts_with(Prefix: "compress.b") || // Added in 9.0
249 Name.starts_with(Prefix: "compress.d") || // Added in 9.0
250 Name.starts_with(Prefix: "compress.p") || // Added in 9.0
251 Name.starts_with(Prefix: "compress.q") || // Added in 9.0
252 Name.starts_with(Prefix: "compress.store.") || // Added in 7.0
253 Name.starts_with(Prefix: "compress.w") || // Added in 9.0
254 Name.starts_with(Prefix: "conflict.") || // Added in 9.0
255 Name.starts_with(Prefix: "cvtdq2pd.") || // Added in 4.0
256 Name.starts_with(Prefix: "cvtdq2ps.") || // Added in 7.0 updated 9.0
257 Name == "cvtpd2dq.256" || // Added in 7.0
258 Name == "cvtpd2ps.256" || // Added in 7.0
259 Name == "cvtps2pd.128" || // Added in 7.0
260 Name == "cvtps2pd.256" || // Added in 7.0
261 Name.starts_with(Prefix: "cvtqq2pd.") || // Added in 7.0 updated 9.0
262 Name == "cvtqq2ps.256" || // Added in 9.0
263 Name == "cvtqq2ps.512" || // Added in 9.0
264 Name == "cvttpd2dq.256" || // Added in 7.0
265 Name == "cvttps2dq.128" || // Added in 7.0
266 Name == "cvttps2dq.256" || // Added in 7.0
267 Name.starts_with(Prefix: "cvtudq2pd.") || // Added in 4.0
268 Name.starts_with(Prefix: "cvtudq2ps.") || // Added in 7.0 updated 9.0
269 Name.starts_with(Prefix: "cvtuqq2pd.") || // Added in 7.0 updated 9.0
270 Name == "cvtuqq2ps.256" || // Added in 9.0
271 Name == "cvtuqq2ps.512" || // Added in 9.0
272 Name.starts_with(Prefix: "dbpsadbw.") || // Added in 7.0
273 Name.starts_with(Prefix: "div.p") || // Added in 7.0. 128/256 in 4.0
274 Name.starts_with(Prefix: "expand.b") || // Added in 9.0
275 Name.starts_with(Prefix: "expand.d") || // Added in 9.0
276 Name.starts_with(Prefix: "expand.load.") || // Added in 7.0
277 Name.starts_with(Prefix: "expand.p") || // Added in 9.0
278 Name.starts_with(Prefix: "expand.q") || // Added in 9.0
279 Name.starts_with(Prefix: "expand.w") || // Added in 9.0
280 Name.starts_with(Prefix: "fpclass.p") || // Added in 7.0
281 Name.starts_with(Prefix: "insert") || // Added in 4.0
282 Name.starts_with(Prefix: "load.") || // Added in 3.9
283 Name.starts_with(Prefix: "loadu.") || // Added in 3.9
284 Name.starts_with(Prefix: "lzcnt.") || // Added in 5.0
285 Name.starts_with(Prefix: "max.p") || // Added in 7.0. 128/256 in 5.0
286 Name.starts_with(Prefix: "min.p") || // Added in 7.0. 128/256 in 5.0
287 Name.starts_with(Prefix: "movddup") || // Added in 3.9
288 Name.starts_with(Prefix: "move.s") || // Added in 4.0
289 Name.starts_with(Prefix: "movshdup") || // Added in 3.9
290 Name.starts_with(Prefix: "movsldup") || // Added in 3.9
291 Name.starts_with(Prefix: "mul.p") || // Added in 7.0. 128/256 in 4.0
292 Name.starts_with(Prefix: "or.") || // Added in 3.9
293 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
294 Name.starts_with(Prefix: "packssdw.") || // Added in 5.0
295 Name.starts_with(Prefix: "packsswb.") || // Added in 5.0
296 Name.starts_with(Prefix: "packusdw.") || // Added in 5.0
297 Name.starts_with(Prefix: "packuswb.") || // Added in 5.0
298 Name.starts_with(Prefix: "padd.") || // Added in 4.0
299 Name.starts_with(Prefix: "padds.") || // Added in 8.0
300 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
301 Name.starts_with(Prefix: "palignr.") || // Added in 3.9
302 Name.starts_with(Prefix: "pand.") || // Added in 3.9
303 Name.starts_with(Prefix: "pandn.") || // Added in 3.9
304 Name.starts_with(Prefix: "pavg") || // Added in 6.0
305 Name.starts_with(Prefix: "pbroadcast") || // Added in 6.0
306 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.9
307 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.9
308 Name.starts_with(Prefix: "perm.df.") || // Added in 3.9
309 Name.starts_with(Prefix: "perm.di.") || // Added in 3.9
310 Name.starts_with(Prefix: "permvar.") || // Added in 7.0
311 Name.starts_with(Prefix: "pmaddubs.w.") || // Added in 7.0
312 Name.starts_with(Prefix: "pmaddw.d.") || // Added in 7.0
313 Name.starts_with(Prefix: "pmax") || // Added in 4.0
314 Name.starts_with(Prefix: "pmin") || // Added in 4.0
315 Name == "pmov.qd.256" || // Added in 9.0
316 Name == "pmov.qd.512" || // Added in 9.0
317 Name == "pmov.wb.256" || // Added in 9.0
318 Name == "pmov.wb.512" || // Added in 9.0
319 Name.starts_with(Prefix: "pmovsx") || // Added in 4.0
320 Name.starts_with(Prefix: "pmovzx") || // Added in 4.0
321 Name.starts_with(Prefix: "pmul.dq.") || // Added in 4.0
322 Name.starts_with(Prefix: "pmul.hr.sw.") || // Added in 7.0
323 Name.starts_with(Prefix: "pmulh.w.") || // Added in 7.0
324 Name.starts_with(Prefix: "pmulhu.w.") || // Added in 7.0
325 Name.starts_with(Prefix: "pmull.") || // Added in 4.0
326 Name.starts_with(Prefix: "pmultishift.qb.") || // Added in 8.0
327 Name.starts_with(Prefix: "pmulu.dq.") || // Added in 4.0
328 Name.starts_with(Prefix: "por.") || // Added in 3.9
329 Name.starts_with(Prefix: "prol.") || // Added in 8.0
330 Name.starts_with(Prefix: "prolv.") || // Added in 8.0
331 Name.starts_with(Prefix: "pror.") || // Added in 8.0
332 Name.starts_with(Prefix: "prorv.") || // Added in 8.0
333 Name.starts_with(Prefix: "pshuf.b.") || // Added in 4.0
334 Name.starts_with(Prefix: "pshuf.d.") || // Added in 3.9
335 Name.starts_with(Prefix: "pshufh.w.") || // Added in 3.9
336 Name.starts_with(Prefix: "pshufl.w.") || // Added in 3.9
337 Name.starts_with(Prefix: "psll.d") || // Added in 4.0
338 Name.starts_with(Prefix: "psll.q") || // Added in 4.0
339 Name.starts_with(Prefix: "psll.w") || // Added in 4.0
340 Name.starts_with(Prefix: "pslli") || // Added in 4.0
341 Name.starts_with(Prefix: "psllv") || // Added in 4.0
342 Name.starts_with(Prefix: "psra.d") || // Added in 4.0
343 Name.starts_with(Prefix: "psra.q") || // Added in 4.0
344 Name.starts_with(Prefix: "psra.w") || // Added in 4.0
345 Name.starts_with(Prefix: "psrai") || // Added in 4.0
346 Name.starts_with(Prefix: "psrav") || // Added in 4.0
347 Name.starts_with(Prefix: "psrl.d") || // Added in 4.0
348 Name.starts_with(Prefix: "psrl.q") || // Added in 4.0
349 Name.starts_with(Prefix: "psrl.w") || // Added in 4.0
350 Name.starts_with(Prefix: "psrli") || // Added in 4.0
351 Name.starts_with(Prefix: "psrlv") || // Added in 4.0
352 Name.starts_with(Prefix: "psub.") || // Added in 4.0
353 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
354 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
355 Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
356 Name.starts_with(Prefix: "punpckh") || // Added in 3.9
357 Name.starts_with(Prefix: "punpckl") || // Added in 3.9
358 Name.starts_with(Prefix: "pxor.") || // Added in 3.9
359 Name.starts_with(Prefix: "shuf.f") || // Added in 6.0
360 Name.starts_with(Prefix: "shuf.i") || // Added in 6.0
361 Name.starts_with(Prefix: "shuf.p") || // Added in 4.0
362 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
363 Name.starts_with(Prefix: "store.b.") || // Added in 3.9
364 Name.starts_with(Prefix: "store.d.") || // Added in 3.9
365 Name.starts_with(Prefix: "store.p") || // Added in 3.9
366 Name.starts_with(Prefix: "store.q.") || // Added in 3.9
367 Name.starts_with(Prefix: "store.w.") || // Added in 3.9
368 Name == "store.ss" || // Added in 7.0
369 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
370 Name.starts_with(Prefix: "sub.p") || // Added in 7.0. 128/256 in 4.0
371 Name.starts_with(Prefix: "ucmp.") || // Added in 5.0
372 Name.starts_with(Prefix: "unpckh.") || // Added in 3.9
373 Name.starts_with(Prefix: "unpckl.") || // Added in 3.9
374 Name.starts_with(Prefix: "valign.") || // Added in 4.0
375 Name == "vcvtph2ps.128" || // Added in 11.0
376 Name == "vcvtph2ps.256" || // Added in 11.0
377 Name.starts_with(Prefix: "vextract") || // Added in 4.0
378 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
379 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
380 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
381 Name.starts_with(Prefix: "vfnmsub.") || // Added in 7.0
382 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
383 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
384 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
385 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
386 Name.starts_with(Prefix: "vpermi2var.") || // Added in 7.0
387 Name.starts_with(Prefix: "vpermil.p") || // Added in 3.9
388 Name.starts_with(Prefix: "vpermilvar.") || // Added in 4.0
389 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
390 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
391 Name.starts_with(Prefix: "vpshld.") || // Added in 7.0
392 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
393 Name.starts_with(Prefix: "vpshrd.") || // Added in 7.0
394 Name.starts_with(Prefix: "vpshrdv.") || // Added in 8.0
395 Name.starts_with(Prefix: "vpshufbitqmb.") || // Added in 8.0
396 Name.starts_with(Prefix: "xor.")); // Added in 3.9
397
398 if (Name.consume_front(Prefix: "mask3."))
399 // 'avx512.mask3.*'
400 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
401 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
402 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
403 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
404 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
405
406 if (Name.consume_front(Prefix: "maskz."))
407 // 'avx512.maskz.*'
408 return (Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
409 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
410 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
411 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
412 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
413 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
414 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
415 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
416 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
417 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
418 Name.starts_with(Prefix: "vpshrdv.")); // Added in 8.0
419
420 // 'avx512.*'
421 return (Name == "movntdqa" || // Added in 5.0
422 Name == "pmul.dq.512" || // Added in 7.0
423 Name == "pmulu.dq.512" || // Added in 7.0
424 Name.starts_with(Prefix: "broadcastm") || // Added in 6.0
425 Name.starts_with(Prefix: "cmp.p") || // Added in 12.0
426 Name.starts_with(Prefix: "cvtb2mask.") || // Added in 7.0
427 Name.starts_with(Prefix: "cvtd2mask.") || // Added in 7.0
428 Name.starts_with(Prefix: "cvtmask2") || // Added in 5.0
429 Name.starts_with(Prefix: "cvtq2mask.") || // Added in 7.0
430 Name == "cvtusi2sd" || // Added in 7.0
431 Name.starts_with(Prefix: "cvtw2mask.") || // Added in 7.0
432 Name == "kand.w" || // Added in 7.0
433 Name == "kandn.w" || // Added in 7.0
434 Name == "knot.w" || // Added in 7.0
435 Name == "kor.w" || // Added in 7.0
436 Name == "kortestc.w" || // Added in 7.0
437 Name == "kortestz.w" || // Added in 7.0
438 Name.starts_with(Prefix: "kunpck") || // added in 6.0
439 Name == "kxnor.w" || // Added in 7.0
440 Name == "kxor.w" || // Added in 7.0
441 Name.starts_with(Prefix: "padds.") || // Added in 8.0
442 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.9
443 Name.starts_with(Prefix: "pmulh.w") || // Added in 24.0
444 Name.starts_with(Prefix: "pmulhu.w") || // Added in 24.0
445 Name.starts_with(Prefix: "prol") || // Added in 8.0
446 Name.starts_with(Prefix: "pror") || // Added in 8.0
447 Name.starts_with(Prefix: "psll.dq") || // Added in 3.9
448 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.9
449 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
450 Name.starts_with(Prefix: "ptestm") || // Added in 6.0
451 Name.starts_with(Prefix: "ptestnm") || // Added in 6.0
452 Name.starts_with(Prefix: "storent.") || // Added in 3.9
453 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 7.0
454 Name.starts_with(Prefix: "vpshld.") || // Added in 8.0
455 Name.starts_with(Prefix: "vpshrd.")); // Added in 8.0
456 }
457
458 if (Name.consume_front(Prefix: "fma."))
459 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
460 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
461 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
462 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
463 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
464
465 if (Name.consume_front(Prefix: "fma4."))
466 return Name.starts_with(Prefix: "vfmadd.s"); // Added in 7.0
467
468 if (Name.consume_front(Prefix: "sse."))
469 return (Name == "add.ss" || // Added in 4.0
470 Name == "cvtsi2ss" || // Added in 7.0
471 Name == "cvtsi642ss" || // Added in 7.0
472 Name == "div.ss" || // Added in 4.0
473 Name == "mul.ss" || // Added in 4.0
474 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
475 Name == "sqrt.ss" || // Added in 7.0
476 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
477 Name == "sub.ss"); // Added in 4.0
478
479 if (Name.consume_front(Prefix: "sse2."))
480 return (Name == "add.sd" || // Added in 4.0
481 Name == "cvtdq2pd" || // Added in 3.9
482 Name == "cvtdq2ps" || // Added in 7.0
483 Name == "cvtps2pd" || // Added in 3.9
484 Name == "cvtsi2sd" || // Added in 7.0
485 Name == "cvtsi642sd" || // Added in 7.0
486 Name == "cvtss2sd" || // Added in 7.0
487 Name == "div.sd" || // Added in 4.0
488 Name == "mul.sd" || // Added in 4.0
489 Name.starts_with(Prefix: "padds.") || // Added in 8.0
490 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
491 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
492 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
493 Name == "pmaxs.w" || // Added in 3.9
494 Name == "pmaxu.b" || // Added in 3.9
495 Name == "pmins.w" || // Added in 3.9
496 Name == "pminu.b" || // Added in 3.9
497 Name == "pmulh.w" || // Added in 24.0
498 Name == "pmulhu.w" || // Added in 24.0
499 Name == "pmulu.dq" || // Added in 7.0
500 Name.starts_with(Prefix: "pshuf") || // Added in 3.9
501 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
502 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
503 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
504 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
505 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
506 Name == "sqrt.sd" || // Added in 7.0
507 Name == "storel.dq" || // Added in 3.9
508 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
509 Name == "sub.sd"); // Added in 4.0
510
511 if (Name.consume_front(Prefix: "sse41."))
512 return (Name.starts_with(Prefix: "blendp") || // Added in 3.7
513 Name == "movntdqa" || // Added in 5.0
514 Name == "pblendw" || // Added in 3.7
515 Name == "pmaxsb" || // Added in 3.9
516 Name == "pmaxsd" || // Added in 3.9
517 Name == "pmaxud" || // Added in 3.9
518 Name == "pmaxuw" || // Added in 3.9
519 Name == "pminsb" || // Added in 3.9
520 Name == "pminsd" || // Added in 3.9
521 Name == "pminud" || // Added in 3.9
522 Name == "pminuw" || // Added in 3.9
523 Name.starts_with(Prefix: "pmovsx") || // Added in 3.8
524 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
525 Name == "pmuldq"); // Added in 7.0
526
527 if (Name.consume_front(Prefix: "sse42."))
528 return Name == "crc32.64.8"; // Added in 3.4
529
530 if (Name.consume_front(Prefix: "sse4a."))
531 return Name.starts_with(Prefix: "movnt."); // Added in 3.9
532
533 if (Name.consume_front(Prefix: "ssse3."))
534 return (Name == "pabs.b.128" || // Added in 6.0
535 Name == "pabs.d.128" || // Added in 6.0
536 Name == "pabs.w.128"); // Added in 6.0
537
538 if (Name.consume_front(Prefix: "xop."))
539 return (Name == "vpcmov" || // Added in 3.8
540 Name == "vpcmov.256" || // Added in 5.0
541 Name.starts_with(Prefix: "vpcom") || // Added in 3.2, Updated in 9.0
542 Name.starts_with(Prefix: "vprot")); // Added in 8.0
543
544 if (Name.consume_front(Prefix: "bmi."))
545 return (Name.starts_with(Prefix: "pdep.") || // Added in 23.0
546 Name.starts_with(Prefix: "pext.")); // Added in 23.0
547
548 return (Name == "addcarry.u32" || // Added in 8.0
549 Name == "addcarry.u64" || // Added in 8.0
550 Name == "addcarryx.u32" || // Added in 8.0
551 Name == "addcarryx.u64" || // Added in 8.0
552 Name == "subborrow.u32" || // Added in 8.0
553 Name == "subborrow.u64" || // Added in 8.0
554 Name.starts_with(Prefix: "vcvtph2ps.")); // Added in 11.0
555}
556
557static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
558 Function *&NewFn) {
559 // Only handle intrinsics that start with "x86.".
560 if (!Name.consume_front(Prefix: "x86."))
561 return false;
562
563 if (shouldUpgradeX86Intrinsic(F, Name)) {
564 NewFn = nullptr;
565 return true;
566 }
567
568 if (Name == "rdtscp") { // Added in 8.0
569 // If this intrinsic has 0 operands, it's the new version.
570 if (F->getFunctionType()->getNumParams() == 0)
571 return false;
572
573 rename(GV: F);
574 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
575 id: Intrinsic::x86_rdtscp);
576 return true;
577 }
578
579 Intrinsic::ID ID;
580
581 // SSE4.1 ptest functions may have an old signature.
582 if (Name.consume_front(Prefix: "sse41.ptest")) { // Added in 3.2
583 ID = StringSwitch<Intrinsic::ID>(Name)
584 .Case(S: "c", Value: Intrinsic::x86_sse41_ptestc)
585 .Case(S: "z", Value: Intrinsic::x86_sse41_ptestz)
586 .Case(S: "nzc", Value: Intrinsic::x86_sse41_ptestnzc)
587 .Default(Value: Intrinsic::not_intrinsic);
588 if (ID != Intrinsic::not_intrinsic)
589 return upgradePTESTIntrinsic(F, IID: ID, NewFn);
590
591 return false;
592 }
593
594 // Several blend and other instructions with masks used the wrong number of
595 // bits.
596
597 // Added in 3.6
598 ID = StringSwitch<Intrinsic::ID>(Name)
599 .Case(S: "sse41.insertps", Value: Intrinsic::x86_sse41_insertps)
600 .Case(S: "sse41.dppd", Value: Intrinsic::x86_sse41_dppd)
601 .Case(S: "sse41.dpps", Value: Intrinsic::x86_sse41_dpps)
602 .Case(S: "sse41.mpsadbw", Value: Intrinsic::x86_sse41_mpsadbw)
603 .Case(S: "avx.dp.ps.256", Value: Intrinsic::x86_avx_dp_ps_256)
604 .Case(S: "avx2.mpsadbw", Value: Intrinsic::x86_avx2_mpsadbw)
605 .Default(Value: Intrinsic::not_intrinsic);
606 if (ID != Intrinsic::not_intrinsic)
607 return upgradeX86IntrinsicsWith8BitMask(F, IID: ID, NewFn);
608
609 if (Name.consume_front(Prefix: "avx512.")) {
610 if (Name.consume_front(Prefix: "mask.cmp.")) {
611 // Added in 7.0
612 ID = StringSwitch<Intrinsic::ID>(Name)
613 .Case(S: "pd.128", Value: Intrinsic::x86_avx512_mask_cmp_pd_128)
614 .Case(S: "pd.256", Value: Intrinsic::x86_avx512_mask_cmp_pd_256)
615 .Case(S: "pd.512", Value: Intrinsic::x86_avx512_mask_cmp_pd_512)
616 .Case(S: "ps.128", Value: Intrinsic::x86_avx512_mask_cmp_ps_128)
617 .Case(S: "ps.256", Value: Intrinsic::x86_avx512_mask_cmp_ps_256)
618 .Case(S: "ps.512", Value: Intrinsic::x86_avx512_mask_cmp_ps_512)
619 .Default(Value: Intrinsic::not_intrinsic);
620 if (ID != Intrinsic::not_intrinsic)
621 return upgradeX86MaskedFPCompare(F, IID: ID, NewFn);
622 } else if (Name.starts_with(Prefix: "vpdpbusd.") ||
623 Name.starts_with(Prefix: "vpdpbusds.")) {
624 // Added in 21.1
625 ID = StringSwitch<Intrinsic::ID>(Name)
626 .Case(S: "vpdpbusd.128", Value: Intrinsic::x86_avx512_vpdpbusd_128)
627 .Case(S: "vpdpbusd.256", Value: Intrinsic::x86_avx512_vpdpbusd_256)
628 .Case(S: "vpdpbusd.512", Value: Intrinsic::x86_avx512_vpdpbusd_512)
629 .Case(S: "vpdpbusds.128", Value: Intrinsic::x86_avx512_vpdpbusds_128)
630 .Case(S: "vpdpbusds.256", Value: Intrinsic::x86_avx512_vpdpbusds_256)
631 .Case(S: "vpdpbusds.512", Value: Intrinsic::x86_avx512_vpdpbusds_512)
632 .Default(Value: Intrinsic::not_intrinsic);
633 if (ID != Intrinsic::not_intrinsic)
634 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
635 } else if (Name.starts_with(Prefix: "vpdpwssd.") ||
636 Name.starts_with(Prefix: "vpdpwssds.")) {
637 // Added in 21.1
638 ID = StringSwitch<Intrinsic::ID>(Name)
639 .Case(S: "vpdpwssd.128", Value: Intrinsic::x86_avx512_vpdpwssd_128)
640 .Case(S: "vpdpwssd.256", Value: Intrinsic::x86_avx512_vpdpwssd_256)
641 .Case(S: "vpdpwssd.512", Value: Intrinsic::x86_avx512_vpdpwssd_512)
642 .Case(S: "vpdpwssds.128", Value: Intrinsic::x86_avx512_vpdpwssds_128)
643 .Case(S: "vpdpwssds.256", Value: Intrinsic::x86_avx512_vpdpwssds_256)
644 .Case(S: "vpdpwssds.512", Value: Intrinsic::x86_avx512_vpdpwssds_512)
645 .Default(Value: Intrinsic::not_intrinsic);
646 if (ID != Intrinsic::not_intrinsic)
647 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
648 }
649 return false; // No other 'x86.avx512.*'.
650 }
651
652 if (Name.consume_front(Prefix: "avx2.")) {
653 if (Name.consume_front(Prefix: "vpdpb")) {
654 // Added in 21.1
655 ID = StringSwitch<Intrinsic::ID>(Name)
656 .Case(S: "ssd.128", Value: Intrinsic::x86_avx2_vpdpbssd_128)
657 .Case(S: "ssd.256", Value: Intrinsic::x86_avx2_vpdpbssd_256)
658 .Case(S: "ssds.128", Value: Intrinsic::x86_avx2_vpdpbssds_128)
659 .Case(S: "ssds.256", Value: Intrinsic::x86_avx2_vpdpbssds_256)
660 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpbsud_128)
661 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpbsud_256)
662 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpbsuds_128)
663 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpbsuds_256)
664 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpbuud_128)
665 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpbuud_256)
666 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpbuuds_128)
667 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpbuuds_256)
668 .Default(Value: Intrinsic::not_intrinsic);
669 if (ID != Intrinsic::not_intrinsic)
670 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
671 } else if (Name.consume_front(Prefix: "vpdpw")) {
672 // Added in 21.1
673 ID = StringSwitch<Intrinsic::ID>(Name)
674 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpwsud_128)
675 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpwsud_256)
676 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpwsuds_128)
677 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpwsuds_256)
678 .Case(S: "usd.128", Value: Intrinsic::x86_avx2_vpdpwusd_128)
679 .Case(S: "usd.256", Value: Intrinsic::x86_avx2_vpdpwusd_256)
680 .Case(S: "usds.128", Value: Intrinsic::x86_avx2_vpdpwusds_128)
681 .Case(S: "usds.256", Value: Intrinsic::x86_avx2_vpdpwusds_256)
682 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpwuud_128)
683 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpwuud_256)
684 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpwuuds_128)
685 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpwuuds_256)
686 .Default(Value: Intrinsic::not_intrinsic);
687 if (ID != Intrinsic::not_intrinsic)
688 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
689 }
690 return false; // No other 'x86.avx2.*'
691 }
692
693 if (Name.consume_front(Prefix: "avx10.")) {
694 if (Name.consume_front(Prefix: "vpdpb")) {
695 // Added in 21.1
696 ID = StringSwitch<Intrinsic::ID>(Name)
697 .Case(S: "ssd.512", Value: Intrinsic::x86_avx10_vpdpbssd_512)
698 .Case(S: "ssds.512", Value: Intrinsic::x86_avx10_vpdpbssds_512)
699 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpbsud_512)
700 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpbsuds_512)
701 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpbuud_512)
702 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpbuuds_512)
703 .Default(Value: Intrinsic::not_intrinsic);
704 if (ID != Intrinsic::not_intrinsic)
705 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
706 } else if (Name.consume_front(Prefix: "vpdpw")) {
707 ID = StringSwitch<Intrinsic::ID>(Name)
708 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpwsud_512)
709 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpwsuds_512)
710 .Case(S: "usd.512", Value: Intrinsic::x86_avx10_vpdpwusd_512)
711 .Case(S: "usds.512", Value: Intrinsic::x86_avx10_vpdpwusds_512)
712 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpwuud_512)
713 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpwuuds_512)
714 .Default(Value: Intrinsic::not_intrinsic);
715 if (ID != Intrinsic::not_intrinsic)
716 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
717 }
718 return false; // No other 'x86.avx10.*'
719 }
720
721 if (Name.consume_front(Prefix: "avx512bf16.")) {
722 // Added in 9.0
723 ID = StringSwitch<Intrinsic::ID>(Name)
724 .Case(S: "cvtne2ps2bf16.128",
725 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
726 .Case(S: "cvtne2ps2bf16.256",
727 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
728 .Case(S: "cvtne2ps2bf16.512",
729 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
730 .Case(S: "mask.cvtneps2bf16.128",
731 Value: Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
732 .Case(S: "cvtneps2bf16.256",
733 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
734 .Case(S: "cvtneps2bf16.512",
735 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
736 .Default(Value: Intrinsic::not_intrinsic);
737 if (ID != Intrinsic::not_intrinsic)
738 return upgradeX86BF16Intrinsic(F, IID: ID, NewFn);
739
740 // Added in 9.0
741 ID = StringSwitch<Intrinsic::ID>(Name)
742 .Case(S: "dpbf16ps.128", Value: Intrinsic::x86_avx512bf16_dpbf16ps_128)
743 .Case(S: "dpbf16ps.256", Value: Intrinsic::x86_avx512bf16_dpbf16ps_256)
744 .Case(S: "dpbf16ps.512", Value: Intrinsic::x86_avx512bf16_dpbf16ps_512)
745 .Default(Value: Intrinsic::not_intrinsic);
746 if (ID != Intrinsic::not_intrinsic)
747 return upgradeX86BF16DPIntrinsic(F, IID: ID, NewFn);
748 return false; // No other 'x86.avx512bf16.*'.
749 }
750
751 if (Name.consume_front(Prefix: "xop.")) {
752 Intrinsic::ID ID = Intrinsic::not_intrinsic;
753 if (Name.starts_with(Prefix: "vpermil2")) { // Added in 3.9
754 // Upgrade any XOP PERMIL2 index operand still using a float/double
755 // vector.
756 auto Idx = F->getFunctionType()->getParamType(i: 2);
757 if (Idx->isFPOrFPVectorTy()) {
758 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
759 unsigned EltSize = Idx->getScalarSizeInBits();
760 if (EltSize == 64 && IdxSize == 128)
761 ID = Intrinsic::x86_xop_vpermil2pd;
762 else if (EltSize == 32 && IdxSize == 128)
763 ID = Intrinsic::x86_xop_vpermil2ps;
764 else if (EltSize == 64 && IdxSize == 256)
765 ID = Intrinsic::x86_xop_vpermil2pd_256;
766 else
767 ID = Intrinsic::x86_xop_vpermil2ps_256;
768 }
769 } else if (F->arg_size() == 2)
770 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
771 ID = StringSwitch<Intrinsic::ID>(Name)
772 .Case(S: "vfrcz.ss", Value: Intrinsic::x86_xop_vfrcz_ss)
773 .Case(S: "vfrcz.sd", Value: Intrinsic::x86_xop_vfrcz_sd)
774 .Default(Value: Intrinsic::not_intrinsic);
775
776 if (ID != Intrinsic::not_intrinsic) {
777 rename(GV: F);
778 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
779 return true;
780 }
781 return false; // No other 'x86.xop.*'
782 }
783
784 if (Name == "seh.recoverfp") {
785 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
786 id: Intrinsic::eh_recoverfp);
787 return true;
788 }
789
790 return false;
791}
792
793// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
794// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
795static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
796 StringRef Name,
797 Function *&NewFn) {
798 if (Name.starts_with(Prefix: "rbit")) {
799 // '(arm|aarch64).rbit'.
800 NewFn = Intrinsic::getOrInsertDeclaration(
801 M: F->getParent(), id: Intrinsic::bitreverse, OverloadTys: F->arg_begin()->getType());
802 return true;
803 }
804
805 if (Name == "thread.pointer") {
806 // '(arm|aarch64).thread.pointer'.
807 NewFn = Intrinsic::getOrInsertDeclaration(
808 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
809 return true;
810 }
811
812 bool Neon = Name.consume_front(Prefix: "neon.");
813 if (Neon) {
814 // '(arm|aarch64).neon.*'.
815 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
816 // v16i8 respectively.
817 if (Name.consume_front(Prefix: "bfdot.")) {
818 // (arm|aarch64).neon.bfdot.*'.
819 Intrinsic::ID ID =
820 StringSwitch<Intrinsic::ID>(Name)
821 .Cases(CaseStrings: {"v2f32.v8i8", "v4f32.v16i8"},
822 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
823 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
824 .Default(Value: Intrinsic::not_intrinsic);
825 if (ID != Intrinsic::not_intrinsic) {
826 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
827 assert((OperandWidth == 64 || OperandWidth == 128) &&
828 "Unexpected operand width");
829 LLVMContext &Ctx = F->getParent()->getContext();
830 std::array<Type *, 2> Tys{
831 ._M_elems: {F->getReturnType(),
832 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), NumElts: OperandWidth / 16)}};
833 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
834 return true;
835 }
836 return false; // No other '(arm|aarch64).neon.bfdot.*'.
837 }
838
839 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
840 // anymore and accept v8bf16 instead of v16i8.
841 if (Name.consume_front(Prefix: "bfm")) {
842 // (arm|aarch64).neon.bfm*'.
843 if (Name.consume_back(Suffix: ".v4f32.v16i8")) {
844 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
845 Intrinsic::ID ID =
846 StringSwitch<Intrinsic::ID>(Name)
847 .Case(S: "mla",
848 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
850 .Case(S: "lalb",
851 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
852 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
853 .Case(S: "lalt",
854 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
855 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
856 .Default(Value: Intrinsic::not_intrinsic);
857 if (ID != Intrinsic::not_intrinsic) {
858 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
859 return true;
860 }
861 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
862 }
863 return false; // No other '(arm|aarch64).neon.bfm*.
864 }
865 // Continue on to Aarch64 Neon or Arm Neon.
866 }
867 // Continue on to Arm or Aarch64.
868
869 if (IsArm) {
870 // 'arm.*'.
871 if (Neon) {
872 // 'arm.neon.*'.
873 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
874 .StartsWith(S: "vclz.", Value: Intrinsic::ctlz)
875 .StartsWith(S: "vcnt.", Value: Intrinsic::ctpop)
876 .StartsWith(S: "vqadds.", Value: Intrinsic::sadd_sat)
877 .StartsWith(S: "vqaddu.", Value: Intrinsic::uadd_sat)
878 .StartsWith(S: "vqsubs.", Value: Intrinsic::ssub_sat)
879 .StartsWith(S: "vqsubu.", Value: Intrinsic::usub_sat)
880 .StartsWith(S: "vrinta.", Value: Intrinsic::round)
881 .StartsWith(S: "vrintn.", Value: Intrinsic::roundeven)
882 .StartsWith(S: "vrintm.", Value: Intrinsic::floor)
883 .StartsWith(S: "vrintp.", Value: Intrinsic::ceil)
884 .StartsWith(S: "vrintx.", Value: Intrinsic::rint)
885 .StartsWith(S: "vrintz.", Value: Intrinsic::trunc)
886 .Default(Value: Intrinsic::not_intrinsic);
887 if (ID != Intrinsic::not_intrinsic) {
888 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
889 OverloadTys: F->arg_begin()->getType());
890 return true;
891 }
892
893 if (Name.consume_front(Prefix: "vst")) {
894 // 'arm.neon.vst*'.
895 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
896 SmallVector<StringRef, 2> Groups;
897 if (vstRegex.match(String: Name, Matches: &Groups)) {
898 static const Intrinsic::ID StoreInts[] = {
899 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
900 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
901
902 static const Intrinsic::ID StoreLaneInts[] = {
903 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
904 Intrinsic::arm_neon_vst4lane};
905
906 auto fArgs = F->getFunctionType()->params();
907 Type *Tys[] = {fArgs[0], fArgs[1]};
908 if (Groups[1].size() == 1)
909 NewFn = Intrinsic::getOrInsertDeclaration(
910 M: F->getParent(), id: StoreInts[fArgs.size() - 3], OverloadTys: Tys);
911 else
912 NewFn = Intrinsic::getOrInsertDeclaration(
913 M: F->getParent(), id: StoreLaneInts[fArgs.size() - 5], OverloadTys: Tys);
914 return true;
915 }
916 return false; // No other 'arm.neon.vst*'.
917 }
918
919 return false; // No other 'arm.neon.*'.
920 }
921
922 if (Name.consume_front(Prefix: "mve.")) {
923 // 'arm.mve.*'.
924 if (Name == "vctp64") {
925 if (cast<FixedVectorType>(Val: F->getReturnType())->getNumElements() == 4) {
926 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
927 // the function and deal with it below in UpgradeIntrinsicCall.
928 rename(GV: F);
929 return true;
930 }
931 return false; // Not 'arm.mve.vctp64'.
932 }
933
934 if (Name.starts_with(Prefix: "vrintn.v")) {
935 NewFn = Intrinsic::getOrInsertDeclaration(
936 M: F->getParent(), id: Intrinsic::roundeven, OverloadTys: F->arg_begin()->getType());
937 return true;
938 }
939
940 // These too are changed to accept a v2i1 instead of the old v4i1.
941 if (Name.consume_back(Suffix: ".v4i1")) {
942 // 'arm.mve.*.v4i1'.
943 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i32"))
944 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
945 return Name == "mull.int" || Name == "vqdmull";
946
947 if (Name.consume_back(Suffix: ".v2i64")) {
948 // 'arm.mve.*.v2i64.v4i1'
949 bool IsGather = Name.consume_front(Prefix: "vldr.gather.");
950 if (IsGather || Name.consume_front(Prefix: "vstr.scatter.")) {
951 if (Name.consume_front(Prefix: "base.")) {
952 // Optional 'wb.' prefix.
953 Name.consume_front(Prefix: "wb.");
954 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
955 // predicated.v2i64.v2i64.v4i1'.
956 return Name == "predicated.v2i64";
957 }
958
959 if (Name.consume_front(Prefix: "offset.predicated."))
960 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
961 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
962
963 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
964 return false;
965 }
966
967 return false; // No other 'arm.mve.*.v2i64.v4i1'.
968 }
969 return false; // No other 'arm.mve.*.v4i1'.
970 }
971 return false; // No other 'arm.mve.*'.
972 }
973
974 if (Name.consume_front(Prefix: "cde.vcx")) {
975 // 'arm.cde.vcx*'.
976 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i1"))
977 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
978 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
979 Name == "3q" || Name == "3qa";
980
981 return false; // No other 'arm.cde.vcx*'.
982 }
983 } else {
984 // 'aarch64.*'.
985 if (Neon) {
986 // 'aarch64.neon.*'.
987 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
988 .StartsWith(S: "frintn", Value: Intrinsic::roundeven)
989 .StartsWith(S: "rbit", Value: Intrinsic::bitreverse)
990 .Default(Value: Intrinsic::not_intrinsic);
991 if (ID != Intrinsic::not_intrinsic) {
992 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
993 OverloadTys: F->arg_begin()->getType());
994 return true;
995 }
996
997 Intrinsic::ID MinMaxID =
998 StringSwitch<Intrinsic::ID>(Name.split(Separator: '.').first)
999 .Case(S: "smax", Value: Intrinsic::smax)
1000 .Case(S: "smin", Value: Intrinsic::smin)
1001 .Case(S: "umax", Value: Intrinsic::umax)
1002 .Case(S: "umin", Value: Intrinsic::umin)
1003 .Default(Value: Intrinsic::not_intrinsic);
1004 if (MinMaxID != Intrinsic::not_intrinsic) {
1005 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1006 return false; // Invalid IR.
1007 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: MinMaxID,
1008 OverloadTys: F->getReturnType());
1009 return true;
1010 }
1011
1012 if (Name.starts_with(Prefix: "addp")) {
1013 // 'aarch64.neon.addp*'.
1014 if (F->arg_size() != 2)
1015 return false; // Invalid IR.
1016 VectorType *Ty = dyn_cast<VectorType>(Val: F->getReturnType());
1017 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1018 NewFn = Intrinsic::getOrInsertDeclaration(
1019 M: F->getParent(), id: Intrinsic::aarch64_neon_faddp, OverloadTys: Ty);
1020 return true;
1021 }
1022 }
1023
1024 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1025 if (Name.starts_with(Prefix: "bfcvt")) {
1026 NewFn = nullptr;
1027 return true;
1028 }
1029
1030 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1031 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1032 NewFn = nullptr;
1033 return true;
1034 }
1035
1036 return false; // No other 'aarch64.neon.*'.
1037 }
1038 if (Name.consume_front(Prefix: "sve.")) {
1039 // 'aarch64.sve.*'.
1040 if (Name.consume_front(Prefix: "bf")) {
1041 if (Name == "mmla") {
1042 Type *Tys[] = {F->getReturnType(),
1043 std::next(x: F->arg_begin())->getType()};
1044 NewFn = Intrinsic::getOrInsertDeclaration(
1045 M: F->getParent(), id: Intrinsic::aarch64_sve_fmmla, OverloadTys: Tys);
1046 return true;
1047 }
1048 if (Name.consume_back(Suffix: ".lane")) {
1049 // 'aarch64.sve.bf*.lane'.
1050 Intrinsic::ID ID =
1051 StringSwitch<Intrinsic::ID>(Name)
1052 .Case(S: "dot", Value: Intrinsic::aarch64_sve_bfdot_lane_v2)
1053 .Case(S: "mlalb", Value: Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1054 .Case(S: "mlalt", Value: Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1055 .Default(Value: Intrinsic::not_intrinsic);
1056 if (ID != Intrinsic::not_intrinsic) {
1057 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1058 return true;
1059 }
1060 return false; // No other 'aarch64.sve.bf*.lane'.
1061 }
1062 return false; // No other 'aarch64.sve.bf*'.
1063 }
1064
1065 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1066 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1067 NewFn = nullptr;
1068 return true;
1069 }
1070
1071 if (Name.consume_front(Prefix: "convert.from.svbool")) {
1072 // 'aarch64.sve.convert.from.svbool'
1073 auto *TTy = dyn_cast<TargetExtType>(Val: F->getReturnType());
1074 if (!TTy || TTy->getName() != "aarch64.svcount")
1075 return false;
1076
1077 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1078 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1079 return true;
1080 }
1081
1082 if (Name.consume_front(Prefix: "convert.to.svbool")) {
1083 // 'aarch64.sve.convert.to.svbool'
1084 auto *TTy = dyn_cast<TargetExtType>(Val: F->arg_begin()->getType());
1085 if (!TTy || TTy->getName() != "aarch64.svcount")
1086 return false;
1087
1088 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1089 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1090 return true;
1091 }
1092
1093 if (Name.consume_front(Prefix: "addqv")) {
1094 // 'aarch64.sve.addqv'.
1095 if (!F->getReturnType()->isFPOrFPVectorTy())
1096 return false;
1097
1098 auto Args = F->getFunctionType()->params();
1099 Type *Tys[] = {F->getReturnType(), Args[1]};
1100 NewFn = Intrinsic::getOrInsertDeclaration(
1101 M: F->getParent(), id: Intrinsic::aarch64_sve_faddqv, OverloadTys: Tys);
1102 return true;
1103 }
1104
1105 if (Name.consume_front(Prefix: "ld")) {
1106 // 'aarch64.sve.ld*'.
1107 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1108 if (LdRegex.match(String: Name)) {
1109 Type *ScalarTy =
1110 cast<VectorType>(Val: F->getReturnType())->getElementType();
1111 ElementCount EC =
1112 cast<VectorType>(Val: F->arg_begin()->getType())->getElementCount();
1113 assert(F->arg_size() == 2 &&
1114 "Expected 2 arguments for ld* intrinsic.");
1115 Type *PtrTy = F->getArg(i: 1)->getType();
1116 Type *Ty = VectorType::get(ElementType: ScalarTy, EC);
1117 static const Intrinsic::ID LoadIDs[] = {
1118 Intrinsic::aarch64_sve_ld2_sret,
1119 Intrinsic::aarch64_sve_ld3_sret,
1120 Intrinsic::aarch64_sve_ld4_sret,
1121 };
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 M: F->getParent(), id: LoadIDs[Name[0] - '2'], OverloadTys: {Ty, PtrTy});
1124 return true;
1125 }
1126 return false; // No other 'aarch64.sve.ld*'.
1127 }
1128
1129 if (Name.consume_front(Prefix: "tuple.")) {
1130 // 'aarch64.sve.tuple.*'.
1131 if (Name.starts_with(Prefix: "get")) {
1132 // 'aarch64.sve.tuple.get*'.
1133 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1134 NewFn = Intrinsic::getOrInsertDeclaration(
1135 M: F->getParent(), id: Intrinsic::vector_extract, OverloadTys: Tys);
1136 return true;
1137 }
1138
1139 if (Name.starts_with(Prefix: "set")) {
1140 // 'aarch64.sve.tuple.set*'.
1141 auto Args = F->getFunctionType()->params();
1142 Type *Tys[] = {Args[0], Args[2], Args[1]};
1143 NewFn = Intrinsic::getOrInsertDeclaration(
1144 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1145 return true;
1146 }
1147
1148 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1149 if (CreateTupleRegex.match(String: Name)) {
1150 // 'aarch64.sve.tuple.create*'.
1151 auto Args = F->getFunctionType()->params();
1152 Type *Tys[] = {F->getReturnType(), Args[1]};
1153 NewFn = Intrinsic::getOrInsertDeclaration(
1154 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1155 return true;
1156 }
1157 return false; // No other 'aarch64.sve.tuple.*'.
1158 }
1159
1160 if (Name.starts_with(Prefix: "rev.nxv")) {
1161 // 'aarch64.sve.rev.<Ty>'
1162 NewFn = Intrinsic::getOrInsertDeclaration(
1163 M: F->getParent(), id: Intrinsic::vector_reverse, OverloadTys: F->getReturnType());
1164 return true;
1165 }
1166
1167 return false; // No other 'aarch64.sve.*'.
1168 }
1169 if (Name.consume_front(Prefix: "sme.")) {
1170 // 'aarch64.sme.*'.
1171 if (Name.consume_front(Prefix: "ftmopa.")) {
1172 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1173 // intrinsics to model their FPMR dependency.
1174 Intrinsic::ID ID =
1175 StringSwitch<Intrinsic::ID>(Name)
1176 .Case(S: "za16.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1177 .Case(S: "za32.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1178 .Default(Value: Intrinsic::not_intrinsic);
1179 if (ID != Intrinsic::not_intrinsic) {
1180 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1181 return true;
1182 }
1183 return false; // No other 'aarch64.sme.ftmopa.*'.
1184 }
1185
1186 return false; // No other 'aarch64.sme.*'.
1187 }
1188 }
1189 return false; // No other 'arm.*', 'aarch64.*'.
1190}
1191
1192// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1193// declarations requiring an auto-upgrade. The same set applies to the
1194// cluster (g2s) and CTA (g2s_cta) variants.
1195#define NVVM_TMA_G2S_MODES(M) \
1196 M(tile_1d, "tile.1d") \
1197 M(tile_2d, "tile.2d") \
1198 M(tile_3d, "tile.3d") \
1199 M(tile_4d, "tile.4d") \
1200 M(tile_5d, "tile.5d") \
1201 M(tile_gather4_2d, "tile.gather4.2d") \
1202 M(im2col_3d, "im2col.3d") \
1203 M(im2col_4d, "im2col.4d") \
1204 M(im2col_5d, "im2col.5d") \
1205 M(im2col_w_3d, "im2col.w.3d") \
1206 M(im2col_w_4d, "im2col.w.4d") \
1207 M(im2col_w_5d, "im2col.w.5d") \
1208 M(im2col_w_128_3d, "im2col.w.128.3d") \
1209 M(im2col_w_128_4d, "im2col.w.128.4d") \
1210 M(im2col_w_128_5d, "im2col.w.128.5d")
1211
1212// Two legacy tails are:
1213//
1214// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1215// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1216//
1217// The current tail appends a trailing i32 %flag_valid_pattern, so both
1218// legacy tails are recognized by an i1 at parameter N-2.
1219static Intrinsic::ID
1220shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1221 SmallVectorImpl<Type *> &OvlTys) {
1222 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s."))
1223 return Intrinsic::not_intrinsic;
1224
1225#define G2S_ID(ID_SUFFIX, NAME) \
1226 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1227 // clang-format off
1228 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1229 NVVM_TMA_G2S_MODES(G2S_ID)
1230 .Default(Value: Intrinsic::not_intrinsic);
1231#undef G2S_ID
1232 // clang-format on
1233 if (ID == Intrinsic::not_intrinsic)
1234 return ID;
1235
1236 size_t NumParams = F->getFunctionType()->getNumParams();
1237
1238 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1239 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1240 if (!F->getFunctionType()->getParamType(i: NumParams - 2)->isIntegerTy(BitWidth: 1))
1241 return Intrinsic::not_intrinsic;
1242
1243 // The multicast mask is the parameter immediately before the i64
1244 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1245 ArrayRef<Type *> Params = F->getFunctionType()->params();
1246 size_t MaskIdx =
1247 Params[NumParams - 1]->isIntegerTy(BitWidth: 1) ? NumParams - 4 : NumParams - 5;
1248 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1249 "expected the i64 cache-hint after the multicast mask");
1250 Type *MaskTy = Params[MaskIdx];
1251 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1252 OvlTys.push_back(Elt: MaskTy);
1253
1254 return ID;
1255}
1256
1257// The legacy tail is:
1258//
1259// arg1, arg2, .. i64 %ch, i1 %flag_ch
1260//
1261// The current tail appends a trailing i32 %flag_valid_pattern, so the
1262// legacy tail is recognized by an i1 at parameter N-1.
1263static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1264 StringRef Name) {
1265 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s.cta."))
1266 return Intrinsic::not_intrinsic;
1267
1268#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1269 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1270 // clang-format off
1271 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1272 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1273 .Default(Value: Intrinsic::not_intrinsic);
1274#undef G2S_CTA_ID
1275 // clang-format on
1276 if (ID == Intrinsic::not_intrinsic)
1277 return ID;
1278
1279 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1280 // with i32 %flag_valid_pattern, for which N-1 is i32.
1281 if (!F->getFunctionType()
1282 ->getParamType(i: F->getFunctionType()->getNumParams() - 1)
1283 ->isIntegerTy(BitWidth: 1))
1284 return Intrinsic::not_intrinsic;
1285
1286 return ID;
1287}
1288// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1289//
1290// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1291//
1292// The current intrinsic is overloaded on the multicast-mask type and takes a
1293// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1294// parameter N-1.
1295static Intrinsic::ID
1296shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1297 SmallVectorImpl<Type *> &OvlTys) {
1298 if (!Name.consume_front(Prefix: "cp.async.bulk.global.to.shared.cluster"))
1299 return Intrinsic::not_intrinsic;
1300
1301 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1302 // i32 %flag_valid_pattern, for which N-1 is i32.
1303 size_t NumParams = F->getFunctionType()->getNumParams();
1304 if (!F->getFunctionType()->getParamType(i: NumParams - 1)->isIntegerTy(BitWidth: 1))
1305 return Intrinsic::not_intrinsic;
1306
1307 // The multicast mask is parameter 4; legacy IR only uses i16.
1308 Type *MaskTy = F->getFunctionType()->getParamType(i: NumParams - 4);
1309 if (!MaskTy->isIntegerTy(BitWidth: 16))
1310 return Intrinsic::not_intrinsic;
1311 OvlTys.push_back(Elt: MaskTy);
1312
1313 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1314}
1315
1316// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1317//
1318// ..., i64 %ch, i1 %flag_ch
1319//
1320// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1321// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1322// recognized by an i1 at parameter N-1, whereas the current tail ends
1323// with an i32.
1324static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1325 StringRef Name) {
1326 if (!Name.consume_front(Prefix: "cp.async.bulk.global.to.shared.cta"))
1327 return Intrinsic::not_intrinsic;
1328
1329 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1330 // i32 %flag_valid_pattern, for which N-1 is i32.
1331 if (!F->getFunctionType()->getParamType(i: 5)->isIntegerTy(BitWidth: 1))
1332 return Intrinsic::not_intrinsic;
1333
1334 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1335}
1336
1337// The legacy TMA reduction intrinsics encode the reduction operator in their
1338// name, while the current ones take it as an immediate argument. Map the
1339// operator part of a legacy name to the corresponding immediate value.
1340static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1341 return StringSwitch<std::optional<unsigned>>(Name)
1342 .Case(S: "add", Value: static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1343 .Case(S: "min", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1344 .Case(S: "max", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1345 .Case(S: "inc", Value: static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1346 .Case(S: "dec", Value: static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1347 .Case(S: "and", Value: static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1348 .Case(S: "or", Value: static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1349 .Case(S: "xor", Value: static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1350 .Default(Value: std::nullopt);
1351}
1352
1353static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1354 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.reduce."))
1355 return Intrinsic::not_intrinsic;
1356
1357 auto [RedOpName, ShapeName] = Name.split(Separator: '.');
1358 if (!getNVPTXTMAReductionOp(Name: RedOpName))
1359 return Intrinsic::not_intrinsic;
1360
1361 return StringSwitch<Intrinsic::ID>(ShapeName)
1362 .Case(S: "tile.1d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1363 .Case(S: "tile.2d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1364 .Case(S: "tile.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1365 .Case(S: "tile.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1366 .Case(S: "tile.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1367 .Case(S: "im2col.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1368 .Case(S: "im2col.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1369 .Case(S: "im2col.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1370 .Default(Value: Intrinsic::not_intrinsic);
1371}
1372
1373static Intrinsic::ID
1374shouldUpgradeNVVMBarrierMulticastIntrinsic(Function *F, StringRef Name) {
1375 if (!Name.consume_front(Prefix: "mbarrier.") ||
1376 !Name.consume_back(Suffix: ".space.cluster") || F->arg_size() != 2)
1377 return Intrinsic::not_intrinsic;
1378
1379 Intrinsic::ID IID = F->getIntrinsicID();
1380 if (IID == Intrinsic::not_intrinsic ||
1381 Intrinsic::getType(Context&: F->getContext(), id: IID) == F->getFunctionType())
1382 return Intrinsic::not_intrinsic;
1383
1384 return IID;
1385}
1386
1387static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1388 StringRef Name) {
1389 if (Name.consume_front(Prefix: "mapa.shared.cluster"))
1390 if (F->getReturnType()->getPointerAddressSpace() ==
1391 NVPTXAS::ADDRESS_SPACE_SHARED)
1392 return Intrinsic::nvvm_mapa_shared_cluster;
1393
1394 if (Name.consume_front(Prefix: "cp.async.bulk.")) {
1395 Intrinsic::ID ID =
1396 StringSwitch<Intrinsic::ID>(Name)
1397 .Case(S: "shared.cta.to.cluster",
1398 Value: Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1399 .Default(Value: Intrinsic::not_intrinsic);
1400
1401 if (ID != Intrinsic::not_intrinsic)
1402 if (F->getArg(i: 0)->getType()->getPointerAddressSpace() ==
1403 NVPTXAS::ADDRESS_SPACE_SHARED)
1404 return ID;
1405 }
1406
1407 return Intrinsic::not_intrinsic;
1408}
1409
1410static Intrinsic::ID
1411shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1412 if (!Name.consume_front(Prefix: "tcgen05.commit."))
1413 return Intrinsic::not_intrinsic;
1414
1415 if (Name.consume_front(Prefix: "shared."))
1416 return StringSwitch<Intrinsic::ID>(Name)
1417 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_cg1)
1418 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_cg2)
1419 .Default(Value: Intrinsic::not_intrinsic);
1420
1421 if (Name.consume_front(Prefix: "mc.shared.")) {
1422 // Only upgrade older i16 mc variants.
1423 if (!F->getArg(i: 1)->getType()->isIntegerTy(BitWidth: 16))
1424 return Intrinsic::not_intrinsic;
1425
1426 return StringSwitch<Intrinsic::ID>(Name)
1427 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1428 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1429 .Default(Value: Intrinsic::not_intrinsic);
1430 }
1431
1432 return Intrinsic::not_intrinsic;
1433}
1434
1435static Intrinsic::ID
1436shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(Function *F, StringRef Name) {
1437 if (F->arg_size() != 2)
1438 return Intrinsic::not_intrinsic;
1439
1440 return StringSwitch<Intrinsic::ID>(Name)
1441 .Case(S: "tcgen05.alloc.shared.cg1", Value: Intrinsic::nvvm_tcgen05_alloc_cg1)
1442 .Case(S: "tcgen05.alloc.shared.cg2", Value: Intrinsic::nvvm_tcgen05_alloc_cg2)
1443 .Default(Value: Intrinsic::not_intrinsic);
1444}
1445
1446static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1447 if (Name.consume_front(Prefix: "fmax."))
1448 return StringSwitch<Intrinsic::ID>(Name)
1449 .Case(S: "bf16", Value: Intrinsic::nvvm_fmax_bf16)
1450 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmax_bf16x2)
1451 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmax_ftz_bf16)
1452 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_bf16x2)
1453 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16)
1454 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1455 .Case(S: "ftz.nan.xorsign.abs.bf16",
1456 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1457 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1458 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1459 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1460 .Case(S: "ftz.xorsign.abs.bf16x2",
1461 Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1462 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmax_nan_bf16)
1463 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmax_nan_bf16x2)
1464 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1465 .Case(S: "nan.xorsign.abs.bf16x2",
1466 Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1467 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1468 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1469 .Default(Value: Intrinsic::not_intrinsic);
1470
1471 if (Name.consume_front(Prefix: "fmin."))
1472 return StringSwitch<Intrinsic::ID>(Name)
1473 .Case(S: "bf16", Value: Intrinsic::nvvm_fmin_bf16)
1474 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmin_bf16x2)
1475 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmin_ftz_bf16)
1476 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_bf16x2)
1477 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16)
1478 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1479 .Case(S: "ftz.nan.xorsign.abs.bf16",
1480 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1481 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1482 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1483 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1484 .Case(S: "ftz.xorsign.abs.bf16x2",
1485 Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1486 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmin_nan_bf16)
1487 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmin_nan_bf16x2)
1488 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1489 .Case(S: "nan.xorsign.abs.bf16x2",
1490 Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1491 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1492 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1493 .Default(Value: Intrinsic::not_intrinsic);
1494
1495 if (Name.consume_front(Prefix: "neg."))
1496 return StringSwitch<Intrinsic::ID>(Name)
1497 .Case(S: "bf16", Value: Intrinsic::nvvm_neg_bf16)
1498 .Case(S: "bf16x2", Value: Intrinsic::nvvm_neg_bf16x2)
1499 .Default(Value: Intrinsic::not_intrinsic);
1500
1501 return Intrinsic::not_intrinsic;
1502}
1503
1504static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1505 FunctionType *NewFnTy = Intrinsic::getType(Context&: F->getContext(), id: IID);
1506 FunctionType *OldFnTy = F->getFunctionType();
1507 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1508 return OldTy->getScalarType()->isIntegerTy() &&
1509 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1510 };
1511
1512 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1513 return false;
1514
1515 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1516 return false;
1517
1518 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1519 if (!IsOldBF16StorageTy(OldFnTy->getParamType(i: I), NewFnTy->getParamType(i: I)))
1520 return false;
1521
1522 return true;
1523}
1524
1525// Overloaded fadd/fmul/fma intrinsic IDs, indexed by
1526// [`.ftz`][none|`.sat`|`.relu`].
1527static constexpr Intrinsic::ID NVVMFAddIIDs[2][3] = {
1528 {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat, Intrinsic::not_intrinsic},
1529 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat,
1530 Intrinsic::not_intrinsic}};
1531static constexpr Intrinsic::ID NVVMFMulIIDs[2][3] = {
1532 {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat, Intrinsic::not_intrinsic},
1533 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat,
1534 Intrinsic::not_intrinsic}};
1535static constexpr Intrinsic::ID NVVMFMAIIDs[2][3] = {
1536 {Intrinsic::nvvm_fma, Intrinsic::nvvm_fma_sat, Intrinsic::nvvm_fma_relu},
1537 {Intrinsic::nvvm_fma_ftz, Intrinsic::nvvm_fma_ftz_sat,
1538 Intrinsic::nvvm_fma_ftz_relu}};
1539
1540// Returns true if the legacy nvvm.{add,mul,fma}.<rnd>{.ftz}{.sat|.relu}.<Type>
1541// intrinsic with these modifiers existed.
1542static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID (&IIDs)[2][3],
1543 StringRef Type, bool IsRN,
1544 bool IsFTZ, bool IsSat,
1545 bool IsRelu) {
1546 const bool IsFMA = &IIDs == &NVVMFMAIIDs;
1547 const bool IsFMul = &IIDs == &NVVMFMulIIDs;
1548 return StringSwitch<bool>(Type)
1549 .Case(S: "f", Value: !IsRelu && !(IsSat && IsFMul))
1550 .Case(S: "d", Value: !IsFTZ && !IsSat && !IsRelu)
1551 .Case(S: "f16", Value: IsRN && (IsFMA || IsSat))
1552 .Case(S: "v2f16", Value: IsRN && !IsFMA && IsSat)
1553 .Case(S: "f16x2", Value: IsRN && IsFMA)
1554 .Cases(CaseStrings: {"bf16", "bf16x2"}, Value: IsRN && IsFMA && !IsFTZ && !IsSat)
1555 .Default(Value: false);
1556}
1557
1558static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1559getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID (&IIDs)[2][3]) {
1560 auto [Modifiers, Type] = Name.rsplit(Separator: '.');
1561
1562 std::optional<llvm::RoundingMode> RoundingMode =
1563 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(N: 2))
1564 .Case(S: "rn", Value: llvm::RoundingMode::NearestTiesToEven)
1565 .Case(S: "rz", Value: llvm::RoundingMode::TowardZero)
1566 .Case(S: "rm", Value: llvm::RoundingMode::TowardNegative)
1567 .Case(S: "rp", Value: llvm::RoundingMode::TowardPositive)
1568 .Default(Value: std::nullopt);
1569 if (!RoundingMode)
1570 return std::nullopt;
1571
1572 StringRef Rest = Modifiers.drop_front(N: 2);
1573 const bool IsFTZ = Rest.consume_front(Prefix: ".ftz");
1574 const bool IsSat = Rest.consume_front(Prefix: ".sat");
1575 const bool IsRelu = !IsSat && Rest.consume_front(Prefix: ".relu");
1576 if (!Rest.empty())
1577 return std::nullopt;
1578
1579 const Intrinsic::ID IID = IIDs[IsFTZ][IsSat ? 1 : IsRelu ? 2 : 0];
1580 if (IID == Intrinsic::not_intrinsic)
1581 return std::nullopt;
1582
1583 // Reject combinations that never existed as legacy intrinsics.
1584 const bool IsRN = *RoundingMode == llvm::RoundingMode::NearestTiesToEven;
1585 if (!isValidLegacyNVVMFPArithIntrinsic(IIDs, Type, IsRN, IsFTZ, IsSat,
1586 IsRelu))
1587 return std::nullopt;
1588
1589 return std::make_pair(x: IID, y&: *RoundingMode);
1590}
1591
1592static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1593 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1594 return Intrinsic::not_intrinsic;
1595
1596 return Intrinsic::nvvm_mbarrier_init;
1597}
1598
1599static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1600 return Name.consume_front(Prefix: "local") || Name.consume_front(Prefix: "shared") ||
1601 Name.consume_front(Prefix: "global") || Name.consume_front(Prefix: "constant") ||
1602 Name.consume_front(Prefix: "param");
1603}
1604
1605static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1606 if (!Name.consume_front(Prefix: "vp."))
1607 return 0;
1608 return StringSwitch<unsigned>(Name)
1609 .StartsWith(S: "select", Value: Instruction::Select)
1610 .StartsWith(S: "add", Value: Instruction::Add)
1611 .StartsWith(S: "sub", Value: Instruction::Sub)
1612 .StartsWith(S: "mul", Value: Instruction::Mul)
1613 .StartsWith(S: "ashr", Value: Instruction::AShr)
1614 .StartsWith(S: "lshr", Value: Instruction::LShr)
1615 .StartsWith(S: "shl", Value: Instruction::Shl)
1616 .StartsWith(S: "or", Value: Instruction::Or)
1617 .StartsWith(S: "and", Value: Instruction::And)
1618 .StartsWith(S: "xor", Value: Instruction::Xor)
1619 .StartsWith(S: "fadd", Value: Instruction::FAdd)
1620 .StartsWith(S: "fsub", Value: Instruction::FSub)
1621 .StartsWith(S: "fmuladd", Value: 0)
1622 .StartsWith(S: "fmul", Value: Instruction::FMul)
1623 .StartsWith(S: "fdiv", Value: Instruction::FDiv)
1624 .StartsWith(S: "frem", Value: Instruction::FRem)
1625 .StartsWith(S: "fneg", Value: Instruction::FNeg)
1626 .StartsWith(S: "trunc", Value: Instruction::Trunc)
1627 .StartsWith(S: "zext", Value: Instruction::ZExt)
1628 .StartsWith(S: "sext", Value: Instruction::SExt)
1629 .StartsWith(S: "fptrunc", Value: Instruction::FPTrunc)
1630 .StartsWith(S: "fpext", Value: Instruction::FPExt)
1631 .StartsWith(S: "fptoui", Value: Instruction::FPToUI)
1632 .StartsWith(S: "fptosi", Value: Instruction::FPToSI)
1633 .StartsWith(S: "uitofp", Value: Instruction::UIToFP)
1634 .StartsWith(S: "sitofp", Value: Instruction::SIToFP)
1635 .StartsWith(S: "ptrtoint", Value: Instruction::PtrToInt)
1636 .StartsWith(S: "inttoptr", Value: Instruction::IntToPtr)
1637 .StartsWith(S: "icmp", Value: Instruction::ICmp)
1638 .StartsWith(S: "fcmp", Value: Instruction::FCmp)
1639 .Default(Value: 0);
1640}
1641
1642static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1643 if (!Name.consume_front(Prefix: "vp."))
1644 return 0;
1645 return StringSwitch<Intrinsic::ID>(Name)
1646 .StartsWith(S: "abs", Value: Intrinsic::abs)
1647 .StartsWith(S: "smax", Value: Intrinsic::smax)
1648 .StartsWith(S: "smin", Value: Intrinsic::smin)
1649 .StartsWith(S: "umax", Value: Intrinsic::umax)
1650 .StartsWith(S: "umin", Value: Intrinsic::umin)
1651 .StartsWith(S: "copysign", Value: Intrinsic::copysign)
1652 .StartsWith(S: "minnum", Value: Intrinsic::minnum)
1653 .StartsWith(S: "maxnum", Value: Intrinsic::maxnum)
1654 .StartsWith(S: "minimum", Value: Intrinsic::minimum)
1655 .StartsWith(S: "maximum", Value: Intrinsic::maximum)
1656 .StartsWith(S: "fabs", Value: Intrinsic::fabs)
1657 .StartsWith(S: "sqrt", Value: Intrinsic::sqrt)
1658 .StartsWith(S: "fma", Value: Intrinsic::fma)
1659 .StartsWith(S: "fmuladd", Value: Intrinsic::fmuladd)
1660 .StartsWith(S: "ceil", Value: Intrinsic::ceil)
1661 .StartsWith(S: "floor", Value: Intrinsic::floor)
1662 .StartsWith(S: "rint", Value: Intrinsic::rint)
1663 .StartsWith(S: "nearbyint", Value: Intrinsic::nearbyint)
1664 .StartsWith(S: "roundeven", Value: Intrinsic::roundeven)
1665 .StartsWith(S: "roundtozero", Value: Intrinsic::trunc)
1666 .StartsWith(S: "round", Value: Intrinsic::round)
1667 .StartsWith(S: "lrint", Value: Intrinsic::lrint)
1668 .StartsWith(S: "llrint", Value: Intrinsic::llrint)
1669 .StartsWith(S: "bitreverse", Value: Intrinsic::bitreverse)
1670 .StartsWith(S: "bswap", Value: Intrinsic::bswap)
1671 .StartsWith(S: "ctpop", Value: Intrinsic::ctpop)
1672 .StartsWith(S: "ctlz", Value: Intrinsic::ctlz)
1673 .StartsWith(S: "cttz.elts", Value: 0)
1674 .StartsWith(S: "cttz", Value: Intrinsic::cttz)
1675 .StartsWith(S: "sadd.sat", Value: Intrinsic::sadd_sat)
1676 .StartsWith(S: "uadd.sat", Value: Intrinsic::uadd_sat)
1677 .StartsWith(S: "ssub.sat", Value: Intrinsic::ssub_sat)
1678 .StartsWith(S: "usub.sat", Value: Intrinsic::usub_sat)
1679 .StartsWith(S: "fshl", Value: Intrinsic::fshl)
1680 .StartsWith(S: "fshr", Value: Intrinsic::fshr)
1681 .StartsWith(S: "is.fpclass", Value: Intrinsic::is_fpclass)
1682 .Default(Value: 0);
1683}
1684
1685static bool shouldUpgradeVPIntrinsic(StringRef Name) {
1686 return getFunctionalOpcodeForVP(Name) || getFunctionalIntrinsicIDForVP(Name);
1687}
1688
1689static bool convertIntrinsicValidType(StringRef Name,
1690 const FunctionType *FuncTy) {
1691 Type *HalfTy = Type::getHalfTy(C&: FuncTy->getContext());
1692 if (Name.starts_with(Prefix: "to.fp16")) {
1693 return CastInst::castIsValid(op: Instruction::FPTrunc, SrcTy: FuncTy->getParamType(i: 0),
1694 DstTy: HalfTy) &&
1695 CastInst::castIsValid(op: Instruction::BitCast, SrcTy: HalfTy,
1696 DstTy: FuncTy->getReturnType());
1697 }
1698
1699 if (Name.starts_with(Prefix: "from.fp16")) {
1700 return CastInst::castIsValid(op: Instruction::BitCast, SrcTy: FuncTy->getParamType(i: 0),
1701 DstTy: HalfTy) &&
1702 CastInst::castIsValid(op: Instruction::FPExt, SrcTy: HalfTy,
1703 DstTy: FuncTy->getReturnType());
1704 }
1705
1706 return false;
1707}
1708
1709static unsigned
1710getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID,
1711 SmallVectorImpl<Type *> &OverloadTys) {
1712 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1713 if (Defaults.empty())
1714 return 0;
1715
1716 unsigned FullArgCount = FirstDefault + Defaults.size();
1717
1718 // Only trailing default arguments can be missing.
1719 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1720 return 0;
1721
1722 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1723 if (!Intrinsic::isSignatureValid(ID: IID, FT: F->getFunctionType(), OverloadTys,
1724 NumMissingTrailingParams))
1725 return 0;
1726
1727 return FullArgCount;
1728}
1729
1730static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1731 Intrinsic::ID IID = F->getIntrinsicID();
1732 SmallVector<Type *, 4> OverloadTys;
1733
1734 unsigned FullArgCount =
1735 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1736 if (FullArgCount == 0)
1737 return false;
1738
1739 rename(GV: F);
1740 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID, OverloadTys);
1741 assert(NewFn->arg_size() == FullArgCount &&
1742 "total number of default args does not match intrinsic signature");
1743 return true;
1744}
1745
1746static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1747 bool CanUpgradeDebugIntrinsicsToRecords) {
1748 assert(F && "Illegal to upgrade a non-existent Function.");
1749
1750 StringRef Name = F->getName();
1751
1752 // Quickly eliminate it, if it's not a candidate.
1753 if (!Name.consume_front(Prefix: "llvm.") || Name.empty())
1754 return false;
1755
1756 switch (Name[0]) {
1757 default: break;
1758 case 'a': {
1759 bool IsArm = Name.consume_front(Prefix: "arm.");
1760 if (IsArm || Name.consume_front(Prefix: "aarch64.")) {
1761 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1762 return true;
1763 break;
1764 }
1765
1766 if (Name.consume_front(Prefix: "amdgcn.")) {
1767 if (Name == "alignbit") {
1768 // Target specific intrinsic became redundant
1769 NewFn = Intrinsic::getOrInsertDeclaration(
1770 M: F->getParent(), id: Intrinsic::fshr, OverloadTys: {F->getReturnType()});
1771 return true;
1772 }
1773
1774 if (Name.consume_front(Prefix: "atomic.")) {
1775 if (Name.starts_with(Prefix: "inc") || Name.starts_with(Prefix: "dec") ||
1776 Name.starts_with(Prefix: "cond.sub") || Name.starts_with(Prefix: "csub")) {
1777 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1778 // and usub_sat so there's no new declaration.
1779 NewFn = nullptr;
1780 return true;
1781 }
1782 break; // No other 'amdgcn.atomic.*'
1783 }
1784
1785 if (Name.starts_with(Prefix: "addrspacecast.nonnull")) {
1786 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1787 // so there's no new declaration.
1788 NewFn = nullptr;
1789 return true;
1790 }
1791
1792 switch (F->getIntrinsicID()) {
1793 default:
1794 break;
1795 // Legacy asyncmark intrinsics without the stage operand.
1796 case Intrinsic::amdgcn_asyncmark:
1797 case Intrinsic::amdgcn_wait_asyncmark: {
1798 Intrinsic::ID ID = F->getIntrinsicID();
1799 if (F->arg_size() == (ID == Intrinsic::amdgcn_asyncmark ? 0u : 1u)) {
1800 rename(GV: F);
1801 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1802 return true;
1803 }
1804 break;
1805 }
1806 // Legacy wmma iu intrinsics without the optional clamp operand.
1807 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1808 if (F->arg_size() == 7) {
1809 NewFn = nullptr;
1810 return true;
1811 }
1812 break;
1813 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1814 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1815 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1816 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1817 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1818 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1819 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1820 if (F->arg_size() == 8) {
1821 NewFn = nullptr;
1822 return true;
1823 }
1824 break;
1825 }
1826
1827 if (Name.consume_front(Prefix: "ds.") || Name.consume_front(Prefix: "global.atomic.") ||
1828 Name.consume_front(Prefix: "flat.atomic.")) {
1829 if (Name.starts_with(Prefix: "fadd") ||
1830 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1831 (Name.starts_with(Prefix: "fmin") && !Name.starts_with(Prefix: "fmin.num")) ||
1832 (Name.starts_with(Prefix: "fmax") && !Name.starts_with(Prefix: "fmax.num"))) {
1833 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1834 // declaration.
1835 NewFn = nullptr;
1836 return true;
1837 }
1838 }
1839
1840 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
1841 NewFn = nullptr;
1842 return true;
1843 }
1844
1845 if (Name.starts_with(Prefix: "ldexp.")) {
1846 // Target specific intrinsic became redundant
1847 NewFn = Intrinsic::getOrInsertDeclaration(
1848 M: F->getParent(), id: Intrinsic::ldexp,
1849 OverloadTys: {F->getReturnType(), F->getArg(i: 1)->getType()});
1850 return true;
1851 }
1852 break; // No other 'amdgcn.*'
1853 }
1854
1855 break;
1856 }
1857 case 'c': {
1858 if (F->arg_size() == 1) {
1859 if (Name.consume_front(Prefix: "convert.")) {
1860 if (convertIntrinsicValidType(Name, FuncTy: F->getFunctionType())) {
1861 NewFn = nullptr;
1862 return true;
1863 }
1864 }
1865
1866 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1867 .StartsWith(S: "ctlz.", Value: Intrinsic::ctlz)
1868 .StartsWith(S: "cttz.", Value: Intrinsic::cttz)
1869 .Default(Value: Intrinsic::not_intrinsic);
1870 if (ID != Intrinsic::not_intrinsic) {
1871 rename(GV: F);
1872 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1873 OverloadTys: F->arg_begin()->getType());
1874 return true;
1875 }
1876 }
1877
1878 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1879 if (Name == "coro.end" &&
1880 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(BitWidth: 1)))
1881 CoroEndID = Intrinsic::coro_end;
1882 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(BitWidth: 1))
1883 CoroEndID = Intrinsic::coro_end_async;
1884
1885 if (CoroEndID != Intrinsic::not_intrinsic) {
1886 rename(GV: F);
1887 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: CoroEndID);
1888 return true;
1889 }
1890
1891 break;
1892 }
1893 case 'd':
1894 if (Name.consume_front(Prefix: "dbg.")) {
1895 // Mark debug intrinsics for upgrade to new debug format.
1896 if (CanUpgradeDebugIntrinsicsToRecords) {
1897 if (Name == "addr" || Name == "value" || Name == "assign" ||
1898 Name == "declare" || Name == "label") {
1899 // There's no function to replace these with.
1900 NewFn = nullptr;
1901 // But we do want these to get upgraded.
1902 return true;
1903 }
1904 }
1905 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1906 // converted to DbgVariableRecords later.
1907 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1908 rename(GV: F);
1909 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1910 id: Intrinsic::dbg_value);
1911 return true;
1912 }
1913 break; // No other 'dbg.*'.
1914 }
1915 break;
1916 case 'e':
1917 if (Name.consume_front(Prefix: "experimental.vector.")) {
1918 Intrinsic::ID ID =
1919 StringSwitch<Intrinsic::ID>(Name)
1920 // Skip over extract.last.active, otherwise it will be 'upgraded'
1921 // to a regular vector extract which is a different operation.
1922 .StartsWith(S: "extract.last.active.", Value: Intrinsic::not_intrinsic)
1923 .StartsWith(S: "extract.", Value: Intrinsic::vector_extract)
1924 .StartsWith(S: "insert.", Value: Intrinsic::vector_insert)
1925 .StartsWith(S: "reverse.", Value: Intrinsic::vector_reverse)
1926 .StartsWith(S: "interleave2.", Value: Intrinsic::vector_interleave2)
1927 .StartsWith(S: "deinterleave2.", Value: Intrinsic::vector_deinterleave2)
1928 .StartsWith(S: "partial.reduce.add",
1929 Value: Intrinsic::vector_partial_reduce_add)
1930 .Default(Value: Intrinsic::not_intrinsic);
1931 if (ID != Intrinsic::not_intrinsic) {
1932 const auto *FT = F->getFunctionType();
1933 SmallVector<Type *, 2> Tys;
1934 if (ID == Intrinsic::vector_extract ||
1935 ID == Intrinsic::vector_interleave2)
1936 // Extracting overloads the return type.
1937 Tys.push_back(Elt: FT->getReturnType());
1938 if (ID != Intrinsic::vector_interleave2)
1939 Tys.push_back(Elt: FT->getParamType(i: 0));
1940 if (ID == Intrinsic::vector_insert ||
1941 ID == Intrinsic::vector_partial_reduce_add)
1942 // Inserting overloads the inserted type.
1943 Tys.push_back(Elt: FT->getParamType(i: 1));
1944 rename(GV: F);
1945 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
1946 return true;
1947 }
1948
1949 if (Name.consume_front(Prefix: "reduce.")) {
1950 SmallVector<StringRef, 2> Groups;
1951 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1952 if (R.match(String: Name, Matches: &Groups))
1953 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1954 .Case(S: "add", Value: Intrinsic::vector_reduce_add)
1955 .Case(S: "mul", Value: Intrinsic::vector_reduce_mul)
1956 .Case(S: "and", Value: Intrinsic::vector_reduce_and)
1957 .Case(S: "or", Value: Intrinsic::vector_reduce_or)
1958 .Case(S: "xor", Value: Intrinsic::vector_reduce_xor)
1959 .Case(S: "smax", Value: Intrinsic::vector_reduce_smax)
1960 .Case(S: "smin", Value: Intrinsic::vector_reduce_smin)
1961 .Case(S: "umax", Value: Intrinsic::vector_reduce_umax)
1962 .Case(S: "umin", Value: Intrinsic::vector_reduce_umin)
1963 .Case(S: "fmax", Value: Intrinsic::vector_reduce_fmax)
1964 .Case(S: "fmin", Value: Intrinsic::vector_reduce_fmin)
1965 .Default(Value: Intrinsic::not_intrinsic);
1966
1967 bool V2 = false;
1968 if (ID == Intrinsic::not_intrinsic) {
1969 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1970 Groups.clear();
1971 V2 = true;
1972 if (R2.match(String: Name, Matches: &Groups))
1973 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1974 .Case(S: "fadd", Value: Intrinsic::vector_reduce_fadd)
1975 .Case(S: "fmul", Value: Intrinsic::vector_reduce_fmul)
1976 .Default(Value: Intrinsic::not_intrinsic);
1977 }
1978 if (ID != Intrinsic::not_intrinsic) {
1979 rename(GV: F);
1980 auto Args = F->getFunctionType()->params();
1981 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1982 OverloadTys: {Args[V2 ? 1 : 0]});
1983 return true;
1984 }
1985 break; // No other 'expermental.vector.reduce.*'.
1986 }
1987
1988 if (Name.consume_front(Prefix: "splice"))
1989 return true;
1990 break; // No other 'experimental.vector.*'.
1991 }
1992 if (Name.consume_front(Prefix: "experimental.stepvector.")) {
1993 Intrinsic::ID ID = Intrinsic::stepvector;
1994 rename(GV: F);
1995 NewFn = Intrinsic::getOrInsertDeclaration(
1996 M: F->getParent(), id: ID, OverloadTys: F->getFunctionType()->getReturnType());
1997 return true;
1998 }
1999 break; // No other 'e*'.
2000 case 'f':
2001 if (Name.starts_with(Prefix: "flt.rounds")) {
2002 rename(GV: F);
2003 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2004 id: Intrinsic::get_rounding);
2005 return true;
2006 }
2007 break;
2008 case 'i':
2009 if (Name.starts_with(Prefix: "invariant.group.barrier")) {
2010 // Rename invariant.group.barrier to launder.invariant.group
2011 auto Args = F->getFunctionType()->params();
2012 Type* ObjectPtr[1] = {Args[0]};
2013 rename(GV: F);
2014 NewFn = Intrinsic::getOrInsertDeclaration(
2015 M: F->getParent(), id: Intrinsic::launder_invariant_group, OverloadTys: ObjectPtr);
2016 return true;
2017 }
2018 break;
2019 case 'l': {
2020 bool IsLifetimeStart = Name.consume_front(Prefix: "lifetime.start");
2021 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front(Prefix: "lifetime.end");
2022 if (IsLifetimeStart || IsLifetimeEnd) {
2023 if (F->arg_size() == 2) {
2024 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
2025 : Intrinsic::lifetime_end;
2026 rename(GV: F);
2027 // Old 2 argument form of these intrinsics have [Size, Ptr] as
2028 // arguments. Use the Ptr argument to create new declaration.
2029 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2030 OverloadTys: F->getArg(i: 1)->getType());
2031 return true;
2032 } else if (F->arg_size() == 1 && Name == ".i64") {
2033 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
2034 // Autoupgrade prior to
2035 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
2036 // intrinsic with no expected calls. To allow auto-upgrade process to
2037 // delete such invalid intrinsic declaration, set NewFn = nullptr
2038 // and return true here. If there are actual calls to this intrinsic
2039 // (which is not expected), they will be deleted in
2040 // UpgradeIntrinsicCall.
2041 NewFn = nullptr;
2042 return true;
2043 }
2044 }
2045 break;
2046 }
2047 case 'm': {
2048 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
2049 // alignment parameter to embedding the alignment as an attribute of
2050 // the pointer args.
2051 if (unsigned ID = StringSwitch<unsigned>(Name)
2052 .StartsWith(S: "memcpy.", Value: Intrinsic::memcpy)
2053 .StartsWith(S: "memmove.", Value: Intrinsic::memmove)
2054 .Default(Value: 0)) {
2055 if (F->arg_size() == 5) {
2056 rename(GV: F);
2057 // Get the types of dest, src, and len
2058 ArrayRef<Type *> ParamTypes =
2059 F->getFunctionType()->params().slice(N: 0, M: 3);
2060 NewFn =
2061 Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: ParamTypes);
2062 return true;
2063 }
2064 }
2065 if (Name.starts_with(Prefix: "memset.") && F->arg_size() == 5) {
2066 rename(GV: F);
2067 // Get the types of dest, and len
2068 const auto *FT = F->getFunctionType();
2069 Type *ParamTypes[2] = {
2070 FT->getParamType(i: 0), // Dest
2071 FT->getParamType(i: 2) // len
2072 };
2073 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2074 id: Intrinsic::memset, OverloadTys: ParamTypes);
2075 return true;
2076 }
2077
2078 unsigned MaskedID =
2079 StringSwitch<unsigned>(Name)
2080 .StartsWith(S: "masked.load", Value: Intrinsic::masked_load)
2081 .StartsWith(S: "masked.gather", Value: Intrinsic::masked_gather)
2082 .StartsWith(S: "masked.store", Value: Intrinsic::masked_store)
2083 .StartsWith(S: "masked.scatter", Value: Intrinsic::masked_scatter)
2084 .Default(Value: 0);
2085 if (MaskedID && F->arg_size() == 4) {
2086 rename(GV: F);
2087 if (MaskedID == Intrinsic::masked_load ||
2088 MaskedID == Intrinsic::masked_gather) {
2089 NewFn = Intrinsic::getOrInsertDeclaration(
2090 M: F->getParent(), id: MaskedID,
2091 OverloadTys: {F->getReturnType(), F->getArg(i: 0)->getType()});
2092 return true;
2093 }
2094 NewFn = Intrinsic::getOrInsertDeclaration(
2095 M: F->getParent(), id: MaskedID,
2096 OverloadTys: {F->getArg(i: 0)->getType(), F->getArg(i: 1)->getType()});
2097 return true;
2098 }
2099 break;
2100 }
2101 case 'n': {
2102 if (Name.consume_front(Prefix: "nvvm.")) {
2103 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2104 if (F->arg_size() == 1) {
2105 Intrinsic::ID IID =
2106 StringSwitch<Intrinsic::ID>(Name)
2107 .Cases(CaseStrings: {"brev32", "brev64"}, Value: Intrinsic::bitreverse)
2108 .Case(S: "clz.i", Value: Intrinsic::ctlz)
2109 .Case(S: "popc.i", Value: Intrinsic::ctpop)
2110 .Default(Value: Intrinsic::not_intrinsic);
2111 if (IID != Intrinsic::not_intrinsic) {
2112 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2113 OverloadTys: {F->getReturnType()});
2114 return true;
2115 }
2116 } else if (F->arg_size() == 2) {
2117 Intrinsic::ID IID =
2118 StringSwitch<Intrinsic::ID>(Name)
2119 .Cases(CaseStrings: {"max.s", "max.i", "max.ll"}, Value: Intrinsic::smax)
2120 .Cases(CaseStrings: {"min.s", "min.i", "min.ll"}, Value: Intrinsic::smin)
2121 .Cases(CaseStrings: {"max.us", "max.ui", "max.ull"}, Value: Intrinsic::umax)
2122 .Cases(CaseStrings: {"min.us", "min.ui", "min.ull"}, Value: Intrinsic::umin)
2123 .Cases(CaseStrings: {"mulhi.s", "mulhi.i", "mulhi.ll"}, Value: Intrinsic::smulh)
2124 .Cases(CaseStrings: {"mulhi.us", "mulhi.ui", "mulhi.ull"}, Value: Intrinsic::umulh)
2125 .Default(Value: Intrinsic::not_intrinsic);
2126 if (IID != Intrinsic::not_intrinsic) {
2127 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2128 OverloadTys: {F->getReturnType()});
2129 return true;
2130 }
2131 }
2132
2133 // Check for nvvm intrinsics that need a return type adjustment.
2134 {
2135 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2136 if (IID != Intrinsic::not_intrinsic &&
2137 isLegacyNVPTXBF16IntSignature(F, IID)) {
2138 NewFn = nullptr;
2139 return true;
2140 }
2141 }
2142
2143 // Upgrade Distributed Shared Memory Intrinsics
2144 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2145 if (IID != Intrinsic::not_intrinsic) {
2146 rename(GV: F);
2147 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2148 return true;
2149 }
2150
2151 // Upgrade TMA reduction intrinsics
2152 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2153 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2154 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2155 if (IID != Intrinsic::not_intrinsic) {
2156 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2157 return true;
2158 }
2159
2160 // Upgrade the shared::cluster mbarrier expect_tx/complete_tx/arrive*
2161 // intrinsics to the forms carrying the 32-bit multicast CTA mask and
2162 // the is-multicast flag.
2163 IID = shouldUpgradeNVVMBarrierMulticastIntrinsic(F, Name);
2164 if (IID != Intrinsic::not_intrinsic) {
2165 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2166 return true;
2167 }
2168
2169 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2170 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2171 if (IID != Intrinsic::not_intrinsic) {
2172 rename(GV: F);
2173 NewFn = Intrinsic::getOrInsertDeclaration(
2174 M: F->getParent(), IID, RetTy: F->getReturnType(),
2175 ArgTys: F->getFunctionType()->params());
2176 return true;
2177 }
2178
2179 // Upgrade tcgen05.alloc.shared.* to anyptr alloc intrinsic.
2180 IID = shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(F, Name);
2181 if (IID != Intrinsic::not_intrinsic) {
2182 rename(GV: F);
2183 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2184 OverloadTys: {F->getArg(i: 0)->getType()});
2185 return true;
2186 }
2187
2188 // Upgrade TMA copy G2S CTA intrinsics.
2189 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2190 if (IID != Intrinsic::not_intrinsic) {
2191 rename(GV: F);
2192 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2193 return true;
2194 }
2195
2196 // Upgrade TMA copy G2S (cluster) intrinsics.
2197 SmallVector<Type *, 1> OvlTys;
2198 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2199 if (IID != Intrinsic::not_intrinsic) {
2200 rename(GV: F);
2201 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID, OverloadTys: OvlTys);
2202 return true;
2203 }
2204
2205 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2206 // (multicast-mask overloading + trailing flag_valid_pattern).
2207 SmallVector<Type *, 1> BulkG2SOvlTys;
2208 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, OvlTys&: BulkG2SOvlTys);
2209 if (IID != Intrinsic::not_intrinsic) {
2210 rename(GV: F);
2211 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2212 OverloadTys: BulkG2SOvlTys);
2213 return true;
2214 }
2215
2216 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2217 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2218 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2219 if (IID != Intrinsic::not_intrinsic) {
2220 rename(GV: F);
2221 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2222 return true;
2223 }
2224
2225 // Upgrade mbarrier.init intrinsics missing the layout operand.
2226 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2227 if (IID != Intrinsic::not_intrinsic) {
2228 rename(GV: F);
2229 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2230 OverloadTys: F->getArg(i: 0)->getType());
2231 return true;
2232 }
2233
2234 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2235 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2236 //
2237 // TODO: We could add lohi.i2d.
2238 bool Expand = false;
2239 if (Name.consume_front(Prefix: "abs."))
2240 // nvvm.abs.{i,ii}
2241 Expand =
2242 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2243 else if (Name.consume_front(Prefix: "fabs."))
2244 // nvvm.fabs.{f,ftz.f,d}
2245 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2246 else if (Name.consume_front(Prefix: "add."))
2247 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2248 Expand = getNVVMFPArithUpgrade(Name, IIDs: NVVMFAddIIDs).has_value();
2249 else if (Name.consume_front(Prefix: "mul."))
2250 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2251 Expand = getNVVMFPArithUpgrade(Name, IIDs: NVVMFMulIIDs).has_value();
2252 else if (Name.consume_front(Prefix: "ex2.approx."))
2253 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2254 Expand =
2255 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2256 else if (Name.consume_front(Prefix: "fma."))
2257 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
2258 Expand = getNVVMFPArithUpgrade(Name, IIDs: NVVMFMAIIDs).has_value();
2259 else if (Name.consume_front(Prefix: "atomic.load."))
2260 // nvvm.atomic.load.add.{f32,f64}.p
2261 // nvvm.atomic.load.{inc,dec}.32.p
2262 Expand = StringSwitch<bool>(Name)
2263 .StartsWith(S: "add.f32.p", Value: true)
2264 .StartsWith(S: "add.f64.p", Value: true)
2265 .StartsWith(S: "inc.32.p", Value: true)
2266 .StartsWith(S: "dec.32.p", Value: true)
2267 .Default(Value: false);
2268 else if (Name.consume_front(Prefix: "atomic."))
2269 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2270 // nvvm.atomic.cas.gen.i.{cta,sys}
2271 Expand = StringSwitch<bool>(Name)
2272 .StartsWith(S: "add.gen.", Value: true)
2273 .StartsWith(S: "exch.gen.", Value: true)
2274 .StartsWith(S: "max.gen.", Value: true)
2275 .StartsWith(S: "min.gen.", Value: true)
2276 .StartsWith(S: "inc.gen.", Value: true)
2277 .StartsWith(S: "dec.gen.", Value: true)
2278 .StartsWith(S: "and.gen.", Value: true)
2279 .StartsWith(S: "or.gen.", Value: true)
2280 .StartsWith(S: "xor.gen.", Value: true)
2281 .StartsWith(S: "cas.gen.", Value: true)
2282 .Default(Value: false);
2283 else if (Name.consume_front(Prefix: "bitcast."))
2284 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2285 Expand =
2286 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2287 else if (Name.consume_front(Prefix: "rotate."))
2288 // nvvm.rotate.{b32,b64,right.b64}
2289 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2290 else if (Name.consume_front(Prefix: "ptr.gen.to."))
2291 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2292 Expand = consumeNVVMPtrAddrSpace(Name);
2293 else if (Name.consume_front(Prefix: "ptr."))
2294 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2295 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(Prefix: ".to.gen");
2296 else if (Name.consume_front(Prefix: "ldg.global."))
2297 // nvvm.ldg.global.{i,p,f}
2298 Expand = (Name.starts_with(Prefix: "i.") || Name.starts_with(Prefix: "f.") ||
2299 Name.starts_with(Prefix: "p."));
2300 else
2301 Expand = StringSwitch<bool>(Name)
2302 .Case(S: "barrier0", Value: true)
2303 .Case(S: "barrier.n", Value: true)
2304 .Case(S: "barrier.sync.cnt", Value: true)
2305 .Case(S: "barrier.sync", Value: true)
2306 .Case(S: "barrier", Value: true)
2307 .Case(S: "bar.sync", Value: true)
2308 .Case(S: "barrier0.popc", Value: true)
2309 .Case(S: "barrier0.and", Value: true)
2310 .Case(S: "barrier0.or", Value: true)
2311 .Case(S: "clz.ll", Value: true)
2312 .Case(S: "popc.ll", Value: true)
2313 .Case(S: "h2f", Value: true)
2314 .Case(S: "swap.lo.hi.b64", Value: true)
2315 .Case(S: "tanh.approx.f32", Value: true)
2316 .Default(Value: false);
2317
2318 if (Expand) {
2319 NewFn = nullptr;
2320 return true;
2321 }
2322 break; // No other 'nvvm.*'.
2323 }
2324 break;
2325 }
2326 case 'o':
2327 if (Name.starts_with(Prefix: "objectsize.")) {
2328 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2329 if (F->arg_size() == 2 || F->arg_size() == 3) {
2330 rename(GV: F);
2331 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2332 id: Intrinsic::objectsize, OverloadTys: Tys);
2333 return true;
2334 }
2335 }
2336 break;
2337
2338 case 'p':
2339 if (Name.starts_with(Prefix: "ptr.annotation.") && F->arg_size() == 4) {
2340 rename(GV: F);
2341 NewFn = Intrinsic::getOrInsertDeclaration(
2342 M: F->getParent(), id: Intrinsic::ptr_annotation,
2343 OverloadTys: {F->arg_begin()->getType(), F->getArg(i: 1)->getType()});
2344 return true;
2345 }
2346 break;
2347
2348 case 'r': {
2349 if (Name.consume_front(Prefix: "riscv.")) {
2350 Intrinsic::ID ID;
2351 ID = StringSwitch<Intrinsic::ID>(Name)
2352 .Case(S: "aes32dsi", Value: Intrinsic::riscv_aes32dsi)
2353 .Case(S: "aes32dsmi", Value: Intrinsic::riscv_aes32dsmi)
2354 .Case(S: "aes32esi", Value: Intrinsic::riscv_aes32esi)
2355 .Case(S: "aes32esmi", Value: Intrinsic::riscv_aes32esmi)
2356 .Default(Value: Intrinsic::not_intrinsic);
2357 if (ID != Intrinsic::not_intrinsic) {
2358 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32)) {
2359 rename(GV: F);
2360 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2361 return true;
2362 }
2363 break; // No other applicable upgrades.
2364 }
2365
2366 ID = StringSwitch<Intrinsic::ID>(Name)
2367 .StartsWith(S: "sm4ks", Value: Intrinsic::riscv_sm4ks)
2368 .StartsWith(S: "sm4ed", Value: Intrinsic::riscv_sm4ed)
2369 .Default(Value: Intrinsic::not_intrinsic);
2370 if (ID != Intrinsic::not_intrinsic) {
2371 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32) ||
2372 F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2373 rename(GV: F);
2374 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2375 return true;
2376 }
2377 break; // No other applicable upgrades.
2378 }
2379
2380 ID = StringSwitch<Intrinsic::ID>(Name)
2381 .StartsWith(S: "sha256sig0", Value: Intrinsic::riscv_sha256sig0)
2382 .StartsWith(S: "sha256sig1", Value: Intrinsic::riscv_sha256sig1)
2383 .StartsWith(S: "sha256sum0", Value: Intrinsic::riscv_sha256sum0)
2384 .StartsWith(S: "sha256sum1", Value: Intrinsic::riscv_sha256sum1)
2385 .StartsWith(S: "sm3p0", Value: Intrinsic::riscv_sm3p0)
2386 .StartsWith(S: "sm3p1", Value: Intrinsic::riscv_sm3p1)
2387 .Default(Value: Intrinsic::not_intrinsic);
2388 if (ID != Intrinsic::not_intrinsic) {
2389 if (F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2390 rename(GV: F);
2391 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2392 return true;
2393 }
2394 break; // No other applicable upgrades.
2395 }
2396
2397 // Replace llvm.riscv.clmul with llvm.clmul.
2398 if (Name == "clmul.i32" || Name == "clmul.i64") {
2399 NewFn = Intrinsic::getOrInsertDeclaration(
2400 M: F->getParent(), id: Intrinsic::clmul, OverloadTys: {F->getReturnType()});
2401 return true;
2402 }
2403
2404 break; // No other 'riscv.*' intrinsics
2405 }
2406 } break;
2407
2408 case 's':
2409 if (Name == "stackprotectorcheck") {
2410 NewFn = nullptr;
2411 return true;
2412 }
2413 if (Name.starts_with(Prefix: "strip.invariant.group")) {
2414 // For clang's usage it would be safe to just drop the
2415 // strip.invariant.group, but to be conservative replace with the
2416 // stronger launder.invariant.group instead.
2417 NewFn = Intrinsic::getOrInsertDeclaration(
2418 M: F->getParent(), id: Intrinsic::launder_invariant_group,
2419 OverloadTys: F->getReturnType());
2420 return true;
2421 }
2422 break;
2423
2424 case 't':
2425 if (Name == "thread.pointer") {
2426 NewFn = Intrinsic::getOrInsertDeclaration(
2427 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
2428 return true;
2429 }
2430 break;
2431
2432 case 'v': {
2433 if (Name == "var.annotation" && F->arg_size() == 4) {
2434 rename(GV: F);
2435 NewFn = Intrinsic::getOrInsertDeclaration(
2436 M: F->getParent(), id: Intrinsic::var_annotation,
2437 OverloadTys: {{F->arg_begin()->getType(), F->getArg(i: 1)->getType()}});
2438 return true;
2439 }
2440 if (Name.consume_front(Prefix: "vector.splice")) {
2441 if (Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"))
2442 break;
2443 return true;
2444 }
2445 if (shouldUpgradeVPIntrinsic(Name))
2446 return true;
2447 break;
2448 }
2449
2450 case 'w':
2451 if (Name.consume_front(Prefix: "wasm.")) {
2452 Intrinsic::ID ID =
2453 StringSwitch<Intrinsic::ID>(Name)
2454 .StartsWith(S: "fma.", Value: Intrinsic::wasm_relaxed_madd)
2455 .StartsWith(S: "fms.", Value: Intrinsic::wasm_relaxed_nmadd)
2456 .StartsWith(S: "laneselect.", Value: Intrinsic::wasm_relaxed_laneselect)
2457 .Default(Value: Intrinsic::not_intrinsic);
2458 if (ID != Intrinsic::not_intrinsic) {
2459 rename(GV: F);
2460 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
2461 OverloadTys: F->getReturnType());
2462 return true;
2463 }
2464
2465 if (Name.consume_front(Prefix: "dot.i8x16.i7x16.")) {
2466 ID = StringSwitch<Intrinsic::ID>(Name)
2467 .Case(S: "signed", Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2468 .Case(S: "add.signed",
2469 Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2470 .Default(Value: Intrinsic::not_intrinsic);
2471 if (ID != Intrinsic::not_intrinsic) {
2472 rename(GV: F);
2473 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2474 return true;
2475 }
2476 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2477 }
2478 break; // No other 'wasm.*'.
2479 }
2480 break;
2481
2482 case 'x':
2483 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2484 return true;
2485 }
2486
2487 auto *ST = dyn_cast<StructType>(Val: F->getReturnType());
2488 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2489 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2490 // Replace return type with literal non-packed struct. Only do this for
2491 // intrinsics declared to return a struct, not for intrinsics with
2492 // overloaded return type, in which case the exact struct type will be
2493 // mangled into the name.
2494 if (Intrinsic::hasStructReturnType(id: F->getIntrinsicID())) {
2495 FunctionType *FT = F->getFunctionType();
2496 auto *NewST = StructType::get(Context&: ST->getContext(), Elements: ST->elements());
2497 auto *NewFT = FunctionType::get(Result: NewST, Params: FT->params(), isVarArg: FT->isVarArg());
2498 std::string Name = F->getName().str();
2499 rename(GV: F);
2500 NewFn = Function::Create(Ty: NewFT, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
2501 N: Name, M: F->getParent());
2502
2503 // The new function may also need remangling.
2504 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F: NewFn))
2505 NewFn = *Result;
2506 return true;
2507 }
2508 }
2509
2510 // Remangle our intrinsic since we upgrade the mangling
2511 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2512 if (Result != std::nullopt) {
2513 NewFn = *Result;
2514 return true;
2515 }
2516
2517 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2518 return true;
2519
2520 // This may not belong here. This function is effectively being overloaded
2521 // to both detect an intrinsic which needs upgrading, and to provide the
2522 // upgraded form of the intrinsic. We should perhaps have two separate
2523 // functions for this.
2524
2525 return false;
2526}
2527
2528bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2529 bool CanUpgradeDebugIntrinsicsToRecords) {
2530 NewFn = nullptr;
2531 bool Upgraded =
2532 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2533
2534 // Upgrade intrinsic attributes. This does not change the function.
2535 if (NewFn)
2536 F = NewFn;
2537 if (Intrinsic::ID id = F->getIntrinsicID()) {
2538 // Only do this if the intrinsic signature is valid.
2539 SmallVector<Type *> OverloadTys;
2540 if (Intrinsic::isSignatureValid(ID: id, FT: F->getFunctionType(), OverloadTys))
2541 F->setAttributes(
2542 Intrinsic::getAttributes(C&: F->getContext(), id, FT: F->getFunctionType()));
2543 }
2544 return Upgraded;
2545}
2546
2547GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2548 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2549 GV->getName() == "llvm.global_dtors")) ||
2550 !GV->hasInitializer())
2551 return nullptr;
2552 ArrayType *ATy = dyn_cast<ArrayType>(Val: GV->getValueType());
2553 if (!ATy)
2554 return nullptr;
2555 StructType *STy = dyn_cast<StructType>(Val: ATy->getElementType());
2556 if (!STy || STy->getNumElements() != 2)
2557 return nullptr;
2558
2559 IRBuilder<> IRB(*GV->getParent());
2560 auto EltTy = StructType::get(elt1: STy->getElementType(N: 0), elts: STy->getElementType(N: 1),
2561 elts: IRB.getPtrTy());
2562 Constant *Init = GV->getInitializer();
2563 unsigned N = Init->getNumOperands();
2564 std::vector<Constant *> NewCtors(N);
2565 for (unsigned i = 0; i != N; ++i) {
2566 auto Ctor = cast<Constant>(Val: Init->getOperand(i));
2567 NewCtors[i] = ConstantStruct::get(T: EltTy, Vs: Ctor->getAggregateElement(Elt: 0u),
2568 Vs: Ctor->getAggregateElement(Elt: 1),
2569 Vs: ConstantPointerNull::get(T: IRB.getPtrTy()));
2570 }
2571 Constant *NewInit = ConstantArray::get(T: ArrayType::get(ElementType: EltTy, NumElements: N), V: NewCtors);
2572
2573 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2574 NewInit, GV->getName());
2575}
2576
2577// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2578// to byte shuffles.
2579static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2580 unsigned Shift) {
2581 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2582 unsigned NumElts = ResultTy->getNumElements() * 8;
2583
2584 // Bitcast from a 64-bit element type to a byte element type.
2585 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2586 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2587
2588 // We'll be shuffling in zeroes.
2589 Value *Res = Constant::getNullValue(Ty: VecTy);
2590
2591 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2592 // we'll just return the zero vector.
2593 if (Shift < 16) {
2594 int Idxs[64];
2595 // 256/512-bit version is split into 2/4 16-byte lanes.
2596 for (unsigned l = 0; l != NumElts; l += 16)
2597 for (unsigned i = 0; i != 16; ++i) {
2598 unsigned Idx = NumElts + i - Shift;
2599 if (Idx < NumElts)
2600 Idx -= NumElts - 16; // end of lane, switch operand.
2601 Idxs[l + i] = Idx + l;
2602 }
2603
2604 Res = Builder.CreateShuffleVector(V1: Res, V2: Op, Mask: ArrayRef(Idxs, NumElts));
2605 }
2606
2607 // Bitcast back to a 64-bit element type.
2608 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2609}
2610
2611// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2612// to byte shuffles.
2613static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2614 unsigned Shift) {
2615 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2616 unsigned NumElts = ResultTy->getNumElements() * 8;
2617
2618 // Bitcast from a 64-bit element type to a byte element type.
2619 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2620 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2621
2622 // We'll be shuffling in zeroes.
2623 Value *Res = Constant::getNullValue(Ty: VecTy);
2624
2625 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2626 // we'll just return the zero vector.
2627 if (Shift < 16) {
2628 int Idxs[64];
2629 // 256/512-bit version is split into 2/4 16-byte lanes.
2630 for (unsigned l = 0; l != NumElts; l += 16)
2631 for (unsigned i = 0; i != 16; ++i) {
2632 unsigned Idx = i + Shift;
2633 if (Idx >= 16)
2634 Idx += NumElts - 16; // end of lane, switch operand.
2635 Idxs[l + i] = Idx + l;
2636 }
2637
2638 Res = Builder.CreateShuffleVector(V1: Op, V2: Res, Mask: ArrayRef(Idxs, NumElts));
2639 }
2640
2641 // Bitcast back to a 64-bit element type.
2642 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2643}
2644
2645static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2646 unsigned NumElts) {
2647 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2648 llvm::VectorType *MaskTy = FixedVectorType::get(
2649 ElementType: Builder.getInt1Ty(), NumElts: cast<IntegerType>(Val: Mask->getType())->getBitWidth());
2650 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2651
2652 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2653 // i8 and we need to extract down to the right number of elements.
2654 if (NumElts <= 4) {
2655 int Indices[4];
2656 for (unsigned i = 0; i != NumElts; ++i)
2657 Indices[i] = i;
2658 Mask = Builder.CreateShuffleVector(V1: Mask, V2: Mask, Mask: ArrayRef(Indices, NumElts),
2659 Name: "extract");
2660 }
2661
2662 return Mask;
2663}
2664
2665static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2666 Value *Op1) {
2667 // If the mask is all ones just emit the first operation.
2668 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2669 if (C->isAllOnesValue())
2670 return Op0;
2671
2672 Mask = getX86MaskVec(Builder, Mask,
2673 NumElts: cast<FixedVectorType>(Val: Op0->getType())->getNumElements());
2674 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2675}
2676
2677static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2678 Value *Op1) {
2679 // If the mask is all ones just emit the first operation.
2680 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2681 if (C->isAllOnesValue())
2682 return Op0;
2683
2684 auto *MaskTy = FixedVectorType::get(ElementType: Builder.getInt1Ty(),
2685 NumElts: Mask->getType()->getIntegerBitWidth());
2686 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2687 Mask = Builder.CreateExtractElement(Vec: Mask, Idx: (uint64_t)0);
2688 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2689}
2690
2691// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2692// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2693// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2694static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2695 Value *Op1, Value *Shift,
2696 Value *Passthru, Value *Mask,
2697 bool IsVALIGN) {
2698 unsigned ShiftVal = cast<llvm::ConstantInt>(Val: Shift)->getZExtValue();
2699
2700 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2701 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2702 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2703 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2704
2705 // Mask the immediate for VALIGN.
2706 if (IsVALIGN)
2707 ShiftVal &= (NumElts - 1);
2708
2709 // If palignr is shifting the pair of vectors more than the size of two
2710 // lanes, emit zero.
2711 if (ShiftVal >= 32)
2712 return llvm::Constant::getNullValue(Ty: Op0->getType());
2713
2714 // If palignr is shifting the pair of input vectors more than one lane,
2715 // but less than two lanes, convert to shifting in zeroes.
2716 if (ShiftVal > 16) {
2717 ShiftVal -= 16;
2718 Op1 = Op0;
2719 Op0 = llvm::Constant::getNullValue(Ty: Op0->getType());
2720 }
2721
2722 int Indices[64];
2723 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2724 for (unsigned l = 0; l < NumElts; l += 16) {
2725 for (unsigned i = 0; i != 16; ++i) {
2726 unsigned Idx = ShiftVal + i;
2727 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2728 Idx += NumElts - 16; // End of lane, switch operand.
2729 Indices[l + i] = Idx + l;
2730 }
2731 }
2732
2733 Value *Align = Builder.CreateShuffleVector(
2734 V1: Op1, V2: Op0, Mask: ArrayRef(Indices, NumElts), Name: "palignr");
2735
2736 return emitX86Select(Builder, Mask, Op0: Align, Op1: Passthru);
2737}
2738
2739static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2740 bool ZeroMask, bool IndexForm) {
2741 Type *Ty = CI.getType();
2742 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2743 unsigned EltWidth = Ty->getScalarSizeInBits();
2744 bool IsFloat = Ty->isFPOrFPVectorTy();
2745 Intrinsic::ID IID;
2746 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2747 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2748 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2749 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2750 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2751 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2752 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2753 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2754 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2755 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2756 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2757 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2758 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2759 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2760 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2761 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2762 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2763 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2764 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2765 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2766 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2767 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2768 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2769 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2770 else if (VecWidth == 128 && EltWidth == 16)
2771 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2772 else if (VecWidth == 256 && EltWidth == 16)
2773 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2774 else if (VecWidth == 512 && EltWidth == 16)
2775 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2776 else if (VecWidth == 128 && EltWidth == 8)
2777 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2778 else if (VecWidth == 256 && EltWidth == 8)
2779 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2780 else if (VecWidth == 512 && EltWidth == 8)
2781 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2782 else
2783 llvm_unreachable("Unexpected intrinsic");
2784
2785 Value *Args[] = { CI.getArgOperand(i: 0) , CI.getArgOperand(i: 1),
2786 CI.getArgOperand(i: 2) };
2787
2788 // If this isn't index form we need to swap operand 0 and 1.
2789 if (!IndexForm)
2790 std::swap(a&: Args[0], b&: Args[1]);
2791
2792 Value *V = Builder.CreateIntrinsic(ID: IID, Args);
2793 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2794 : Builder.CreateBitCast(V: CI.getArgOperand(i: 1),
2795 DestTy: Ty);
2796 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: V, Op1: PassThru);
2797}
2798
2799static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2800 Intrinsic::ID IID) {
2801 Type *Ty = CI.getType();
2802 Value *Op0 = CI.getOperand(i_nocapture: 0);
2803 Value *Op1 = CI.getOperand(i_nocapture: 1);
2804 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1});
2805
2806 if (CI.arg_size() == 4) { // For masked intrinsics.
2807 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2808 Value *Mask = CI.getOperand(i_nocapture: 3);
2809 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2810 }
2811 return Res;
2812}
2813
2814static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2815 bool IsRotateRight) {
2816 Type *Ty = CI.getType();
2817 Value *Src = CI.getArgOperand(i: 0);
2818 Value *Amt = CI.getArgOperand(i: 1);
2819
2820 // Amount may be scalar immediate, in which case create a splat vector.
2821 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2822 // we only care about the lowest log2 bits anyway.
2823 if (Amt->getType() != Ty) {
2824 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2825 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2826 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2827 }
2828
2829 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2830 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Src, Src, Amt});
2831
2832 if (CI.arg_size() == 4) { // For masked intrinsics.
2833 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2834 Value *Mask = CI.getOperand(i_nocapture: 3);
2835 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2836 }
2837 return Res;
2838}
2839
2840static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2841 bool IsSigned) {
2842 Type *Ty = CI.getType();
2843 Value *LHS = CI.getArgOperand(i: 0);
2844 Value *RHS = CI.getArgOperand(i: 1);
2845
2846 CmpInst::Predicate Pred;
2847 switch (Imm) {
2848 case 0x0:
2849 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2850 break;
2851 case 0x1:
2852 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2853 break;
2854 case 0x2:
2855 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2856 break;
2857 case 0x3:
2858 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2859 break;
2860 case 0x4:
2861 Pred = ICmpInst::ICMP_EQ;
2862 break;
2863 case 0x5:
2864 Pred = ICmpInst::ICMP_NE;
2865 break;
2866 case 0x6:
2867 return Constant::getNullValue(Ty); // FALSE
2868 case 0x7:
2869 return Constant::getAllOnesValue(Ty); // TRUE
2870 default:
2871 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2872 }
2873
2874 Value *Cmp = Builder.CreateICmp(P: Pred, LHS, RHS);
2875 Value *Ext = Builder.CreateSExt(V: Cmp, DestTy: Ty);
2876 return Ext;
2877}
2878
2879static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2880 bool IsShiftRight, bool ZeroMask) {
2881 Type *Ty = CI.getType();
2882 Value *Op0 = CI.getArgOperand(i: 0);
2883 Value *Op1 = CI.getArgOperand(i: 1);
2884 Value *Amt = CI.getArgOperand(i: 2);
2885
2886 if (IsShiftRight)
2887 std::swap(a&: Op0, b&: Op1);
2888
2889 // Amount may be scalar immediate, in which case create a splat vector.
2890 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2891 // we only care about the lowest log2 bits anyway.
2892 if (Amt->getType() != Ty) {
2893 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2894 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2895 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2896 }
2897
2898 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2899 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1, Amt});
2900
2901 unsigned NumArgs = CI.arg_size();
2902 if (NumArgs >= 4) { // For masked intrinsics.
2903 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(i: 3) :
2904 ZeroMask ? ConstantAggregateZero::get(Ty: CI.getType()) :
2905 CI.getArgOperand(i: 0);
2906 Value *Mask = CI.getOperand(i_nocapture: NumArgs - 1);
2907 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2908 }
2909 return Res;
2910}
2911
2912static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2913 Value *Mask, bool Aligned) {
2914 const Align Alignment =
2915 Aligned
2916 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2917 : Align(1);
2918
2919 // If the mask is all ones just emit a regular store.
2920 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2921 if (C->isAllOnesValue())
2922 return Builder.CreateAlignedStore(Val: Data, Ptr, Align: Alignment);
2923
2924 // Convert the mask from an integer type to a vector of i1.
2925 unsigned NumElts = cast<FixedVectorType>(Val: Data->getType())->getNumElements();
2926 Mask = getX86MaskVec(Builder, Mask, NumElts);
2927 return Builder.CreateMaskedStore(Val: Data, Ptr, Alignment, Mask);
2928}
2929
2930static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2931 Value *Passthru, Value *Mask, bool Aligned) {
2932 Type *ValTy = Passthru->getType();
2933 const Align Alignment =
2934 Aligned
2935 ? Align(
2936 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2937 8)
2938 : Align(1);
2939
2940 // If the mask is all ones just emit a regular store.
2941 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2942 if (C->isAllOnesValue())
2943 return Builder.CreateAlignedLoad(Ty: ValTy, Ptr, Align: Alignment);
2944
2945 // Convert the mask from an integer type to a vector of i1.
2946 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
2947 Mask = getX86MaskVec(Builder, Mask, NumElts);
2948 return Builder.CreateMaskedLoad(Ty: ValTy, Ptr, Alignment, Mask, PassThru: Passthru);
2949}
2950
2951static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2952 Type *Ty = CI.getType();
2953 Value *Op0 = CI.getArgOperand(i: 0);
2954 Value *Res = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: Ty,
2955 Args: {Op0, Builder.getInt1(V: false)});
2956 if (CI.arg_size() == 3)
2957 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 2), Op0: Res, Op1: CI.getArgOperand(i: 1));
2958 return Res;
2959}
2960
2961static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2962 Type *Ty = CI.getType();
2963
2964 // Arguments have a vXi32 type so cast to vXi64.
2965 Value *LHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 0), DestTy: Ty);
2966 Value *RHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 1), DestTy: Ty);
2967
2968 if (IsSigned) {
2969 // Shift left then arithmetic shift right.
2970 Constant *ShiftAmt = ConstantInt::get(Ty, V: 32);
2971 LHS = Builder.CreateShl(LHS, RHS: ShiftAmt);
2972 LHS = Builder.CreateAShr(LHS, RHS: ShiftAmt);
2973 RHS = Builder.CreateShl(LHS: RHS, RHS: ShiftAmt);
2974 RHS = Builder.CreateAShr(LHS: RHS, RHS: ShiftAmt);
2975 } else {
2976 // Clear the upper bits.
2977 Constant *Mask = ConstantInt::get(Ty, V: 0xffffffff);
2978 LHS = Builder.CreateAnd(LHS, RHS: Mask);
2979 RHS = Builder.CreateAnd(LHS: RHS, RHS: Mask);
2980 }
2981
2982 Value *Res = Builder.CreateMul(LHS, RHS);
2983
2984 if (CI.arg_size() == 4)
2985 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Res, Op1: CI.getArgOperand(i: 2));
2986
2987 return Res;
2988}
2989
2990// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2991static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2992 Value *Mask) {
2993 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2994 if (Mask) {
2995 const auto *C = dyn_cast<Constant>(Val: Mask);
2996 if (!C || !C->isAllOnesValue())
2997 Vec = Builder.CreateAnd(LHS: Vec, RHS: getX86MaskVec(Builder, Mask, NumElts));
2998 }
2999
3000 if (NumElts < 8) {
3001 int Indices[8];
3002 for (unsigned i = 0; i != NumElts; ++i)
3003 Indices[i] = i;
3004 for (unsigned i = NumElts; i != 8; ++i)
3005 Indices[i] = NumElts + i % NumElts;
3006 Vec = Builder.CreateShuffleVector(V1: Vec,
3007 V2: Constant::getNullValue(Ty: Vec->getType()),
3008 Mask: Indices);
3009 }
3010 return Builder.CreateBitCast(V: Vec, DestTy: Builder.getIntNTy(N: std::max(a: NumElts, b: 8U)));
3011}
3012
3013static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
3014 unsigned CC, bool Signed) {
3015 Value *Op0 = CI.getArgOperand(i: 0);
3016 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
3017
3018 Value *Cmp;
3019 if (CC == 3) {
3020 Cmp = Constant::getNullValue(
3021 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
3022 } else if (CC == 7) {
3023 Cmp = Constant::getAllOnesValue(
3024 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
3025 } else {
3026 ICmpInst::Predicate Pred;
3027 switch (CC) {
3028 default: llvm_unreachable("Unknown condition code");
3029 case 0: Pred = ICmpInst::ICMP_EQ; break;
3030 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
3031 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
3032 case 4: Pred = ICmpInst::ICMP_NE; break;
3033 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
3034 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
3035 }
3036 Cmp = Builder.CreateICmp(P: Pred, LHS: Op0, RHS: CI.getArgOperand(i: 1));
3037 }
3038
3039 Value *Mask = CI.getArgOperand(i: CI.arg_size() - 1);
3040
3041 return applyX86MaskOn1BitsVec(Builder, Vec: Cmp, Mask);
3042}
3043
3044// Replace a masked intrinsic with an older unmasked intrinsic.
3045static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
3046 Intrinsic::ID IID) {
3047 Value *Rep =
3048 Builder.CreateIntrinsic(ID: IID, Args: {CI.getArgOperand(i: 0), CI.getArgOperand(i: 1)});
3049 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Rep, Op1: CI.getArgOperand(i: 2));
3050}
3051
3052static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
3053 Value* A = CI.getArgOperand(i: 0);
3054 Value* B = CI.getArgOperand(i: 1);
3055 Value* Src = CI.getArgOperand(i: 2);
3056 Value* Mask = CI.getArgOperand(i: 3);
3057
3058 Value* AndNode = Builder.CreateAnd(LHS: Mask, RHS: APInt(8, 1));
3059 Value* Cmp = Builder.CreateIsNotNull(Arg: AndNode);
3060 Value* Extract1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
3061 Value* Extract2 = Builder.CreateExtractElement(Vec: Src, Idx: (uint64_t)0);
3062 Value* Select = Builder.CreateSelect(C: Cmp, True: Extract1, False: Extract2);
3063 return Builder.CreateInsertElement(Vec: A, NewElt: Select, Idx: (uint64_t)0);
3064}
3065
3066static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
3067 Value* Op = CI.getArgOperand(i: 0);
3068 Type* ReturnOp = CI.getType();
3069 unsigned NumElts = cast<FixedVectorType>(Val: CI.getType())->getNumElements();
3070 Value *Mask = getX86MaskVec(Builder, Mask: Op, NumElts);
3071 return Builder.CreateSExt(V: Mask, DestTy: ReturnOp, Name: "vpmovm2");
3072}
3073
3074// Replace intrinsic with unmasked version and a select.
3075static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
3076 CallBase &CI, Value *&Rep) {
3077 Name = Name.substr(Start: 12); // Remove avx512.mask.
3078
3079 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3080 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3081 Intrinsic::ID IID;
3082 if (Name.starts_with(Prefix: "max.p")) {
3083 if (VecWidth == 128 && EltWidth == 32)
3084 IID = Intrinsic::x86_sse_max_ps;
3085 else if (VecWidth == 128 && EltWidth == 64)
3086 IID = Intrinsic::x86_sse2_max_pd;
3087 else if (VecWidth == 256 && EltWidth == 32)
3088 IID = Intrinsic::x86_avx_max_ps_256;
3089 else if (VecWidth == 256 && EltWidth == 64)
3090 IID = Intrinsic::x86_avx_max_pd_256;
3091 else
3092 llvm_unreachable("Unexpected intrinsic");
3093 } else if (Name.starts_with(Prefix: "min.p")) {
3094 if (VecWidth == 128 && EltWidth == 32)
3095 IID = Intrinsic::x86_sse_min_ps;
3096 else if (VecWidth == 128 && EltWidth == 64)
3097 IID = Intrinsic::x86_sse2_min_pd;
3098 else if (VecWidth == 256 && EltWidth == 32)
3099 IID = Intrinsic::x86_avx_min_ps_256;
3100 else if (VecWidth == 256 && EltWidth == 64)
3101 IID = Intrinsic::x86_avx_min_pd_256;
3102 else
3103 llvm_unreachable("Unexpected intrinsic");
3104 } else if (Name.starts_with(Prefix: "pshuf.b.")) {
3105 if (VecWidth == 128)
3106 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3107 else if (VecWidth == 256)
3108 IID = Intrinsic::x86_avx2_pshuf_b;
3109 else if (VecWidth == 512)
3110 IID = Intrinsic::x86_avx512_pshuf_b_512;
3111 else
3112 llvm_unreachable("Unexpected intrinsic");
3113 } else if (Name.starts_with(Prefix: "pmul.hr.sw.")) {
3114 if (VecWidth == 128)
3115 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3116 else if (VecWidth == 256)
3117 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3118 else if (VecWidth == 512)
3119 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3120 else
3121 llvm_unreachable("Unexpected intrinsic");
3122 } else if (Name.starts_with(Prefix: "pmulh.w")) {
3123 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3124 "Unexpected intrinsic");
3125 Rep = upgradeX86BinaryIntrinsics(Builder, CI, IID: Intrinsic::smulh);
3126 return true;
3127 } else if (Name.starts_with(Prefix: "pmulhu.w")) {
3128 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3129 "Unexpected intrinsic");
3130 Rep = upgradeX86BinaryIntrinsics(Builder, CI, IID: Intrinsic::umulh);
3131 return true;
3132 } else if (Name.starts_with(Prefix: "pmaddw.d.")) {
3133 if (VecWidth == 128)
3134 IID = Intrinsic::x86_sse2_pmadd_wd;
3135 else if (VecWidth == 256)
3136 IID = Intrinsic::x86_avx2_pmadd_wd;
3137 else if (VecWidth == 512)
3138 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3139 else
3140 llvm_unreachable("Unexpected intrinsic");
3141 } else if (Name.starts_with(Prefix: "pmaddubs.w.")) {
3142 if (VecWidth == 128)
3143 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3144 else if (VecWidth == 256)
3145 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3146 else if (VecWidth == 512)
3147 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3148 else
3149 llvm_unreachable("Unexpected intrinsic");
3150 } else if (Name.starts_with(Prefix: "packsswb.")) {
3151 if (VecWidth == 128)
3152 IID = Intrinsic::x86_sse2_packsswb_128;
3153 else if (VecWidth == 256)
3154 IID = Intrinsic::x86_avx2_packsswb;
3155 else if (VecWidth == 512)
3156 IID = Intrinsic::x86_avx512_packsswb_512;
3157 else
3158 llvm_unreachable("Unexpected intrinsic");
3159 } else if (Name.starts_with(Prefix: "packssdw.")) {
3160 if (VecWidth == 128)
3161 IID = Intrinsic::x86_sse2_packssdw_128;
3162 else if (VecWidth == 256)
3163 IID = Intrinsic::x86_avx2_packssdw;
3164 else if (VecWidth == 512)
3165 IID = Intrinsic::x86_avx512_packssdw_512;
3166 else
3167 llvm_unreachable("Unexpected intrinsic");
3168 } else if (Name.starts_with(Prefix: "packuswb.")) {
3169 if (VecWidth == 128)
3170 IID = Intrinsic::x86_sse2_packuswb_128;
3171 else if (VecWidth == 256)
3172 IID = Intrinsic::x86_avx2_packuswb;
3173 else if (VecWidth == 512)
3174 IID = Intrinsic::x86_avx512_packuswb_512;
3175 else
3176 llvm_unreachable("Unexpected intrinsic");
3177 } else if (Name.starts_with(Prefix: "packusdw.")) {
3178 if (VecWidth == 128)
3179 IID = Intrinsic::x86_sse41_packusdw;
3180 else if (VecWidth == 256)
3181 IID = Intrinsic::x86_avx2_packusdw;
3182 else if (VecWidth == 512)
3183 IID = Intrinsic::x86_avx512_packusdw_512;
3184 else
3185 llvm_unreachable("Unexpected intrinsic");
3186 } else if (Name.starts_with(Prefix: "vpermilvar.")) {
3187 if (VecWidth == 128 && EltWidth == 32)
3188 IID = Intrinsic::x86_avx_vpermilvar_ps;
3189 else if (VecWidth == 128 && EltWidth == 64)
3190 IID = Intrinsic::x86_avx_vpermilvar_pd;
3191 else if (VecWidth == 256 && EltWidth == 32)
3192 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3193 else if (VecWidth == 256 && EltWidth == 64)
3194 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3195 else if (VecWidth == 512 && EltWidth == 32)
3196 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3197 else if (VecWidth == 512 && EltWidth == 64)
3198 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3199 else
3200 llvm_unreachable("Unexpected intrinsic");
3201 } else if (Name == "cvtpd2dq.256") {
3202 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3203 } else if (Name == "cvtpd2ps.256") {
3204 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3205 } else if (Name == "cvttpd2dq.256") {
3206 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3207 } else if (Name == "cvttps2dq.128") {
3208 IID = Intrinsic::x86_sse2_cvttps2dq;
3209 } else if (Name == "cvttps2dq.256") {
3210 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3211 } else if (Name.starts_with(Prefix: "permvar.")) {
3212 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3213 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3214 IID = Intrinsic::x86_avx2_permps;
3215 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3216 IID = Intrinsic::x86_avx2_permd;
3217 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3218 IID = Intrinsic::x86_avx512_permvar_df_256;
3219 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3220 IID = Intrinsic::x86_avx512_permvar_di_256;
3221 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3222 IID = Intrinsic::x86_avx512_permvar_sf_512;
3223 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3224 IID = Intrinsic::x86_avx512_permvar_si_512;
3225 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3226 IID = Intrinsic::x86_avx512_permvar_df_512;
3227 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3228 IID = Intrinsic::x86_avx512_permvar_di_512;
3229 else if (VecWidth == 128 && EltWidth == 16)
3230 IID = Intrinsic::x86_avx512_permvar_hi_128;
3231 else if (VecWidth == 256 && EltWidth == 16)
3232 IID = Intrinsic::x86_avx512_permvar_hi_256;
3233 else if (VecWidth == 512 && EltWidth == 16)
3234 IID = Intrinsic::x86_avx512_permvar_hi_512;
3235 else if (VecWidth == 128 && EltWidth == 8)
3236 IID = Intrinsic::x86_avx512_permvar_qi_128;
3237 else if (VecWidth == 256 && EltWidth == 8)
3238 IID = Intrinsic::x86_avx512_permvar_qi_256;
3239 else if (VecWidth == 512 && EltWidth == 8)
3240 IID = Intrinsic::x86_avx512_permvar_qi_512;
3241 else
3242 llvm_unreachable("Unexpected intrinsic");
3243 } else if (Name.starts_with(Prefix: "dbpsadbw.")) {
3244 if (VecWidth == 128)
3245 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3246 else if (VecWidth == 256)
3247 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3248 else if (VecWidth == 512)
3249 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3250 else
3251 llvm_unreachable("Unexpected intrinsic");
3252 } else if (Name.starts_with(Prefix: "pmultishift.qb.")) {
3253 if (VecWidth == 128)
3254 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3255 else if (VecWidth == 256)
3256 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3257 else if (VecWidth == 512)
3258 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3259 else
3260 llvm_unreachable("Unexpected intrinsic");
3261 } else if (Name.starts_with(Prefix: "conflict.")) {
3262 if (Name[9] == 'd' && VecWidth == 128)
3263 IID = Intrinsic::x86_avx512_conflict_d_128;
3264 else if (Name[9] == 'd' && VecWidth == 256)
3265 IID = Intrinsic::x86_avx512_conflict_d_256;
3266 else if (Name[9] == 'd' && VecWidth == 512)
3267 IID = Intrinsic::x86_avx512_conflict_d_512;
3268 else if (Name[9] == 'q' && VecWidth == 128)
3269 IID = Intrinsic::x86_avx512_conflict_q_128;
3270 else if (Name[9] == 'q' && VecWidth == 256)
3271 IID = Intrinsic::x86_avx512_conflict_q_256;
3272 else if (Name[9] == 'q' && VecWidth == 512)
3273 IID = Intrinsic::x86_avx512_conflict_q_512;
3274 else
3275 llvm_unreachable("Unexpected intrinsic");
3276 } else if (Name.starts_with(Prefix: "pavg.")) {
3277 if (Name[5] == 'b' && VecWidth == 128)
3278 IID = Intrinsic::x86_sse2_pavg_b;
3279 else if (Name[5] == 'b' && VecWidth == 256)
3280 IID = Intrinsic::x86_avx2_pavg_b;
3281 else if (Name[5] == 'b' && VecWidth == 512)
3282 IID = Intrinsic::x86_avx512_pavg_b_512;
3283 else if (Name[5] == 'w' && VecWidth == 128)
3284 IID = Intrinsic::x86_sse2_pavg_w;
3285 else if (Name[5] == 'w' && VecWidth == 256)
3286 IID = Intrinsic::x86_avx2_pavg_w;
3287 else if (Name[5] == 'w' && VecWidth == 512)
3288 IID = Intrinsic::x86_avx512_pavg_w_512;
3289 else
3290 llvm_unreachable("Unexpected intrinsic");
3291 } else
3292 return false;
3293
3294 SmallVector<Value *, 4> Args(CI.args());
3295 Args.pop_back();
3296 Args.pop_back();
3297 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3298 unsigned NumArgs = CI.arg_size();
3299 Rep = emitX86Select(Builder, Mask: CI.getArgOperand(i: NumArgs - 1), Op0: Rep,
3300 Op1: CI.getArgOperand(i: NumArgs - 2));
3301 return true;
3302}
3303
3304/// Upgrade comment in call to inline asm that represents an objc retain release
3305/// marker.
3306void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3307 size_t Pos;
3308 if (AsmStr->find(s: "mov\tfp") == 0 &&
3309 AsmStr->find(s: "objc_retainAutoreleaseReturnValue") != std::string::npos &&
3310 (Pos = AsmStr->find(s: "# marker")) != std::string::npos) {
3311 AsmStr->replace(pos: Pos, n1: 1, s: ";");
3312 }
3313}
3314
3315static Value *upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI,
3316 StringRef Name,
3317 const Intrinsic::ID (&IIDs)[2][3]) {
3318 auto Result = getNVVMFPArithUpgrade(Name, IIDs);
3319 assert(Result && "unsupported nvvm.{add,mul,fma}.* intrinsic");
3320 auto [IID, RoundingMode] = *Result;
3321
3322 // The legacy bf16 variants used integer storage types for the bf16
3323 // operands; bitcast them to the floating-point types of the new
3324 // overloaded intrinsic.
3325 Type *Ty = CI->getType();
3326 const bool IsLegacyBF16Storage = Ty->getScalarType()->isIntegerTy();
3327 if (IsLegacyBF16Storage)
3328 Ty = Name.ends_with(Suffix: "bf16x2")
3329 ? FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2)
3330 : static_cast<Type *>(Builder.getBFloatTy());
3331
3332 SmallVector<Value *, 4> Args;
3333 for (Value *Arg : CI->args())
3334 Args.push_back(Elt: IsLegacyBF16Storage ? Builder.CreateBitCast(V: Arg, DestTy: Ty) : Arg);
3335 Args.push_back(Elt: Builder.getInt32(C: static_cast<int>(RoundingMode)));
3336
3337 Value *Rep = Builder.CreateIntrinsic(RetTy: Ty, ID: IID, Args);
3338 return IsLegacyBF16Storage ? Builder.CreateBitCast(V: Rep, DestTy: CI->getType()) : Rep;
3339}
3340
3341static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3342 Function *F, IRBuilder<> &Builder) {
3343 Value *Rep = nullptr;
3344
3345 if (Name == "abs.i" || Name == "abs.ll") {
3346 Value *Arg = CI->getArgOperand(i: 0);
3347 Rep = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: {Arg->getType()},
3348 Args: {Arg, Builder.getTrue()},
3349 /*FMFSource=*/nullptr, Name: "abs");
3350 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3351 Type *Ty = (Name == "abs.bf16")
3352 ? Builder.getBFloatTy()
3353 : FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2);
3354 Value *Arg = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Ty);
3355 Value *Abs = Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs, Op: Arg);
3356 Rep = Builder.CreateBitCast(V: Abs, DestTy: CI->getType());
3357 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3358 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3359 : Intrinsic::nvvm_fabs;
3360 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3361 } else if (Name.consume_front(Prefix: "add.")) {
3362 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3363 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, IIDs: NVVMFAddIIDs);
3364 } else if (Name.consume_front(Prefix: "mul.")) {
3365 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3366 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, IIDs: NVVMFMulIIDs);
3367 } else if (Name.consume_front(Prefix: "ex2.approx.")) {
3368 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3369 Intrinsic::ID IID = Name.starts_with(Prefix: "ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3370 : Intrinsic::nvvm_ex2_approx;
3371 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3372 } else if (Name.consume_front(Prefix: "fma.")) {
3373 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
3374 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, IIDs: NVVMFMAIIDs);
3375 } else if (Name.starts_with(Prefix: "atomic.load.add.f32.p") ||
3376 Name.starts_with(Prefix: "atomic.load.add.f64.p")) {
3377 Value *Ptr = CI->getArgOperand(i: 0);
3378 Value *Val = CI->getArgOperand(i: 1);
3379 Rep = Builder.CreateAtomicRMW(
3380 Op: AtomicRMWInst::FAdd, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3381 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3382 cast<Instruction>(Val: Rep)->setMetadata(
3383 KindID: LLVMContext::MD_atomic_ignore_denormal_mode,
3384 Node: MDNode::get(Context&: CI->getContext(), MDs: {}));
3385 // The default scope for atomic.load.* intrinsics is device
3386 // (= gpu scope in ptx), but the default LLVM atomic scope is
3387 // "system"
3388 } else if (Name.starts_with(Prefix: "atomic.load.inc.32.p") ||
3389 Name.starts_with(Prefix: "atomic.load.dec.32.p")) {
3390 Value *Ptr = CI->getArgOperand(i: 0);
3391 Value *Val = CI->getArgOperand(i: 1);
3392 auto Op = Name.starts_with(Prefix: "atomic.load.inc") ? AtomicRMWInst::UIncWrap
3393 : AtomicRMWInst::UDecWrap;
3394 Rep = Builder.CreateAtomicRMW(
3395 Op, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3396 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3397 // See comment above.
3398 } else if (Name.starts_with(Prefix: "atomic.") && Name.contains(Other: ".gen.")) {
3399 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3400 StringRef Op = Name.substr(Start: StringRef("atomic.").size());
3401 Value *Ptr = CI->getArgOperand(i: 0);
3402 Value *Val = CI->getArgOperand(i: 1);
3403 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3404 SSN: Op.contains(Other: ".cta.") ? "block" : "");
3405 if (Op.starts_with(Prefix: "cas.")) {
3406 Value *New = CI->getArgOperand(i: 2);
3407 Value *Pair = Builder.CreateAtomicCmpXchg(
3408 Ptr, Cmp: Val, New, Align: MaybeAlign(), SuccessOrdering: AtomicOrdering::Monotonic,
3409 FailureOrdering: AtomicOrdering::Monotonic, SSID);
3410 Rep = Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
3411 } else {
3412 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3413 // because we were actually missing those intrinsics!
3414 AtomicRMWInst::BinOp BinOp =
3415 StringSwitch<AtomicRMWInst::BinOp>(Op)
3416 .StartsWith(S: "add.gen.f", Value: AtomicRMWInst::FAdd)
3417 .StartsWith(S: "add.gen.i", Value: AtomicRMWInst::Add)
3418 .StartsWith(S: "exch.", Value: AtomicRMWInst::Xchg)
3419 .StartsWith(S: "max.", Value: AtomicRMWInst::Max)
3420 .StartsWith(S: "min.", Value: AtomicRMWInst::Min)
3421 .StartsWith(S: "inc.", Value: AtomicRMWInst::UIncWrap)
3422 .StartsWith(S: "dec.", Value: AtomicRMWInst::UDecWrap)
3423 .StartsWith(S: "and.", Value: AtomicRMWInst::And)
3424 .StartsWith(S: "or.", Value: AtomicRMWInst::Or)
3425 .StartsWith(S: "xor.", Value: AtomicRMWInst::Xor)
3426 .Default(Value: AtomicRMWInst::BAD_BINOP);
3427 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3428 "unexpected nvvm scoped atomic intrinsic");
3429 Rep = Builder.CreateAtomicRMW(Op: BinOp, Ptr, Val, Align: MaybeAlign(),
3430 Ordering: AtomicOrdering::Monotonic, SSID);
3431 if (BinOp == AtomicRMWInst::FAdd) {
3432 cast<Instruction>(Val: Rep)->setMetadata(
3433 KindID: LLVMContext::MD_atomic_ignore_denormal_mode,
3434 Node: MDNode::get(Context&: CI->getContext(), MDs: {}));
3435 }
3436 }
3437 } else if (Name == "clz.ll") {
3438 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3439 Value *Arg = CI->getArgOperand(i: 0);
3440 Value *Ctlz = Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {Arg->getType()},
3441 Args: {Arg, Builder.getFalse()},
3442 /*FMFSource=*/nullptr, Name: "ctlz");
3443 Rep = Builder.CreateTrunc(V: Ctlz, DestTy: Builder.getInt32Ty(), Name: "ctlz.trunc");
3444 } else if (Name == "popc.ll") {
3445 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3446 // i64.
3447 Value *Arg = CI->getArgOperand(i: 0);
3448 Value *Popc = Builder.CreateIntrinsic(ID: Intrinsic::ctpop, OverloadTypes: {Arg->getType()},
3449 Args: Arg, /*FMFSource=*/nullptr, Name: "ctpop");
3450 Rep = Builder.CreateTrunc(V: Popc, DestTy: Builder.getInt32Ty(), Name: "ctpop.trunc");
3451 } else if (Name == "h2f") {
3452 Value *Cast =
3453 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
3454 Rep = Builder.CreateFPExt(V: Cast, DestTy: Builder.getFloatTy());
3455 } else if (Name.consume_front(Prefix: "bitcast.") &&
3456 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3457 Name == "d2ll")) {
3458 Rep = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3459 } else if (Name == "rotate.b32") {
3460 Value *Arg = CI->getOperand(i_nocapture: 0);
3461 Value *ShiftAmt = CI->getOperand(i_nocapture: 1);
3462 Rep = Builder.CreateIntrinsic(RetTy: Builder.getInt32Ty(), ID: Intrinsic::fshl,
3463 Args: {Arg, Arg, ShiftAmt});
3464 } else if (Name == "rotate.b64") {
3465 Type *Int64Ty = Builder.getInt64Ty();
3466 Value *Arg = CI->getOperand(i_nocapture: 0);
3467 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3468 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3469 Args: {Arg, Arg, ZExtShiftAmt});
3470 } else if (Name == "rotate.right.b64") {
3471 Type *Int64Ty = Builder.getInt64Ty();
3472 Value *Arg = CI->getOperand(i_nocapture: 0);
3473 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3474 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshr,
3475 Args: {Arg, Arg, ZExtShiftAmt});
3476 } else if (Name == "swap.lo.hi.b64") {
3477 Type *Int64Ty = Builder.getInt64Ty();
3478 Value *Arg = CI->getOperand(i_nocapture: 0);
3479 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3480 Args: {Arg, Arg, Builder.getInt64(C: 32)});
3481 } else if ((Name.consume_front(Prefix: "ptr.gen.to.") &&
3482 consumeNVVMPtrAddrSpace(Name)) ||
3483 (Name.consume_front(Prefix: "ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3484 Name.starts_with(Prefix: ".to.gen"))) {
3485 Rep = Builder.CreateAddrSpaceCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3486 } else if (Name.consume_front(Prefix: "ldg.global")) {
3487 Value *Ptr = CI->getArgOperand(i: 0);
3488 Align PtrAlign = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getAlignValue();
3489 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3490 Value *ASC = Builder.CreateAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 1));
3491 Instruction *LD = Builder.CreateAlignedLoad(Ty: CI->getType(), Ptr: ASC, Align: PtrAlign);
3492 MDNode *MD = MDNode::get(Context&: Builder.getContext(), MDs: {});
3493 LD->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MD);
3494 return LD;
3495 } else if (Name == "tanh.approx.f32") {
3496 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3497 FastMathFlags FMF;
3498 FMF.setApproxFunc();
3499 Rep = Builder.CreateUnaryIntrinsic(ID: Intrinsic::tanh, Op: CI->getArgOperand(i: 0),
3500 FMFSource: FMF);
3501 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3502 Value *Arg =
3503 Name.ends_with(Suffix: '0') ? Builder.getInt32(C: 0) : CI->getArgOperand(i: 0);
3504 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3505 OverloadTypes: {}, Args: {Arg});
3506 } else if (Name == "barrier") {
3507 Rep = Builder.CreateIntrinsic(
3508 ID: Intrinsic::nvvm_barrier_cta_sync_aligned_count, OverloadTypes: {},
3509 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3510 } else if (Name == "barrier.sync") {
3511 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_all, OverloadTypes: {},
3512 Args: {CI->getArgOperand(i: 0)});
3513 } else if (Name == "barrier.sync.cnt") {
3514 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_count, OverloadTypes: {},
3515 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3516 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3517 Name == "barrier0.or") {
3518 Value *C = CI->getArgOperand(i: 0);
3519 C = Builder.CreateICmpNE(LHS: C, RHS: Builder.getInt32(C: 0));
3520
3521 Intrinsic::ID IID =
3522 StringSwitch<Intrinsic::ID>(Name)
3523 .Case(S: "barrier0.popc",
3524 Value: Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3525 .Case(S: "barrier0.and",
3526 Value: Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3527 .Case(S: "barrier0.or",
3528 Value: Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3529 Value *Bar = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {}, Args: {Builder.getInt32(C: 0), C});
3530 Rep = Builder.CreateZExt(V: Bar, DestTy: CI->getType());
3531 } else {
3532 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3533 if (IID != Intrinsic::not_intrinsic &&
3534 isLegacyNVPTXBF16IntSignature(F, IID)) {
3535 rename(GV: F);
3536 Function *NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
3537 SmallVector<Value *, 2> Args;
3538 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3539 Value *Arg = CI->getArgOperand(i: I);
3540 Type *OldType = Arg->getType();
3541 Type *NewType = NewFn->getArg(i: I)->getType();
3542 Args.push_back(
3543 Elt: (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3544 ? Builder.CreateBitCast(V: Arg, DestTy: NewType)
3545 : Arg);
3546 }
3547 Rep = Builder.CreateCall(Callee: NewFn, Args);
3548 if (F->getReturnType()->isIntegerTy())
3549 Rep = Builder.CreateBitCast(V: Rep, DestTy: F->getReturnType());
3550 }
3551 }
3552
3553 return Rep;
3554}
3555
3556static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3557 IRBuilder<> &Builder) {
3558 LLVMContext &C = F->getContext();
3559 Value *Rep = nullptr;
3560
3561 if (Name.starts_with(Prefix: "sse4a.movnt.")) {
3562 SmallVector<Metadata *, 1> Elts;
3563 Elts.push_back(
3564 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3565 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3566
3567 Value *Arg0 = CI->getArgOperand(i: 0);
3568 Value *Arg1 = CI->getArgOperand(i: 1);
3569
3570 // Nontemporal (unaligned) store of the 0'th element of the float/double
3571 // vector.
3572 Value *Extract =
3573 Builder.CreateExtractElement(Vec: Arg1, Idx: (uint64_t)0, Name: "extractelement");
3574
3575 StoreInst *SI = Builder.CreateAlignedStore(Val: Extract, Ptr: Arg0, Align: Align(1));
3576 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3577 } else if (Name.starts_with(Prefix: "avx.movnt.") ||
3578 Name.starts_with(Prefix: "avx512.storent.")) {
3579 SmallVector<Metadata *, 1> Elts;
3580 Elts.push_back(
3581 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3582 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3583
3584 Value *Arg0 = CI->getArgOperand(i: 0);
3585 Value *Arg1 = CI->getArgOperand(i: 1);
3586
3587 StoreInst *SI = Builder.CreateAlignedStore(
3588 Val: Arg1, Ptr: Arg0,
3589 Align: Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3590 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3591 } else if (Name == "sse2.storel.dq") {
3592 Value *Arg0 = CI->getArgOperand(i: 0);
3593 Value *Arg1 = CI->getArgOperand(i: 1);
3594
3595 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
3596 Value *BC0 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
3597 Value *Elt = Builder.CreateExtractElement(Vec: BC0, Idx: (uint64_t)0);
3598 Builder.CreateAlignedStore(Val: Elt, Ptr: Arg0, Align: Align(1));
3599 } else if (Name.starts_with(Prefix: "sse.storeu.") ||
3600 Name.starts_with(Prefix: "sse2.storeu.") ||
3601 Name.starts_with(Prefix: "avx.storeu.")) {
3602 Value *Arg0 = CI->getArgOperand(i: 0);
3603 Value *Arg1 = CI->getArgOperand(i: 1);
3604 Builder.CreateAlignedStore(Val: Arg1, Ptr: Arg0, Align: Align(1));
3605 } else if (Name == "avx512.mask.store.ss") {
3606 Value *Mask = Builder.CreateAnd(LHS: CI->getArgOperand(i: 2), RHS: Builder.getInt8(C: 1));
3607 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3608 Mask, Aligned: false);
3609 } else if (Name.starts_with(Prefix: "avx512.mask.store")) {
3610 // "avx512.mask.storeu." or "avx512.mask.store."
3611 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3612 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3613 Mask: CI->getArgOperand(i: 2), Aligned);
3614 } else if (Name.starts_with(Prefix: "sse2.pcmp") || Name.starts_with(Prefix: "avx2.pcmp")) {
3615 // Upgrade packed integer vector compare intrinsics to compare instructions.
3616 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3617 bool CmpEq = Name[9] == 'e';
3618 Rep = Builder.CreateICmp(P: CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3619 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
3620 Rep = Builder.CreateSExt(V: Rep, DestTy: CI->getType(), Name: "");
3621 } else if (Name.starts_with(Prefix: "avx512.broadcastm")) {
3622 Type *ExtTy = Type::getInt32Ty(C);
3623 if (CI->getOperand(i_nocapture: 0)->getType()->isIntegerTy(BitWidth: 8))
3624 ExtTy = Type::getInt64Ty(C);
3625 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3626 ExtTy->getPrimitiveSizeInBits();
3627 Rep = Builder.CreateZExt(V: CI->getArgOperand(i: 0), DestTy: ExtTy);
3628 Rep = Builder.CreateVectorSplat(NumElts, V: Rep);
3629 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3630 Value *Vec = CI->getArgOperand(i: 0);
3631 Value *Elt0 = Builder.CreateExtractElement(Vec, Idx: (uint64_t)0);
3632 Elt0 = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: Elt0->getType(), Args: Elt0);
3633 Rep = Builder.CreateInsertElement(Vec, NewElt: Elt0, Idx: (uint64_t)0);
3634 } else if (Name.starts_with(Prefix: "avx.sqrt.p") ||
3635 Name.starts_with(Prefix: "sse2.sqrt.p") ||
3636 Name.starts_with(Prefix: "sse.sqrt.p")) {
3637 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3638 Args: {CI->getArgOperand(i: 0)});
3639 } else if (Name.starts_with(Prefix: "avx512.mask.sqrt.p")) {
3640 if (CI->arg_size() == 4 &&
3641 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3642 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3643 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3644 : Intrinsic::x86_avx512_sqrt_pd_512;
3645
3646 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 3)};
3647 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3648 } else {
3649 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3650 Args: {CI->getArgOperand(i: 0)});
3651 }
3652 Rep =
3653 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3654 } else if (Name.starts_with(Prefix: "avx512.ptestm") ||
3655 Name.starts_with(Prefix: "avx512.ptestnm")) {
3656 Value *Op0 = CI->getArgOperand(i: 0);
3657 Value *Op1 = CI->getArgOperand(i: 1);
3658 Value *Mask = CI->getArgOperand(i: 2);
3659 Rep = Builder.CreateAnd(LHS: Op0, RHS: Op1);
3660 llvm::Type *Ty = Op0->getType();
3661 Value *Zero = llvm::Constant::getNullValue(Ty);
3662 ICmpInst::Predicate Pred = Name.starts_with(Prefix: "avx512.ptestm")
3663 ? ICmpInst::ICMP_NE
3664 : ICmpInst::ICMP_EQ;
3665 Rep = Builder.CreateICmp(P: Pred, LHS: Rep, RHS: Zero);
3666 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask);
3667 } else if (Name.starts_with(Prefix: "avx512.mask.pbroadcast")) {
3668 unsigned NumElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 1)->getType())
3669 ->getNumElements();
3670 Rep = Builder.CreateVectorSplat(NumElts, V: CI->getArgOperand(i: 0));
3671 Rep =
3672 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3673 } else if (Name.starts_with(Prefix: "avx512.kunpck")) {
3674 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3675 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts);
3676 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts);
3677 int Indices[64];
3678 for (unsigned i = 0; i != NumElts; ++i)
3679 Indices[i] = i;
3680
3681 // First extract half of each vector. This gives better codegen than
3682 // doing it in a single shuffle.
3683 LHS = Builder.CreateShuffleVector(V1: LHS, V2: LHS, Mask: ArrayRef(Indices, NumElts / 2));
3684 RHS = Builder.CreateShuffleVector(V1: RHS, V2: RHS, Mask: ArrayRef(Indices, NumElts / 2));
3685 // Concat the vectors.
3686 // NOTE: Operands have to be swapped to match intrinsic definition.
3687 Rep = Builder.CreateShuffleVector(V1: RHS, V2: LHS, Mask: ArrayRef(Indices, NumElts));
3688 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3689 } else if (Name == "avx512.kand.w") {
3690 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3691 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3692 Rep = Builder.CreateAnd(LHS, RHS);
3693 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3694 } else if (Name == "avx512.kandn.w") {
3695 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3696 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3697 LHS = Builder.CreateNot(V: LHS);
3698 Rep = Builder.CreateAnd(LHS, RHS);
3699 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3700 } else if (Name == "avx512.kor.w") {
3701 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3702 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3703 Rep = Builder.CreateOr(LHS, RHS);
3704 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3705 } else if (Name == "avx512.kxor.w") {
3706 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3707 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3708 Rep = Builder.CreateXor(LHS, RHS);
3709 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3710 } else if (Name == "avx512.kxnor.w") {
3711 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3712 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3713 LHS = Builder.CreateNot(V: LHS);
3714 Rep = Builder.CreateXor(LHS, RHS);
3715 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3716 } else if (Name == "avx512.knot.w") {
3717 Rep = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3718 Rep = Builder.CreateNot(V: Rep);
3719 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3720 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3721 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3722 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3723 Rep = Builder.CreateOr(LHS, RHS);
3724 Rep = Builder.CreateBitCast(V: Rep, DestTy: Builder.getInt16Ty());
3725 Value *C;
3726 if (Name[14] == 'c')
3727 C = ConstantInt::getAllOnesValue(Ty: Builder.getInt16Ty());
3728 else
3729 C = ConstantInt::getNullValue(Ty: Builder.getInt16Ty());
3730 Rep = Builder.CreateICmpEQ(LHS: Rep, RHS: C);
3731 Rep = Builder.CreateZExt(V: Rep, DestTy: Builder.getInt32Ty());
3732 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3733 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3734 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3735 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3736 Type *I32Ty = Type::getInt32Ty(C);
3737 Value *Elt0 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 0),
3738 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3739 Value *Elt1 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1),
3740 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3741 Value *EltOp;
3742 if (Name.contains(Other: ".add."))
3743 EltOp = Builder.CreateFAdd(L: Elt0, R: Elt1);
3744 else if (Name.contains(Other: ".sub."))
3745 EltOp = Builder.CreateFSub(L: Elt0, R: Elt1);
3746 else if (Name.contains(Other: ".mul."))
3747 EltOp = Builder.CreateFMul(L: Elt0, R: Elt1);
3748 else
3749 EltOp = Builder.CreateFDiv(L: Elt0, R: Elt1);
3750 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: EltOp,
3751 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3752 } else if (Name.starts_with(Prefix: "avx512.mask.pcmp")) {
3753 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3754 bool CmpEq = Name[16] == 'e';
3755 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: CmpEq ? 0 : 6, Signed: true);
3756 } else if (Name.starts_with(Prefix: "avx512.mask.vpshufbitqmb.")) {
3757 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3758 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3759 Intrinsic::ID IID;
3760 switch (VecWidth) {
3761 default:
3762 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3763 break;
3764 case 128:
3765 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3766 break;
3767 case 256:
3768 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3769 break;
3770 case 512:
3771 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3772 break;
3773 }
3774
3775 Rep =
3776 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3777 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3778 } else if (Name.starts_with(Prefix: "avx512.mask.fpclass.p")) {
3779 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3780 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3781 unsigned EltWidth = OpTy->getScalarSizeInBits();
3782 Intrinsic::ID IID;
3783 if (VecWidth == 128 && EltWidth == 32)
3784 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3785 else if (VecWidth == 256 && EltWidth == 32)
3786 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3787 else if (VecWidth == 512 && EltWidth == 32)
3788 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3789 else if (VecWidth == 128 && EltWidth == 64)
3790 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3791 else if (VecWidth == 256 && EltWidth == 64)
3792 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3793 else if (VecWidth == 512 && EltWidth == 64)
3794 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3795 else
3796 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3797
3798 Rep =
3799 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3800 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3801 } else if (Name.starts_with(Prefix: "avx512.cmp.p")) {
3802 SmallVector<Value *, 4> Args(CI->args());
3803 Type *OpTy = Args[0]->getType();
3804 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3805 unsigned EltWidth = OpTy->getScalarSizeInBits();
3806 Intrinsic::ID IID;
3807 if (VecWidth == 128 && EltWidth == 32)
3808 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3809 else if (VecWidth == 256 && EltWidth == 32)
3810 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3811 else if (VecWidth == 512 && EltWidth == 32)
3812 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3813 else if (VecWidth == 128 && EltWidth == 64)
3814 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3815 else if (VecWidth == 256 && EltWidth == 64)
3816 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3817 else if (VecWidth == 512 && EltWidth == 64)
3818 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3819 else
3820 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3821
3822 Value *Mask = Constant::getAllOnesValue(Ty: CI->getType());
3823 if (VecWidth == 512)
3824 std::swap(a&: Mask, b&: Args.back());
3825 Args.push_back(Elt: Mask);
3826
3827 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3828 } else if (Name.starts_with(Prefix: "avx512.mask.cmp.")) {
3829 // Integer compare intrinsics.
3830 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3831 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: true);
3832 } else if (Name.starts_with(Prefix: "avx512.mask.ucmp.")) {
3833 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3834 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: false);
3835 } else if (Name.starts_with(Prefix: "avx512.cvtb2mask.") ||
3836 Name.starts_with(Prefix: "avx512.cvtw2mask.") ||
3837 Name.starts_with(Prefix: "avx512.cvtd2mask.") ||
3838 Name.starts_with(Prefix: "avx512.cvtq2mask.")) {
3839 Value *Op = CI->getArgOperand(i: 0);
3840 Value *Zero = llvm::Constant::getNullValue(Ty: Op->getType());
3841 Rep = Builder.CreateICmp(P: ICmpInst::ICMP_SLT, LHS: Op, RHS: Zero);
3842 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: nullptr);
3843 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3844 Name == "ssse3.pabs.d.128" || Name.starts_with(Prefix: "avx2.pabs") ||
3845 Name.starts_with(Prefix: "avx512.mask.pabs")) {
3846 Rep = upgradeAbs(Builder, CI&: *CI);
3847 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3848 Name == "sse41.pmaxsd" || Name.starts_with(Prefix: "avx2.pmaxs") ||
3849 Name.starts_with(Prefix: "avx512.mask.pmaxs")) {
3850 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smax);
3851 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3852 Name == "sse41.pmaxud" || Name.starts_with(Prefix: "avx2.pmaxu") ||
3853 Name.starts_with(Prefix: "avx512.mask.pmaxu")) {
3854 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umax);
3855 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3856 Name == "sse41.pminsd" || Name.starts_with(Prefix: "avx2.pmins") ||
3857 Name.starts_with(Prefix: "avx512.mask.pmins")) {
3858 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smin);
3859 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3860 Name == "sse41.pminud" || Name.starts_with(Prefix: "avx2.pminu") ||
3861 Name.starts_with(Prefix: "avx512.mask.pminu")) {
3862 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umin);
3863 } else if (Name == "sse2.pmulh.w" || Name.starts_with(Prefix: "avx2.pmulh.w") ||
3864 Name.starts_with(Prefix: "avx512.pmulh.w")) {
3865 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smulh);
3866 } else if (Name == "sse2.pmulhu.w" || Name.starts_with(Prefix: "avx2.pmulhu.w") ||
3867 Name.starts_with(Prefix: "avx512.pmulhu.w")) {
3868 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umulh);
3869 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3870 Name == "avx512.pmulu.dq.512" ||
3871 Name.starts_with(Prefix: "avx512.mask.pmulu.dq.")) {
3872 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: false);
3873 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3874 Name == "avx512.pmul.dq.512" ||
3875 Name.starts_with(Prefix: "avx512.mask.pmul.dq.")) {
3876 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: true);
3877 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3878 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3879 Rep =
3880 Builder.CreateSIToFP(V: CI->getArgOperand(i: 1),
3881 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3882 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3883 } else if (Name == "avx512.cvtusi2sd") {
3884 Rep =
3885 Builder.CreateUIToFP(V: CI->getArgOperand(i: 1),
3886 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3887 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3888 } else if (Name == "sse2.cvtss2sd") {
3889 Rep = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1), Idx: (uint64_t)0);
3890 Rep = Builder.CreateFPExt(
3891 V: Rep, DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3892 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3893 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3894 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3895 Name.starts_with(Prefix: "avx512.mask.cvtdq2pd.") ||
3896 Name.starts_with(Prefix: "avx512.mask.cvtudq2pd.") ||
3897 Name.starts_with(Prefix: "avx512.mask.cvtdq2ps.") ||
3898 Name.starts_with(Prefix: "avx512.mask.cvtudq2ps.") ||
3899 Name.starts_with(Prefix: "avx512.mask.cvtqq2pd.") ||
3900 Name.starts_with(Prefix: "avx512.mask.cvtuqq2pd.") ||
3901 Name == "avx512.mask.cvtqq2ps.256" ||
3902 Name == "avx512.mask.cvtqq2ps.512" ||
3903 Name == "avx512.mask.cvtuqq2ps.256" ||
3904 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3905 Name == "avx.cvt.ps2.pd.256" ||
3906 Name == "avx512.mask.cvtps2pd.128" ||
3907 Name == "avx512.mask.cvtps2pd.256") {
3908 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3909 Rep = CI->getArgOperand(i: 0);
3910 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3911
3912 unsigned NumDstElts = DstTy->getNumElements();
3913 if (NumDstElts < SrcTy->getNumElements()) {
3914 assert(NumDstElts == 2 && "Unexpected vector size");
3915 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1});
3916 }
3917
3918 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3919 bool IsUnsigned = Name.contains(Other: "cvtu");
3920 if (IsPS2PD)
3921 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtps2pd");
3922 else if (CI->arg_size() == 4 &&
3923 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3924 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3925 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3926 : Intrinsic::x86_avx512_sitofp_round;
3927 Rep = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {DstTy, SrcTy},
3928 Args: {Rep, CI->getArgOperand(i: 3)});
3929 } else {
3930 Rep = IsUnsigned ? Builder.CreateUIToFP(V: Rep, DestTy: DstTy, Name: "cvt")
3931 : Builder.CreateSIToFP(V: Rep, DestTy: DstTy, Name: "cvt");
3932 }
3933
3934 if (CI->arg_size() >= 3)
3935 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3936 Op1: CI->getArgOperand(i: 1));
3937 } else if (Name.starts_with(Prefix: "avx512.mask.vcvtph2ps.") ||
3938 Name.starts_with(Prefix: "vcvtph2ps.")) {
3939 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3940 Rep = CI->getArgOperand(i: 0);
3941 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3942 unsigned NumDstElts = DstTy->getNumElements();
3943 if (NumDstElts != SrcTy->getNumElements()) {
3944 assert(NumDstElts == 4 && "Unexpected vector size");
3945 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1, 2, 3});
3946 }
3947 Rep = Builder.CreateBitCast(
3948 V: Rep, DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C), NumElts: NumDstElts));
3949 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtph2ps");
3950 if (CI->arg_size() >= 3)
3951 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3952 Op1: CI->getArgOperand(i: 1));
3953 } else if (Name.starts_with(Prefix: "avx512.mask.load")) {
3954 // "avx512.mask.loadu." or "avx512.mask.load."
3955 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3956 Rep = upgradeMaskedLoad(Builder, Ptr: CI->getArgOperand(i: 0), Passthru: CI->getArgOperand(i: 1),
3957 Mask: CI->getArgOperand(i: 2), Aligned);
3958 } else if (Name.starts_with(Prefix: "avx512.mask.expand.load.")) {
3959 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3960 auto *PtrTy = CI->getOperand(i_nocapture: 0)->getType();
3961 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3962 NumElts: ResultTy->getNumElements());
3963 Rep = Builder.CreateIntrinsic(
3964 ID: Intrinsic::masked_expandload, OverloadTypes: {ResultTy, PtrTy},
3965 Args: {CI->getOperand(i_nocapture: 0), MaskVec, CI->getOperand(i_nocapture: 1)});
3966 } else if (Name.starts_with(Prefix: "avx512.mask.compress.store.")) {
3967 auto *ResultTy = cast<VectorType>(Val: CI->getArgOperand(i: 1)->getType());
3968 auto *PtrTy = CI->getArgOperand(i: 0)->getType();
3969 Value *MaskVec =
3970 getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3971 NumElts: cast<FixedVectorType>(Val: ResultTy)->getNumElements());
3972 Rep = Builder.CreateIntrinsic(
3973 ID: Intrinsic::masked_compressstore, OverloadTypes: {ResultTy, PtrTy},
3974 Args: {CI->getArgOperand(i: 1), CI->getArgOperand(i: 0), MaskVec});
3975 } else if (Name.starts_with(Prefix: "avx512.mask.compress.") ||
3976 Name.starts_with(Prefix: "avx512.mask.expand.")) {
3977 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3978
3979 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3980 NumElts: ResultTy->getNumElements());
3981
3982 bool IsCompress = Name[12] == 'c';
3983 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3984 : Intrinsic::x86_avx512_mask_expand;
3985 Rep = Builder.CreateIntrinsic(
3986 ID: IID, OverloadTypes: ResultTy, Args: {CI->getOperand(i_nocapture: 0), CI->getOperand(i_nocapture: 1), MaskVec});
3987 } else if (Name.starts_with(Prefix: "xop.vpcom")) {
3988 bool IsSigned;
3989 if (Name.ends_with(Suffix: "ub") || Name.ends_with(Suffix: "uw") || Name.ends_with(Suffix: "ud") ||
3990 Name.ends_with(Suffix: "uq"))
3991 IsSigned = false;
3992 else if (Name.ends_with(Suffix: "b") || Name.ends_with(Suffix: "w") ||
3993 Name.ends_with(Suffix: "d") || Name.ends_with(Suffix: "q"))
3994 IsSigned = true;
3995 else
3996 reportFatalUsageErrorWithCI(reason: "Intrinsic has unknown suffix", CI);
3997
3998 unsigned Imm;
3999 if (CI->arg_size() == 3) {
4000 Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4001 } else {
4002 Name = Name.substr(Start: 9); // strip off "xop.vpcom"
4003 if (Name.starts_with(Prefix: "lt"))
4004 Imm = 0;
4005 else if (Name.starts_with(Prefix: "le"))
4006 Imm = 1;
4007 else if (Name.starts_with(Prefix: "gt"))
4008 Imm = 2;
4009 else if (Name.starts_with(Prefix: "ge"))
4010 Imm = 3;
4011 else if (Name.starts_with(Prefix: "eq"))
4012 Imm = 4;
4013 else if (Name.starts_with(Prefix: "ne"))
4014 Imm = 5;
4015 else if (Name.starts_with(Prefix: "false"))
4016 Imm = 6;
4017 else if (Name.starts_with(Prefix: "true"))
4018 Imm = 7;
4019 else
4020 llvm_unreachable("Unknown condition");
4021 }
4022
4023 Rep = upgradeX86vpcom(Builder, CI&: *CI, Imm, IsSigned);
4024 } else if (Name.starts_with(Prefix: "xop.vpcmov")) {
4025 Value *Sel = CI->getArgOperand(i: 2);
4026 Value *NotSel = Builder.CreateNot(V: Sel);
4027 Value *Sel0 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 0), RHS: Sel);
4028 Value *Sel1 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 1), RHS: NotSel);
4029 Rep = Builder.CreateOr(LHS: Sel0, RHS: Sel1);
4030 } else if (Name.starts_with(Prefix: "xop.vprot") || Name.starts_with(Prefix: "avx512.prol") ||
4031 Name.starts_with(Prefix: "avx512.mask.prol")) {
4032 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: false);
4033 } else if (Name.starts_with(Prefix: "avx512.pror") ||
4034 Name.starts_with(Prefix: "avx512.mask.pror")) {
4035 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: true);
4036 } else if (Name.starts_with(Prefix: "avx512.vpshld.") ||
4037 Name.starts_with(Prefix: "avx512.mask.vpshld") ||
4038 Name.starts_with(Prefix: "avx512.maskz.vpshld")) {
4039 bool ZeroMask = Name[11] == 'z';
4040 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: false, ZeroMask);
4041 } else if (Name.starts_with(Prefix: "avx512.vpshrd.") ||
4042 Name.starts_with(Prefix: "avx512.mask.vpshrd") ||
4043 Name.starts_with(Prefix: "avx512.maskz.vpshrd")) {
4044 bool ZeroMask = Name[11] == 'z';
4045 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: true, ZeroMask);
4046 } else if (Name == "sse42.crc32.64.8") {
4047 Value *Trunc0 =
4048 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Type::getInt32Ty(C));
4049 Rep = Builder.CreateIntrinsic(ID: Intrinsic::x86_sse42_crc32_32_8,
4050 Args: {Trunc0, CI->getArgOperand(i: 1)});
4051 Rep = Builder.CreateZExt(V: Rep, DestTy: CI->getType(), Name: "");
4052 } else if (Name.starts_with(Prefix: "avx.vbroadcast.s") ||
4053 Name.starts_with(Prefix: "avx512.vbroadcast.s")) {
4054 // Replace broadcasts with a series of insertelements.
4055 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4056 Type *EltTy = VecTy->getElementType();
4057 unsigned EltNum = VecTy->getNumElements();
4058 Value *Load = Builder.CreateLoad(Ty: EltTy, Ptr: CI->getArgOperand(i: 0));
4059 Type *I32Ty = Type::getInt32Ty(C);
4060 Rep = PoisonValue::get(T: VecTy);
4061 for (unsigned I = 0; I < EltNum; ++I)
4062 Rep = Builder.CreateInsertElement(Vec: Rep, NewElt: Load, Idx: ConstantInt::get(Ty: I32Ty, V: I));
4063 } else if (Name.starts_with(Prefix: "sse41.pmovsx") ||
4064 Name.starts_with(Prefix: "sse41.pmovzx") ||
4065 Name.starts_with(Prefix: "avx2.pmovsx") ||
4066 Name.starts_with(Prefix: "avx2.pmovzx") ||
4067 Name.starts_with(Prefix: "avx512.mask.pmovsx") ||
4068 Name.starts_with(Prefix: "avx512.mask.pmovzx")) {
4069 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
4070 unsigned NumDstElts = DstTy->getNumElements();
4071
4072 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
4073 SmallVector<int, 8> ShuffleMask(NumDstElts);
4074 for (unsigned i = 0; i != NumDstElts; ++i)
4075 ShuffleMask[i] = i;
4076
4077 Value *SV = Builder.CreateShuffleVector(V: CI->getArgOperand(i: 0), Mask: ShuffleMask);
4078
4079 bool DoSext = Name.contains(Other: "pmovsx");
4080 Rep =
4081 DoSext ? Builder.CreateSExt(V: SV, DestTy: DstTy) : Builder.CreateZExt(V: SV, DestTy: DstTy);
4082 // If there are 3 arguments, it's a masked intrinsic so we need a select.
4083 if (CI->arg_size() == 3)
4084 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
4085 Op1: CI->getArgOperand(i: 1));
4086 } else if (Name == "avx512.mask.pmov.qd.256" ||
4087 Name == "avx512.mask.pmov.qd.512" ||
4088 Name == "avx512.mask.pmov.wb.256" ||
4089 Name == "avx512.mask.pmov.wb.512") {
4090 Type *Ty = CI->getArgOperand(i: 1)->getType();
4091 Rep = Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Ty);
4092 Rep =
4093 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4094 } else if (Name.starts_with(Prefix: "avx.vbroadcastf128") ||
4095 Name == "avx2.vbroadcasti128") {
4096 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4097 Type *EltTy = cast<VectorType>(Val: CI->getType())->getElementType();
4098 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4099 auto *VT = FixedVectorType::get(ElementType: EltTy, NumElts: NumSrcElts);
4100 Value *Load = Builder.CreateAlignedLoad(Ty: VT, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
4101 if (NumSrcElts == 2)
4102 Rep = Builder.CreateShuffleVector(V: Load, Mask: ArrayRef<int>{0, 1, 0, 1});
4103 else
4104 Rep = Builder.CreateShuffleVector(V: Load,
4105 Mask: ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4106 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.i") ||
4107 Name.starts_with(Prefix: "avx512.mask.shuf.f")) {
4108 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4109 Type *VT = CI->getType();
4110 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4111 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4112 unsigned ControlBitsMask = NumLanes - 1;
4113 unsigned NumControlBits = NumLanes / 2;
4114 SmallVector<int, 8> ShuffleMask(0);
4115
4116 for (unsigned l = 0; l != NumLanes; ++l) {
4117 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4118 // We actually need the other source.
4119 if (l >= NumLanes / 2)
4120 LaneMask += NumLanes;
4121 for (unsigned i = 0; i != NumElementsInLane; ++i)
4122 ShuffleMask.push_back(Elt: LaneMask * NumElementsInLane + i);
4123 }
4124 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
4125 V2: CI->getArgOperand(i: 1), Mask: ShuffleMask);
4126 Rep =
4127 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
4128 } else if (Name.starts_with(Prefix: "avx512.mask.broadcastf") ||
4129 Name.starts_with(Prefix: "avx512.mask.broadcasti")) {
4130 unsigned NumSrcElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 0)->getType())
4131 ->getNumElements();
4132 unsigned NumDstElts =
4133 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4134
4135 SmallVector<int, 8> ShuffleMask(NumDstElts);
4136 for (unsigned i = 0; i != NumDstElts; ++i)
4137 ShuffleMask[i] = i % NumSrcElts;
4138
4139 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
4140 V2: CI->getArgOperand(i: 0), Mask: ShuffleMask);
4141 Rep =
4142 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4143 } else if (Name.starts_with(Prefix: "avx2.pbroadcast") ||
4144 Name.starts_with(Prefix: "avx2.vbroadcast") ||
4145 Name.starts_with(Prefix: "avx512.pbroadcast") ||
4146 Name.starts_with(Prefix: "avx512.mask.broadcast.s")) {
4147 // Replace vp?broadcasts with a vector shuffle.
4148 Value *Op = CI->getArgOperand(i: 0);
4149 ElementCount EC = cast<VectorType>(Val: CI->getType())->getElementCount();
4150 Type *MaskTy = VectorType::get(ElementType: Type::getInt32Ty(C), EC);
4151 SmallVector<int, 8> M;
4152 ShuffleVectorInst::getShuffleMask(Mask: Constant::getNullValue(Ty: MaskTy), Result&: M);
4153 Rep = Builder.CreateShuffleVector(V: Op, Mask: M);
4154
4155 if (CI->arg_size() == 3)
4156 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
4157 Op1: CI->getArgOperand(i: 1));
4158 } else if (Name.starts_with(Prefix: "sse2.padds.") ||
4159 Name.starts_with(Prefix: "avx2.padds.") ||
4160 Name.starts_with(Prefix: "avx512.padds.") ||
4161 Name.starts_with(Prefix: "avx512.mask.padds.")) {
4162 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::sadd_sat);
4163 } else if (Name.starts_with(Prefix: "sse2.psubs.") ||
4164 Name.starts_with(Prefix: "avx2.psubs.") ||
4165 Name.starts_with(Prefix: "avx512.psubs.") ||
4166 Name.starts_with(Prefix: "avx512.mask.psubs.")) {
4167 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::ssub_sat);
4168 } else if (Name.starts_with(Prefix: "sse2.paddus.") ||
4169 Name.starts_with(Prefix: "avx2.paddus.") ||
4170 Name.starts_with(Prefix: "avx512.mask.paddus.")) {
4171 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::uadd_sat);
4172 } else if (Name.starts_with(Prefix: "sse2.psubus.") ||
4173 Name.starts_with(Prefix: "avx2.psubus.") ||
4174 Name.starts_with(Prefix: "avx512.mask.psubus.")) {
4175 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::usub_sat);
4176 } else if (Name.starts_with(Prefix: "avx512.mask.palignr.")) {
4177 Rep = upgradeX86ALIGNIntrinsics(Builder, Op0: CI->getArgOperand(i: 0),
4178 Op1: CI->getArgOperand(i: 1), Shift: CI->getArgOperand(i: 2),
4179 Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4),
4180 IsVALIGN: false);
4181 } else if (Name.starts_with(Prefix: "avx512.mask.valign.")) {
4182 Rep = upgradeX86ALIGNIntrinsics(
4183 Builder, Op0: CI->getArgOperand(i: 0), Op1: CI->getArgOperand(i: 1),
4184 Shift: CI->getArgOperand(i: 2), Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4), IsVALIGN: true);
4185 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4186 // 128/256-bit shift left specified in bits.
4187 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4188 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
4189 Shift: Shift / 8); // Shift is in bits.
4190 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4191 // 128/256-bit shift right specified in bits.
4192 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4193 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
4194 Shift: Shift / 8); // Shift is in bits.
4195 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4196 Name == "avx512.psll.dq.512") {
4197 // 128/256/512-bit shift left specified in bytes.
4198 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4199 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
4200 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4201 Name == "avx512.psrl.dq.512") {
4202 // 128/256/512-bit shift right specified in bytes.
4203 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4204 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
4205 } else if (Name == "sse41.pblendw" || Name.starts_with(Prefix: "sse41.blendp") ||
4206 Name.starts_with(Prefix: "avx.blend.p") || Name == "avx2.pblendw" ||
4207 Name.starts_with(Prefix: "avx2.pblendd.")) {
4208 Value *Op0 = CI->getArgOperand(i: 0);
4209 Value *Op1 = CI->getArgOperand(i: 1);
4210 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4211 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4212 unsigned NumElts = VecTy->getNumElements();
4213
4214 SmallVector<int, 16> Idxs(NumElts);
4215 for (unsigned i = 0; i != NumElts; ++i)
4216 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4217
4218 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4219 } else if (Name.starts_with(Prefix: "avx.vinsertf128.") ||
4220 Name == "avx2.vinserti128" ||
4221 Name.starts_with(Prefix: "avx512.mask.insert")) {
4222 Value *Op0 = CI->getArgOperand(i: 0);
4223 Value *Op1 = CI->getArgOperand(i: 1);
4224 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4225 unsigned DstNumElts =
4226 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4227 unsigned SrcNumElts =
4228 cast<FixedVectorType>(Val: Op1->getType())->getNumElements();
4229 unsigned Scale = DstNumElts / SrcNumElts;
4230
4231 // Mask off the high bits of the immediate value; hardware ignores those.
4232 Imm = Imm % Scale;
4233
4234 // Extend the second operand into a vector the size of the destination.
4235 SmallVector<int, 8> Idxs(DstNumElts);
4236 for (unsigned i = 0; i != SrcNumElts; ++i)
4237 Idxs[i] = i;
4238 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4239 Idxs[i] = SrcNumElts;
4240 Rep = Builder.CreateShuffleVector(V: Op1, Mask: Idxs);
4241
4242 // Insert the second operand into the first operand.
4243
4244 // Note that there is no guarantee that instruction lowering will actually
4245 // produce a vinsertf128 instruction for the created shuffles. In
4246 // particular, the 0 immediate case involves no lane changes, so it can
4247 // be handled as a blend.
4248
4249 // Example of shuffle mask for 32-bit elements:
4250 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4251 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4252
4253 // First fill with identify mask.
4254 for (unsigned i = 0; i != DstNumElts; ++i)
4255 Idxs[i] = i;
4256 // Then replace the elements where we need to insert.
4257 for (unsigned i = 0; i != SrcNumElts; ++i)
4258 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4259 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Rep, Mask: Idxs);
4260
4261 // If the intrinsic has a mask operand, handle that.
4262 if (CI->arg_size() == 5)
4263 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep,
4264 Op1: CI->getArgOperand(i: 3));
4265 } else if (Name.starts_with(Prefix: "avx.vextractf128.") ||
4266 Name == "avx2.vextracti128" ||
4267 Name.starts_with(Prefix: "avx512.mask.vextract")) {
4268 Value *Op0 = CI->getArgOperand(i: 0);
4269 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4270 unsigned DstNumElts =
4271 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4272 unsigned SrcNumElts =
4273 cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
4274 unsigned Scale = SrcNumElts / DstNumElts;
4275
4276 // Mask off the high bits of the immediate value; hardware ignores those.
4277 Imm = Imm % Scale;
4278
4279 // Get indexes for the subvector of the input vector.
4280 SmallVector<int, 8> Idxs(DstNumElts);
4281 for (unsigned i = 0; i != DstNumElts; ++i) {
4282 Idxs[i] = i + (Imm * DstNumElts);
4283 }
4284 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4285
4286 // If the intrinsic has a mask operand, handle that.
4287 if (CI->arg_size() == 4)
4288 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4289 Op1: CI->getArgOperand(i: 2));
4290 } else if (Name.starts_with(Prefix: "avx512.mask.perm.df.") ||
4291 Name.starts_with(Prefix: "avx512.mask.perm.di.")) {
4292 Value *Op0 = CI->getArgOperand(i: 0);
4293 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4294 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4295 unsigned NumElts = VecTy->getNumElements();
4296
4297 SmallVector<int, 8> Idxs(NumElts);
4298 for (unsigned i = 0; i != NumElts; ++i)
4299 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4300
4301 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4302
4303 if (CI->arg_size() == 4)
4304 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4305 Op1: CI->getArgOperand(i: 2));
4306 } else if (Name.starts_with(Prefix: "avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4307 // The immediate permute control byte looks like this:
4308 // [1:0] - select 128 bits from sources for low half of destination
4309 // [2] - ignore
4310 // [3] - zero low half of destination
4311 // [5:4] - select 128 bits from sources for high half of destination
4312 // [6] - ignore
4313 // [7] - zero high half of destination
4314
4315 uint8_t Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4316
4317 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4318 unsigned HalfSize = NumElts / 2;
4319 SmallVector<int, 8> ShuffleMask(NumElts);
4320
4321 // Determine which operand(s) are actually in use for this instruction.
4322 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4323 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4324
4325 // If needed, replace operands based on zero mask.
4326 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(Ty: CI->getType()) : V0;
4327 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(Ty: CI->getType()) : V1;
4328
4329 // Permute low half of result.
4330 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4331 for (unsigned i = 0; i < HalfSize; ++i)
4332 ShuffleMask[i] = StartIndex + i;
4333
4334 // Permute high half of result.
4335 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4336 for (unsigned i = 0; i < HalfSize; ++i)
4337 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4338
4339 Rep = Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
4340
4341 } else if (Name.starts_with(Prefix: "avx.vpermil.") || Name == "sse2.pshuf.d" ||
4342 Name.starts_with(Prefix: "avx512.mask.vpermil.p") ||
4343 Name.starts_with(Prefix: "avx512.mask.pshuf.d.")) {
4344 Value *Op0 = CI->getArgOperand(i: 0);
4345 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4346 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4347 unsigned NumElts = VecTy->getNumElements();
4348 // Calculate the size of each index in the immediate.
4349 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4350 unsigned IdxMask = ((1 << IdxSize) - 1);
4351
4352 SmallVector<int, 8> Idxs(NumElts);
4353 // Lookup the bits for this element, wrapping around the immediate every
4354 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4355 // to offset by the first index of each group.
4356 for (unsigned i = 0; i != NumElts; ++i)
4357 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4358
4359 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4360
4361 if (CI->arg_size() == 4)
4362 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4363 Op1: CI->getArgOperand(i: 2));
4364 } else if (Name == "sse2.pshufl.w" ||
4365 Name.starts_with(Prefix: "avx512.mask.pshufl.w.")) {
4366 Value *Op0 = CI->getArgOperand(i: 0);
4367 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4368 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4369
4370 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4371 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4372
4373 SmallVector<int, 16> Idxs(NumElts);
4374 for (unsigned l = 0; l != NumElts; l += 8) {
4375 for (unsigned i = 0; i != 4; ++i)
4376 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4377 for (unsigned i = 4; i != 8; ++i)
4378 Idxs[i + l] = i + l;
4379 }
4380
4381 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4382
4383 if (CI->arg_size() == 4)
4384 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4385 Op1: CI->getArgOperand(i: 2));
4386 } else if (Name == "sse2.pshufh.w" ||
4387 Name.starts_with(Prefix: "avx512.mask.pshufh.w.")) {
4388 Value *Op0 = CI->getArgOperand(i: 0);
4389 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4390 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4391
4392 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4393 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4394
4395 SmallVector<int, 16> Idxs(NumElts);
4396 for (unsigned l = 0; l != NumElts; l += 8) {
4397 for (unsigned i = 0; i != 4; ++i)
4398 Idxs[i + l] = i + l;
4399 for (unsigned i = 0; i != 4; ++i)
4400 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4401 }
4402
4403 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4404
4405 if (CI->arg_size() == 4)
4406 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4407 Op1: CI->getArgOperand(i: 2));
4408 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.p")) {
4409 Value *Op0 = CI->getArgOperand(i: 0);
4410 Value *Op1 = CI->getArgOperand(i: 1);
4411 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4412 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4413
4414 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4415 unsigned HalfLaneElts = NumLaneElts / 2;
4416
4417 SmallVector<int, 16> Idxs(NumElts);
4418 for (unsigned i = 0; i != NumElts; ++i) {
4419 // Base index is the starting element of the lane.
4420 Idxs[i] = i - (i % NumLaneElts);
4421 // If we are half way through the lane switch to the other source.
4422 if ((i % NumLaneElts) >= HalfLaneElts)
4423 Idxs[i] += NumElts;
4424 // Now select the specific element. By adding HalfLaneElts bits from
4425 // the immediate. Wrapping around the immediate every 8-bits.
4426 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4427 }
4428
4429 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4430
4431 Rep =
4432 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
4433 } else if (Name.starts_with(Prefix: "avx512.mask.movddup") ||
4434 Name.starts_with(Prefix: "avx512.mask.movshdup") ||
4435 Name.starts_with(Prefix: "avx512.mask.movsldup")) {
4436 Value *Op0 = CI->getArgOperand(i: 0);
4437 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4438 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4439
4440 unsigned Offset = 0;
4441 if (Name.starts_with(Prefix: "avx512.mask.movshdup."))
4442 Offset = 1;
4443
4444 SmallVector<int, 16> Idxs(NumElts);
4445 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4446 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4447 Idxs[i + l + 0] = i + l + Offset;
4448 Idxs[i + l + 1] = i + l + Offset;
4449 }
4450
4451 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4452
4453 Rep =
4454 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4455 } else if (Name.starts_with(Prefix: "avx512.mask.punpckl") ||
4456 Name.starts_with(Prefix: "avx512.mask.unpckl.")) {
4457 Value *Op0 = CI->getArgOperand(i: 0);
4458 Value *Op1 = CI->getArgOperand(i: 1);
4459 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4460 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4461
4462 SmallVector<int, 64> Idxs(NumElts);
4463 for (int l = 0; l != NumElts; l += NumLaneElts)
4464 for (int i = 0; i != NumLaneElts; ++i)
4465 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4466
4467 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4468
4469 Rep =
4470 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4471 } else if (Name.starts_with(Prefix: "avx512.mask.punpckh") ||
4472 Name.starts_with(Prefix: "avx512.mask.unpckh.")) {
4473 Value *Op0 = CI->getArgOperand(i: 0);
4474 Value *Op1 = CI->getArgOperand(i: 1);
4475 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4476 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4477
4478 SmallVector<int, 64> Idxs(NumElts);
4479 for (int l = 0; l != NumElts; l += NumLaneElts)
4480 for (int i = 0; i != NumLaneElts; ++i)
4481 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4482
4483 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4484
4485 Rep =
4486 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4487 } else if (Name.starts_with(Prefix: "avx512.mask.and.") ||
4488 Name.starts_with(Prefix: "avx512.mask.pand.")) {
4489 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4490 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4491 Rep = Builder.CreateAnd(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4492 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4493 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4494 Rep =
4495 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4496 } else if (Name.starts_with(Prefix: "avx512.mask.andn.") ||
4497 Name.starts_with(Prefix: "avx512.mask.pandn.")) {
4498 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4499 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4500 Rep = Builder.CreateNot(V: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy));
4501 Rep = Builder.CreateAnd(LHS: Rep,
4502 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4503 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4504 Rep =
4505 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4506 } else if (Name.starts_with(Prefix: "avx512.mask.or.") ||
4507 Name.starts_with(Prefix: "avx512.mask.por.")) {
4508 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4509 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4510 Rep = Builder.CreateOr(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4511 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4512 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4513 Rep =
4514 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4515 } else if (Name.starts_with(Prefix: "avx512.mask.xor.") ||
4516 Name.starts_with(Prefix: "avx512.mask.pxor.")) {
4517 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4518 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4519 Rep = Builder.CreateXor(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4520 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4521 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4522 Rep =
4523 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4524 } else if (Name.starts_with(Prefix: "avx512.mask.padd.")) {
4525 Rep = Builder.CreateAdd(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4526 Rep =
4527 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4528 } else if (Name.starts_with(Prefix: "avx512.mask.psub.")) {
4529 Rep = Builder.CreateSub(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4530 Rep =
4531 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4532 } else if (Name.starts_with(Prefix: "avx512.mask.pmull.")) {
4533 Rep = Builder.CreateMul(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4534 Rep =
4535 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4536 } else if (Name.starts_with(Prefix: "avx512.mask.add.p")) {
4537 if (Name.ends_with(Suffix: ".512")) {
4538 Intrinsic::ID IID;
4539 if (Name[17] == 's')
4540 IID = Intrinsic::x86_avx512_add_ps_512;
4541 else
4542 IID = Intrinsic::x86_avx512_add_pd_512;
4543
4544 Rep = Builder.CreateIntrinsic(
4545 ID: IID,
4546 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4547 } else {
4548 Rep = Builder.CreateFAdd(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4549 }
4550 Rep =
4551 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4552 } else if (Name.starts_with(Prefix: "avx512.mask.div.p")) {
4553 if (Name.ends_with(Suffix: ".512")) {
4554 Intrinsic::ID IID;
4555 if (Name[17] == 's')
4556 IID = Intrinsic::x86_avx512_div_ps_512;
4557 else
4558 IID = Intrinsic::x86_avx512_div_pd_512;
4559
4560 Rep = Builder.CreateIntrinsic(
4561 ID: IID,
4562 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4563 } else {
4564 Rep = Builder.CreateFDiv(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4565 }
4566 Rep =
4567 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4568 } else if (Name.starts_with(Prefix: "avx512.mask.mul.p")) {
4569 if (Name.ends_with(Suffix: ".512")) {
4570 Intrinsic::ID IID;
4571 if (Name[17] == 's')
4572 IID = Intrinsic::x86_avx512_mul_ps_512;
4573 else
4574 IID = Intrinsic::x86_avx512_mul_pd_512;
4575
4576 Rep = Builder.CreateIntrinsic(
4577 ID: IID,
4578 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4579 } else {
4580 Rep = Builder.CreateFMul(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4581 }
4582 Rep =
4583 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4584 } else if (Name.starts_with(Prefix: "avx512.mask.sub.p")) {
4585 if (Name.ends_with(Suffix: ".512")) {
4586 Intrinsic::ID IID;
4587 if (Name[17] == 's')
4588 IID = Intrinsic::x86_avx512_sub_ps_512;
4589 else
4590 IID = Intrinsic::x86_avx512_sub_pd_512;
4591
4592 Rep = Builder.CreateIntrinsic(
4593 ID: IID,
4594 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4595 } else {
4596 Rep = Builder.CreateFSub(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4597 }
4598 Rep =
4599 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4600 } else if ((Name.starts_with(Prefix: "avx512.mask.max.p") ||
4601 Name.starts_with(Prefix: "avx512.mask.min.p")) &&
4602 Name.drop_front(N: 18) == ".512") {
4603 bool IsDouble = Name[17] == 'd';
4604 bool IsMin = Name[13] == 'i';
4605 static const Intrinsic::ID MinMaxTbl[2][2] = {
4606 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4607 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4608 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4609
4610 Rep = Builder.CreateIntrinsic(
4611 ID: IID,
4612 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4613 Rep =
4614 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4615 } else if (Name.starts_with(Prefix: "avx512.mask.lzcnt.")) {
4616 Rep =
4617 Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: CI->getType(),
4618 Args: {CI->getArgOperand(i: 0), Builder.getInt1(V: false)});
4619 Rep =
4620 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4621 } else if (Name.starts_with(Prefix: "avx512.mask.psll")) {
4622 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4623 bool IsVariable = Name[16] == 'v';
4624 char Size = Name[16] == '.' ? Name[17]
4625 : Name[17] == '.' ? Name[18]
4626 : Name[18] == '.' ? Name[19]
4627 : Name[20];
4628
4629 Intrinsic::ID IID;
4630 if (IsVariable && Name[17] != '.') {
4631 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4632 IID = Intrinsic::x86_avx2_psllv_q;
4633 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4634 IID = Intrinsic::x86_avx2_psllv_q_256;
4635 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4636 IID = Intrinsic::x86_avx2_psllv_d;
4637 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4638 IID = Intrinsic::x86_avx2_psllv_d_256;
4639 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4640 IID = Intrinsic::x86_avx512_psllv_w_128;
4641 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4642 IID = Intrinsic::x86_avx512_psllv_w_256;
4643 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4644 IID = Intrinsic::x86_avx512_psllv_w_512;
4645 else
4646 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4647 } else if (Name.ends_with(Suffix: ".128")) {
4648 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4649 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4650 : Intrinsic::x86_sse2_psll_d;
4651 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4652 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4653 : Intrinsic::x86_sse2_psll_q;
4654 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4655 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4656 : Intrinsic::x86_sse2_psll_w;
4657 else
4658 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4659 } else if (Name.ends_with(Suffix: ".256")) {
4660 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4661 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4662 : Intrinsic::x86_avx2_psll_d;
4663 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4664 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4665 : Intrinsic::x86_avx2_psll_q;
4666 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4667 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4668 : Intrinsic::x86_avx2_psll_w;
4669 else
4670 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4671 } else {
4672 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4673 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4674 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4675 : Intrinsic::x86_avx512_psll_d_512;
4676 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4677 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4678 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4679 : Intrinsic::x86_avx512_psll_q_512;
4680 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4681 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4682 : Intrinsic::x86_avx512_psll_w_512;
4683 else
4684 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4685 }
4686
4687 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4688 } else if (Name.starts_with(Prefix: "avx512.mask.psrl")) {
4689 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4690 bool IsVariable = Name[16] == 'v';
4691 char Size = Name[16] == '.' ? Name[17]
4692 : Name[17] == '.' ? Name[18]
4693 : Name[18] == '.' ? Name[19]
4694 : Name[20];
4695
4696 Intrinsic::ID IID;
4697 if (IsVariable && Name[17] != '.') {
4698 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4699 IID = Intrinsic::x86_avx2_psrlv_q;
4700 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4701 IID = Intrinsic::x86_avx2_psrlv_q_256;
4702 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4703 IID = Intrinsic::x86_avx2_psrlv_d;
4704 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4705 IID = Intrinsic::x86_avx2_psrlv_d_256;
4706 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4707 IID = Intrinsic::x86_avx512_psrlv_w_128;
4708 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4709 IID = Intrinsic::x86_avx512_psrlv_w_256;
4710 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4711 IID = Intrinsic::x86_avx512_psrlv_w_512;
4712 else
4713 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4714 } else if (Name.ends_with(Suffix: ".128")) {
4715 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4716 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4717 : Intrinsic::x86_sse2_psrl_d;
4718 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4719 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4720 : Intrinsic::x86_sse2_psrl_q;
4721 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4722 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4723 : Intrinsic::x86_sse2_psrl_w;
4724 else
4725 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4726 } else if (Name.ends_with(Suffix: ".256")) {
4727 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4728 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4729 : Intrinsic::x86_avx2_psrl_d;
4730 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4731 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4732 : Intrinsic::x86_avx2_psrl_q;
4733 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4734 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4735 : Intrinsic::x86_avx2_psrl_w;
4736 else
4737 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4738 } else {
4739 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4740 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4741 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4742 : Intrinsic::x86_avx512_psrl_d_512;
4743 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4744 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4745 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4746 : Intrinsic::x86_avx512_psrl_q_512;
4747 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4748 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4749 : Intrinsic::x86_avx512_psrl_w_512;
4750 else
4751 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4752 }
4753
4754 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4755 } else if (Name.starts_with(Prefix: "avx512.mask.psra")) {
4756 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4757 bool IsVariable = Name[16] == 'v';
4758 char Size = Name[16] == '.' ? Name[17]
4759 : Name[17] == '.' ? Name[18]
4760 : Name[18] == '.' ? Name[19]
4761 : Name[20];
4762
4763 Intrinsic::ID IID;
4764 if (IsVariable && Name[17] != '.') {
4765 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4766 IID = Intrinsic::x86_avx2_psrav_d;
4767 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4768 IID = Intrinsic::x86_avx2_psrav_d_256;
4769 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4770 IID = Intrinsic::x86_avx512_psrav_w_128;
4771 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4772 IID = Intrinsic::x86_avx512_psrav_w_256;
4773 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4774 IID = Intrinsic::x86_avx512_psrav_w_512;
4775 else
4776 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4777 } else if (Name.ends_with(Suffix: ".128")) {
4778 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4779 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4780 : Intrinsic::x86_sse2_psra_d;
4781 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4782 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4783 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4784 : Intrinsic::x86_avx512_psra_q_128;
4785 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4786 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4787 : Intrinsic::x86_sse2_psra_w;
4788 else
4789 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4790 } else if (Name.ends_with(Suffix: ".256")) {
4791 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4792 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4793 : Intrinsic::x86_avx2_psra_d;
4794 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4795 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4796 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4797 : Intrinsic::x86_avx512_psra_q_256;
4798 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4799 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4800 : Intrinsic::x86_avx2_psra_w;
4801 else
4802 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4803 } else {
4804 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4805 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4806 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4807 : Intrinsic::x86_avx512_psra_d_512;
4808 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4809 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4810 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4811 : Intrinsic::x86_avx512_psra_q_512;
4812 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4813 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4814 : Intrinsic::x86_avx512_psra_w_512;
4815 else
4816 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4817 }
4818
4819 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4820 } else if (Name.starts_with(Prefix: "avx512.mask.move.s")) {
4821 Rep = upgradeMaskedMove(Builder, CI&: *CI);
4822 } else if (Name.starts_with(Prefix: "avx512.cvtmask2")) {
4823 Rep = upgradeMaskToInt(Builder, CI&: *CI);
4824 } else if (Name.ends_with(Suffix: ".movntdqa")) {
4825 MDNode *Node = MDNode::get(
4826 Context&: C, MDs: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
4827
4828 LoadInst *LI = Builder.CreateAlignedLoad(
4829 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0),
4830 Align: Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4831 LI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
4832 Rep = LI;
4833 } else if (Name.starts_with(Prefix: "fma.vfmadd.") ||
4834 Name.starts_with(Prefix: "fma.vfmsub.") ||
4835 Name.starts_with(Prefix: "fma.vfnmadd.") ||
4836 Name.starts_with(Prefix: "fma.vfnmsub.")) {
4837 bool NegMul = Name[6] == 'n';
4838 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4839 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4840
4841 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4842 CI->getArgOperand(i: 2)};
4843
4844 if (IsScalar) {
4845 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4846 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4847 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4848 }
4849
4850 if (NegMul && !IsScalar)
4851 Ops[0] = Builder.CreateFNeg(V: Ops[0]);
4852 if (NegMul && IsScalar)
4853 Ops[1] = Builder.CreateFNeg(V: Ops[1]);
4854 if (NegAcc)
4855 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4856
4857 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4858
4859 if (IsScalar)
4860 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
4861 } else if (Name.starts_with(Prefix: "fma4.vfmadd.s")) {
4862 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4863 CI->getArgOperand(i: 2)};
4864
4865 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4866 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4867 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4868
4869 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4870
4871 Rep = Builder.CreateInsertElement(Vec: Constant::getNullValue(Ty: CI->getType()),
4872 NewElt: Rep, Idx: (uint64_t)0);
4873 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.s") ||
4874 Name.starts_with(Prefix: "avx512.maskz.vfmadd.s") ||
4875 Name.starts_with(Prefix: "avx512.mask3.vfmadd.s") ||
4876 Name.starts_with(Prefix: "avx512.mask3.vfmsub.s") ||
4877 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.s")) {
4878 bool IsMask3 = Name[11] == '3';
4879 bool IsMaskZ = Name[11] == 'z';
4880 // Drop the "avx512.mask." to make it easier.
4881 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4882 bool NegMul = Name[2] == 'n';
4883 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4884
4885 Value *A = CI->getArgOperand(i: 0);
4886 Value *B = CI->getArgOperand(i: 1);
4887 Value *C = CI->getArgOperand(i: 2);
4888
4889 if (NegMul && (IsMask3 || IsMaskZ))
4890 A = Builder.CreateFNeg(V: A);
4891 if (NegMul && !(IsMask3 || IsMaskZ))
4892 B = Builder.CreateFNeg(V: B);
4893 if (NegAcc)
4894 C = Builder.CreateFNeg(V: C);
4895
4896 A = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)0);
4897 B = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
4898 C = Builder.CreateExtractElement(Vec: C, Idx: (uint64_t)0);
4899
4900 if (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4901 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4) {
4902 Value *Ops[] = {A, B, C, CI->getArgOperand(i: 4)};
4903
4904 Intrinsic::ID IID;
4905 if (Name.back() == 'd')
4906 IID = Intrinsic::x86_avx512_vfmadd_f64;
4907 else
4908 IID = Intrinsic::x86_avx512_vfmadd_f32;
4909 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4910 } else {
4911 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4912 }
4913
4914 Value *PassThru = IsMaskZ ? Constant::getNullValue(Ty: Rep->getType())
4915 : IsMask3 ? C
4916 : A;
4917
4918 // For Mask3 with NegAcc, we need to create a new extractelement that
4919 // avoids the negation above.
4920 if (NegAcc && IsMask3)
4921 PassThru =
4922 Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 2), Idx: (uint64_t)0);
4923
4924 Rep = emitX86ScalarSelect(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4925 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: IsMask3 ? 2 : 0), NewElt: Rep,
4926 Idx: (uint64_t)0);
4927 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.p") ||
4928 Name.starts_with(Prefix: "avx512.mask.vfnmadd.p") ||
4929 Name.starts_with(Prefix: "avx512.mask.vfnmsub.p") ||
4930 Name.starts_with(Prefix: "avx512.mask3.vfmadd.p") ||
4931 Name.starts_with(Prefix: "avx512.mask3.vfmsub.p") ||
4932 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.p") ||
4933 Name.starts_with(Prefix: "avx512.maskz.vfmadd.p")) {
4934 bool IsMask3 = Name[11] == '3';
4935 bool IsMaskZ = Name[11] == 'z';
4936 // Drop the "avx512.mask." to make it easier.
4937 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4938 bool NegMul = Name[2] == 'n';
4939 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4940
4941 Value *A = CI->getArgOperand(i: 0);
4942 Value *B = CI->getArgOperand(i: 1);
4943 Value *C = CI->getArgOperand(i: 2);
4944
4945 if (NegMul && (IsMask3 || IsMaskZ))
4946 A = Builder.CreateFNeg(V: A);
4947 if (NegMul && !(IsMask3 || IsMaskZ))
4948 B = Builder.CreateFNeg(V: B);
4949 if (NegAcc)
4950 C = Builder.CreateFNeg(V: C);
4951
4952 if (CI->arg_size() == 5 &&
4953 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4954 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4)) {
4955 Intrinsic::ID IID;
4956 // Check the character before ".512" in string.
4957 if (Name[Name.size() - 5] == 's')
4958 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4959 else
4960 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4961
4962 Rep = Builder.CreateIntrinsic(ID: IID, Args: {A, B, C, CI->getArgOperand(i: 4)});
4963 } else {
4964 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4965 }
4966
4967 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4968 : IsMask3 ? CI->getArgOperand(i: 2)
4969 : CI->getArgOperand(i: 0);
4970
4971 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4972 } else if (Name.starts_with(Prefix: "fma.vfmsubadd.p")) {
4973 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4974 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4975 Intrinsic::ID IID;
4976 if (VecWidth == 128 && EltWidth == 32)
4977 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4978 else if (VecWidth == 256 && EltWidth == 32)
4979 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4980 else if (VecWidth == 128 && EltWidth == 64)
4981 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4982 else if (VecWidth == 256 && EltWidth == 64)
4983 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4984 else
4985 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4986
4987 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4988 CI->getArgOperand(i: 2)};
4989 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4990 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4991 } else if (Name.starts_with(Prefix: "avx512.mask.vfmaddsub.p") ||
4992 Name.starts_with(Prefix: "avx512.mask3.vfmaddsub.p") ||
4993 Name.starts_with(Prefix: "avx512.maskz.vfmaddsub.p") ||
4994 Name.starts_with(Prefix: "avx512.mask3.vfmsubadd.p")) {
4995 bool IsMask3 = Name[11] == '3';
4996 bool IsMaskZ = Name[11] == 'z';
4997 // Drop the "avx512.mask." to make it easier.
4998 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4999 bool IsSubAdd = Name[3] == 's';
5000 if (CI->arg_size() == 5) {
5001 Intrinsic::ID IID;
5002 // Check the character before ".512" in string.
5003 if (Name[Name.size() - 5] == 's')
5004 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
5005 else
5006 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
5007
5008 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5009 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
5010 if (IsSubAdd)
5011 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
5012
5013 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
5014 } else {
5015 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
5016
5017 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5018 CI->getArgOperand(i: 2)};
5019
5020 Function *FMA = Intrinsic::getOrInsertDeclaration(
5021 M: CI->getModule(), id: Intrinsic::fma, OverloadTys: Ops[0]->getType());
5022 Value *Odd = Builder.CreateCall(Callee: FMA, Args: Ops);
5023 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
5024 Value *Even = Builder.CreateCall(Callee: FMA, Args: Ops);
5025
5026 if (IsSubAdd)
5027 std::swap(a&: Even, b&: Odd);
5028
5029 SmallVector<int, 32> Idxs(NumElts);
5030 for (int i = 0; i != NumElts; ++i)
5031 Idxs[i] = i + (i % 2) * NumElts;
5032
5033 Rep = Builder.CreateShuffleVector(V1: Even, V2: Odd, Mask: Idxs);
5034 }
5035
5036 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
5037 : IsMask3 ? CI->getArgOperand(i: 2)
5038 : CI->getArgOperand(i: 0);
5039
5040 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5041 } else if (Name.starts_with(Prefix: "avx512.mask.pternlog.") ||
5042 Name.starts_with(Prefix: "avx512.maskz.pternlog.")) {
5043 bool ZeroMask = Name[11] == 'z';
5044 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5045 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
5046 Intrinsic::ID IID;
5047 if (VecWidth == 128 && EltWidth == 32)
5048 IID = Intrinsic::x86_avx512_pternlog_d_128;
5049 else if (VecWidth == 256 && EltWidth == 32)
5050 IID = Intrinsic::x86_avx512_pternlog_d_256;
5051 else if (VecWidth == 512 && EltWidth == 32)
5052 IID = Intrinsic::x86_avx512_pternlog_d_512;
5053 else if (VecWidth == 128 && EltWidth == 64)
5054 IID = Intrinsic::x86_avx512_pternlog_q_128;
5055 else if (VecWidth == 256 && EltWidth == 64)
5056 IID = Intrinsic::x86_avx512_pternlog_q_256;
5057 else if (VecWidth == 512 && EltWidth == 64)
5058 IID = Intrinsic::x86_avx512_pternlog_q_512;
5059 else
5060 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5061
5062 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5063 CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)};
5064 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5065 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5066 : CI->getArgOperand(i: 0);
5067 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: PassThru);
5068 } else if (Name.starts_with(Prefix: "avx512.mask.vpmadd52") ||
5069 Name.starts_with(Prefix: "avx512.maskz.vpmadd52")) {
5070 bool ZeroMask = Name[11] == 'z';
5071 bool High = Name[20] == 'h' || Name[21] == 'h';
5072 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5073 Intrinsic::ID IID;
5074 if (VecWidth == 128 && !High)
5075 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5076 else if (VecWidth == 256 && !High)
5077 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5078 else if (VecWidth == 512 && !High)
5079 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5080 else if (VecWidth == 128 && High)
5081 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5082 else if (VecWidth == 256 && High)
5083 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5084 else if (VecWidth == 512 && High)
5085 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5086 else
5087 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5088
5089 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5090 CI->getArgOperand(i: 2)};
5091 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5092 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5093 : CI->getArgOperand(i: 0);
5094 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5095 } else if (Name.starts_with(Prefix: "avx512.mask.vpermi2var.") ||
5096 Name.starts_with(Prefix: "avx512.mask.vpermt2var.") ||
5097 Name.starts_with(Prefix: "avx512.maskz.vpermt2var.")) {
5098 bool ZeroMask = Name[11] == 'z';
5099 bool IndexForm = Name[17] == 'i';
5100 Rep = upgradeX86VPERMT2Intrinsics(Builder, CI&: *CI, ZeroMask, IndexForm);
5101 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpbusd.") ||
5102 Name.starts_with(Prefix: "avx512.maskz.vpdpbusd.") ||
5103 Name.starts_with(Prefix: "avx512.mask.vpdpbusds.") ||
5104 Name.starts_with(Prefix: "avx512.maskz.vpdpbusds.")) {
5105 bool ZeroMask = Name[11] == 'z';
5106 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5107 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5108 Intrinsic::ID IID;
5109 if (VecWidth == 128 && !IsSaturating)
5110 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5111 else if (VecWidth == 256 && !IsSaturating)
5112 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5113 else if (VecWidth == 512 && !IsSaturating)
5114 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5115 else if (VecWidth == 128 && IsSaturating)
5116 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5117 else if (VecWidth == 256 && IsSaturating)
5118 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5119 else if (VecWidth == 512 && IsSaturating)
5120 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5121 else
5122 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5123
5124 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5125 CI->getArgOperand(i: 2)};
5126
5127 // Input arguments types were incorrectly set to vectors of i32 before but
5128 // they should be vectors of i8. Insert bit cast when encountering the old
5129 // types
5130 if (Args[1]->getType()->isVectorTy() &&
5131 cast<VectorType>(Val: Args[1]->getType())
5132 ->getElementType()
5133 ->isIntegerTy(BitWidth: 32) &&
5134 Args[2]->getType()->isVectorTy() &&
5135 cast<VectorType>(Val: Args[2]->getType())
5136 ->getElementType()
5137 ->isIntegerTy(BitWidth: 32)) {
5138 Type *NewArgType = nullptr;
5139 if (VecWidth == 128)
5140 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 16, Scalable: false);
5141 else if (VecWidth == 256)
5142 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 32, Scalable: false);
5143 else if (VecWidth == 512)
5144 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 64, Scalable: false);
5145 else
5146 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
5147 CI);
5148
5149 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
5150 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
5151 }
5152
5153 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5154 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5155 : CI->getArgOperand(i: 0);
5156 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5157 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpwssd.") ||
5158 Name.starts_with(Prefix: "avx512.maskz.vpdpwssd.") ||
5159 Name.starts_with(Prefix: "avx512.mask.vpdpwssds.") ||
5160 Name.starts_with(Prefix: "avx512.maskz.vpdpwssds.")) {
5161 bool ZeroMask = Name[11] == 'z';
5162 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5163 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5164 Intrinsic::ID IID;
5165 if (VecWidth == 128 && !IsSaturating)
5166 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5167 else if (VecWidth == 256 && !IsSaturating)
5168 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5169 else if (VecWidth == 512 && !IsSaturating)
5170 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5171 else if (VecWidth == 128 && IsSaturating)
5172 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5173 else if (VecWidth == 256 && IsSaturating)
5174 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5175 else if (VecWidth == 512 && IsSaturating)
5176 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5177 else
5178 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5179
5180 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5181 CI->getArgOperand(i: 2)};
5182
5183 // Input arguments types were incorrectly set to vectors of i32 before but
5184 // they should be vectors of i16. Insert bit cast when encountering the old
5185 // types
5186 if (Args[1]->getType()->isVectorTy() &&
5187 cast<VectorType>(Val: Args[1]->getType())
5188 ->getElementType()
5189 ->isIntegerTy(BitWidth: 32) &&
5190 Args[2]->getType()->isVectorTy() &&
5191 cast<VectorType>(Val: Args[2]->getType())
5192 ->getElementType()
5193 ->isIntegerTy(BitWidth: 32)) {
5194 Type *NewArgType = nullptr;
5195 if (VecWidth == 128)
5196 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 8, Scalable: false);
5197 else if (VecWidth == 256)
5198 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 16, Scalable: false);
5199 else if (VecWidth == 512)
5200 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 32, Scalable: false);
5201 else
5202 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
5203 CI);
5204
5205 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
5206 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
5207 }
5208
5209 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5210 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5211 : CI->getArgOperand(i: 0);
5212 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5213 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5214 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5215 Name == "subborrow.u32" || Name == "subborrow.u64") {
5216 Intrinsic::ID IID;
5217 if (Name[0] == 'a' && Name.back() == '2')
5218 IID = Intrinsic::x86_addcarry_32;
5219 else if (Name[0] == 'a' && Name.back() == '4')
5220 IID = Intrinsic::x86_addcarry_64;
5221 else if (Name[0] == 's' && Name.back() == '2')
5222 IID = Intrinsic::x86_subborrow_32;
5223 else if (Name[0] == 's' && Name.back() == '4')
5224 IID = Intrinsic::x86_subborrow_64;
5225 else
5226 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5227
5228 // Make a call with 3 operands.
5229 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5230 CI->getArgOperand(i: 2)};
5231 Value *NewCall = Builder.CreateIntrinsic(ID: IID, Args);
5232
5233 // Extract the second result and store it.
5234 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
5235 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 3), Align: Align(1));
5236 // Replace the original call result with the first result of the new call.
5237 Value *CF = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
5238
5239 CI->replaceAllUsesWith(V: CF);
5240 Rep = nullptr;
5241 } else if (Name.starts_with(Prefix: "avx512.mask.") &&
5242 upgradeAVX512MaskToSelect(Name, Builder, CI&: *CI, Rep)) {
5243 // Rep will be updated by the call in the condition.
5244 } else if (Name.starts_with(Prefix: "bmi.pdep.")) {
5245 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pdep);
5246 } else if (Name.starts_with(Prefix: "bmi.pext.")) {
5247 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pext);
5248 } else
5249 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5250
5251 return Rep;
5252}
5253
5254static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5255 Function *F, IRBuilder<> &Builder) {
5256 if (Name.starts_with(Prefix: "neon.bfcvt")) {
5257 if (Name.starts_with(Prefix: "neon.bfcvtn2")) {
5258 SmallVector<int, 32> LoMask(4);
5259 std::iota(first: LoMask.begin(), last: LoMask.end(), value: 0);
5260 SmallVector<int, 32> ConcatMask(8);
5261 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
5262 Value *Inactive = Builder.CreateShuffleVector(V: CI->getOperand(i_nocapture: 0), Mask: LoMask);
5263 Value *Trunc =
5264 Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 1), DestTy: Inactive->getType());
5265 return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
5266 } else if (Name.starts_with(Prefix: "neon.bfcvtn")) {
5267 SmallVector<int, 32> ConcatMask(8);
5268 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
5269 Type *V4BF16 =
5270 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: F->getContext()), NumElts: 4);
5271 Value *Trunc = Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0), DestTy: V4BF16);
5272 dbgs() << "Trunc: " << *Trunc << "\n";
5273 return Builder.CreateShuffleVector(
5274 V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
5275 } else {
5276 return Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0),
5277 DestTy: Type::getBFloatTy(C&: F->getContext()));
5278 }
5279 } else if (Name.starts_with(Prefix: "sve.fcvt")) {
5280 Intrinsic::ID NewID =
5281 StringSwitch<Intrinsic::ID>(Name)
5282 .Case(S: "sve.fcvt.bf16f32", Value: Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5283 .Case(S: "sve.fcvtnt.bf16f32",
5284 Value: Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5285 .Default(Value: Intrinsic::not_intrinsic);
5286 if (NewID == Intrinsic::not_intrinsic)
5287 llvm_unreachable("Unhandled Intrinsic!");
5288
5289 SmallVector<Value *, 3> Args(CI->args());
5290
5291 // The original intrinsics incorrectly used a predicate based on the
5292 // smallest element type rather than the largest.
5293 Type *BadPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
5294 Type *GoodPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
5295
5296 if (Args[1]->getType() != BadPredTy)
5297 llvm_unreachable("Unexpected predicate type!");
5298
5299 Args[1] = Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
5300 OverloadTypes: BadPredTy, Args: Args[1]);
5301 Args[1] = Builder.CreateIntrinsic(
5302 ID: Intrinsic::aarch64_sve_convert_from_svbool, OverloadTypes: GoodPredTy, Args: Args[1]);
5303
5304 return Builder.CreateIntrinsic(ID: NewID, Args, /*FMFSource=*/nullptr,
5305 Name: CI->getName());
5306 }
5307
5308 if (Name == "neon.vcvtfp2hf")
5309 return Builder.CreateBitCast(
5310 V: Builder.CreateFPTrunc(
5311 V: CI->getOperand(i_nocapture: 0),
5312 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
5313 DestTy: FixedVectorType::get(ElementType: Type::getInt16Ty(C&: F->getContext()), NumElts: 4));
5314 if (Name == "neon.vcvthf2fp")
5315 return Builder.CreateFPExt(
5316 V: Builder.CreateBitCast(
5317 V: CI->getOperand(i_nocapture: 0),
5318 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
5319 DestTy: FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4));
5320
5321 llvm_unreachable("Unhandled Intrinsic!");
5322}
5323
5324static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5325 IRBuilder<> &Builder) {
5326 if (Name == "mve.vctp64.old") {
5327 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5328 // correct type.
5329 Value *VCTP = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vctp64, OverloadTypes: {},
5330 Args: CI->getArgOperand(i: 0),
5331 /*FMFSource=*/nullptr, Name: CI->getName());
5332 Value *C1 = Builder.CreateIntrinsic(
5333 ID: Intrinsic::arm_mve_pred_v2i,
5334 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 2, Scalable: false)}, Args: VCTP);
5335 return Builder.CreateIntrinsic(
5336 ID: Intrinsic::arm_mve_pred_i2v,
5337 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: C1);
5338 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5339 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5340 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5341 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5342 Name ==
5343 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5344 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5345 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5346 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5347 Name ==
5348 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5349 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5350 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5351 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5352 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5353 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5354 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5355 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5356 std::vector<Type *> Tys;
5357 unsigned ID = CI->getIntrinsicID();
5358 Type *V2I1Ty = FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts: 2);
5359 switch (ID) {
5360 case Intrinsic::arm_mve_mull_int_predicated:
5361 case Intrinsic::arm_mve_vqdmull_predicated:
5362 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5363 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(), V2I1Ty};
5364 break;
5365 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5366 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5367 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5368 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5369 V2I1Ty};
5370 break;
5371 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5372 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5373 CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5374 break;
5375 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5376 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 1)->getType(),
5377 CI->getOperand(i_nocapture: 2)->getType(), V2I1Ty};
5378 break;
5379 case Intrinsic::arm_cde_vcx1q_predicated:
5380 case Intrinsic::arm_cde_vcx1qa_predicated:
5381 case Intrinsic::arm_cde_vcx2q_predicated:
5382 case Intrinsic::arm_cde_vcx2qa_predicated:
5383 case Intrinsic::arm_cde_vcx3q_predicated:
5384 case Intrinsic::arm_cde_vcx3qa_predicated:
5385 Tys = {CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5386 break;
5387 default:
5388 llvm_unreachable("Unhandled Intrinsic!");
5389 }
5390
5391 std::vector<Value *> Ops;
5392 for (Value *Op : CI->args()) {
5393 Type *Ty = Op->getType();
5394 if (Ty->getScalarSizeInBits() == 1) {
5395 Value *C1 = Builder.CreateIntrinsic(
5396 ID: Intrinsic::arm_mve_pred_v2i,
5397 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: Op);
5398 Op = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_pred_i2v, OverloadTypes: {V2I1Ty}, Args: C1);
5399 }
5400 Ops.push_back(x: Op);
5401 }
5402
5403 return Builder.CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops, /*FMFSource=*/nullptr,
5404 Name: CI->getName());
5405 }
5406 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5407}
5408
5409// These are expected to have the arguments:
5410// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5411//
5412// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5413//
5414static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5415 Function *F, IRBuilder<> &Builder) {
5416 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5417 // for compatibility.
5418 auto UpgradeLegacyWMMAIUIntrinsicCall =
5419 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5420 ArrayRef<Type *> OverloadTys) -> Value * {
5421 // Prepare arguments, append clamp=0 for compatibility
5422 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5423 Args.push_back(Elt: Builder.getFalse());
5424
5425 // Insert the declaration for the right overload types
5426 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5427 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys);
5428
5429 // Copy operand bundles if any
5430 SmallVector<OperandBundleDef, 1> Bundles;
5431 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5432
5433 // Create the new call and copy calling properties
5434 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5435 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5436 NewCall->setCallingConv(CI->getCallingConv());
5437 NewCall->setAttributes(CI->getAttributes());
5438 NewCall->copyMetadata(SrcInst: *CI);
5439 return NewCall;
5440 };
5441
5442 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5443 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5444 "intrinsic should have 7 arguments");
5445 Type *T1 = CI->getArgOperand(i: 4)->getType();
5446 Type *T2 = CI->getArgOperand(i: 1)->getType();
5447 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5448 }
5449 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5450 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5451 "intrinsic should have 8 arguments");
5452 Type *T1 = CI->getArgOperand(i: 4)->getType();
5453 Type *T2 = CI->getArgOperand(i: 1)->getType();
5454 Type *T3 = CI->getArgOperand(i: 3)->getType();
5455 Type *T4 = CI->getArgOperand(i: 5)->getType();
5456 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5457 }
5458
5459 switch (F->getIntrinsicID()) {
5460 default:
5461 break;
5462 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5463 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5464 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5465 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5466 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5467 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5468 // Drop src0 and src1 modifiers.
5469 const Value *Op0 = CI->getArgOperand(i: 0);
5470 const Value *Op2 = CI->getArgOperand(i: 2);
5471 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5472 const ConstantInt *ModA = dyn_cast<ConstantInt>(Val: Op0);
5473 const ConstantInt *ModB = dyn_cast<ConstantInt>(Val: Op2);
5474 if (!ModA->isZero() || !ModB->isZero())
5475 reportFatalUsageError(reason: Name + " matrix A and B modifiers shall be zero");
5476
5477 SmallVector<Value *, 8> Args{CI->getArgOperand(i: 1), CI->getArgOperand(i: 3)};
5478 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5479 Args.push_back(Elt: CI->getArgOperand(i: I));
5480
5481 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5482 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5483 Overloads.push_back(Elt: Args[3]->getType());
5484 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5485 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys: Overloads);
5486
5487 SmallVector<OperandBundleDef, 1> Bundles;
5488 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5489
5490 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5491 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5492 NewCall->setCallingConv(CI->getCallingConv());
5493 NewCall->setAttributes(CI->getAttributes());
5494 NewCall->copyMetadata(SrcInst: *CI);
5495 NewCall->takeName(V: CI);
5496 return NewCall;
5497 }
5498 }
5499
5500 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
5501 Value *LHS = CI->getArgOperand(i: 0);
5502 Value *RHS = CI->getArgOperand(i: 1);
5503 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5504 cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue());
5505 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5506 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5507 RetTy: CI->getType(), ID: Intrinsic::amdgcn_ballot, Args: Cmp);
5508 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5509 NewCall->setCallingConv(CI->getCallingConv());
5510 NewCall->copyMetadata(SrcInst: *CI);
5511 NewCall->takeName(V: CI);
5512 return NewCall;
5513 }
5514
5515 if (Name.starts_with(Prefix: "addrspacecast.nonnull")) {
5516 if (CI->getNumOperands() < 2) // Malformed bitcode.
5517 return nullptr;
5518 Value *ASC = Builder.CreateAddrSpaceCast(
5519 V: CI->getArgOperand(i: 0), DestTy: CI->getType(), Name: "", /*IsNonNull=*/true);
5520 ASC->takeName(V: CI);
5521 return ASC;
5522 }
5523
5524 AtomicRMWInst::BinOp RMWOp =
5525 StringSwitch<AtomicRMWInst::BinOp>(Name)
5526 .StartsWith(S: "ds.fadd", Value: AtomicRMWInst::FAdd)
5527 .StartsWith(S: "ds.fmin", Value: AtomicRMWInst::FMin)
5528 .StartsWith(S: "ds.fmax", Value: AtomicRMWInst::FMax)
5529 .StartsWith(S: "atomic.inc.", Value: AtomicRMWInst::UIncWrap)
5530 .StartsWith(S: "atomic.dec.", Value: AtomicRMWInst::UDecWrap)
5531 .StartsWith(S: "global.atomic.fadd", Value: AtomicRMWInst::FAdd)
5532 .StartsWith(S: "flat.atomic.fadd", Value: AtomicRMWInst::FAdd)
5533 .StartsWith(S: "global.atomic.fmin", Value: AtomicRMWInst::FMin)
5534 .StartsWith(S: "flat.atomic.fmin", Value: AtomicRMWInst::FMin)
5535 .StartsWith(S: "global.atomic.fmax", Value: AtomicRMWInst::FMax)
5536 .StartsWith(S: "flat.atomic.fmax", Value: AtomicRMWInst::FMax)
5537 .StartsWith(S: "atomic.cond.sub", Value: AtomicRMWInst::USubCond)
5538 .StartsWith(S: "atomic.csub", Value: AtomicRMWInst::USubSat);
5539
5540 unsigned NumOperands = CI->getNumOperands();
5541 if (NumOperands < 3) // Malformed bitcode.
5542 return nullptr;
5543
5544 Value *Ptr = CI->getArgOperand(i: 0);
5545 PointerType *PtrTy = dyn_cast<PointerType>(Val: Ptr->getType());
5546 if (!PtrTy) // Malformed.
5547 return nullptr;
5548
5549 Value *Val = CI->getArgOperand(i: 1);
5550 if (Val->getType() != CI->getType()) // Malformed.
5551 return nullptr;
5552
5553 ConstantInt *OrderArg = nullptr;
5554 bool IsVolatile = false;
5555
5556 // These should have 5 arguments (plus the callee). A separate version of the
5557 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5558 if (NumOperands > 3)
5559 OrderArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5560
5561 // Ignore scope argument at 3
5562
5563 if (NumOperands > 5) {
5564 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 4));
5565 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5566 }
5567
5568 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5569 if (OrderArg && isValidAtomicOrdering(I: OrderArg->getZExtValue()))
5570 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5571 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5572 Order = AtomicOrdering::SequentiallyConsistent;
5573
5574 LLVMContext &Ctx = F->getContext();
5575
5576 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5577 Type *RetTy = CI->getType();
5578 if (VectorType *VT = dyn_cast<VectorType>(Val: RetTy)) {
5579 if (VT->getElementType()->isIntegerTy(BitWidth: 16)) {
5580 VectorType *AsBF16 =
5581 VectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), EC: VT->getElementCount());
5582 Val = Builder.CreateBitCast(V: Val, DestTy: AsBF16);
5583 }
5584 }
5585
5586 // The scope argument never really worked correctly. Use agent as the most
5587 // conservative option which should still always produce the instruction.
5588 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID(SSN: "agent");
5589 AtomicRMWInst *RMW =
5590 Builder.CreateAtomicRMW(Op: RMWOp, Ptr, Val, Align: std::nullopt, Ordering: Order, SSID);
5591
5592 unsigned AddrSpace = PtrTy->getAddressSpace();
5593 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5594 MDNode *EmptyMD = MDNode::get(Context&: F->getContext(), MDs: {});
5595 RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: EmptyMD);
5596 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5597 RMW->setMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode, Node: EmptyMD);
5598 }
5599
5600 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5601 MDBuilder MDB(F->getContext());
5602 MDNode *RangeNotPrivate =
5603 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5604 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5605 RMW->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: RangeNotPrivate);
5606 }
5607
5608 if (IsVolatile)
5609 RMW->setVolatile(true);
5610
5611 return Builder.CreateBitCast(V: RMW, DestTy: RetTy);
5612}
5613
5614/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5615/// plain MDNode, as it's the verifier's job to check these are the correct
5616/// types later.
5617static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5618 if (Op < CI->arg_size()) {
5619 if (MetadataAsValue *MAV =
5620 dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op))) {
5621 Metadata *MD = MAV->getMetadata();
5622 return dyn_cast_if_present<MDNode>(Val: MD);
5623 }
5624 }
5625 return nullptr;
5626}
5627
5628/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5629static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5630 if (Op < CI->arg_size())
5631 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op)))
5632 return MAV->getMetadata();
5633 return nullptr;
5634}
5635
5636/// Convert debug intrinsic calls to non-instruction debug records.
5637/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5638/// \p CI - The debug intrinsic call.
5639static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5640 DbgRecord *DR = nullptr;
5641 if (Name == "label") {
5642 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(Label: unwrapMAVOp(CI, Op: 0));
5643 } else if (Name == "assign") {
5644 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5645 Type: DbgVariableRecord::LocationType::Assign, Val: unwrapMAVMetadataOp(CI, Op: 0),
5646 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: unwrapMAVOp(CI, Op: 3),
5647 Address: unwrapMAVMetadataOp(CI, Op: 4),
5648 /*The address is a Value ref, it will be stored as a Metadata */
5649 AddressExpression: unwrapMAVOp(CI, Op: 5));
5650 } else if (Name == "declare") {
5651 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5652 Type: DbgVariableRecord::LocationType::Declare, Val: unwrapMAVMetadataOp(CI, Op: 0),
5653 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5654 } else if (Name == "addr") {
5655 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5656 MDNode *ExprNode = unwrapMAVOp(CI, Op: 2);
5657 // Don't try to add something to the expression if it's not an expression.
5658 // Instead, allow the verifier to fail later.
5659 if (DIExpression *Expr = dyn_cast<DIExpression>(Val: ExprNode)) {
5660 ExprNode = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
5661 }
5662 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5663 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5664 Variable: unwrapMAVOp(CI, Op: 1), Expression: ExprNode, AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5665 } else if (Name == "value") {
5666 // An old version of dbg.value had an extra offset argument.
5667 unsigned VarOp = 1;
5668 unsigned ExprOp = 2;
5669 if (CI->arg_size() == 4) {
5670 auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1));
5671 // Nonzero offset dbg.values get dropped without a replacement.
5672 if (!Offset || !Offset->isNullValue())
5673 return;
5674 VarOp = 2;
5675 ExprOp = 3;
5676 }
5677 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5678 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5679 Variable: unwrapMAVOp(CI, Op: VarOp), Expression: unwrapMAVOp(CI, Op: ExprOp), AssignID: nullptr, Address: nullptr,
5680 AddressExpression: nullptr);
5681 }
5682 DR->setDebugLoc(CI->getDebugLoc());
5683 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5684 CI->getParent()->insertDbgRecordBefore(DR, Here: CI->getIterator());
5685}
5686
5687static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5688 auto *Offset = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5689 if (!Offset)
5690 reportFatalUsageError(reason: "Invalid llvm.vector.splice offset argument");
5691 int64_t OffsetVal = Offset->getSExtValue();
5692 return Builder.CreateIntrinsic(ID: OffsetVal >= 0
5693 ? Intrinsic::vector_splice_left
5694 : Intrinsic::vector_splice_right,
5695 OverloadTypes: CI->getType(),
5696 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5697 Builder.getInt32(C: std::abs(i: OffsetVal))});
5698}
5699
5700static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5701 Function *F, IRBuilder<> &Builder) {
5702 if (Name.starts_with(Prefix: "to.fp16")) {
5703 Value *Cast =
5704 Builder.CreateFPTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5705 return Builder.CreateBitCast(V: Cast, DestTy: CI->getType());
5706 }
5707
5708 if (Name.starts_with(Prefix: "from.fp16")) {
5709 Value *Cast =
5710 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5711 return Builder.CreateFPExt(V: Cast, DestTy: CI->getType());
5712 }
5713
5714 return nullptr;
5715}
5716
5717static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5718 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5719 if (!MD || !isa<MDString>(Val: MD))
5720 return ICmpInst::BAD_ICMP_PREDICATE;
5721 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5722 .Case(S: "eq", Value: ICmpInst::ICMP_EQ)
5723 .Case(S: "ne", Value: ICmpInst::ICMP_NE)
5724 .Case(S: "ugt", Value: ICmpInst::ICMP_UGT)
5725 .Case(S: "uge", Value: ICmpInst::ICMP_UGE)
5726 .Case(S: "ult", Value: ICmpInst::ICMP_ULT)
5727 .Case(S: "ule", Value: ICmpInst::ICMP_ULE)
5728 .Case(S: "sgt", Value: ICmpInst::ICMP_SGT)
5729 .Case(S: "sge", Value: ICmpInst::ICMP_SGE)
5730 .Case(S: "slt", Value: ICmpInst::ICMP_SLT)
5731 .Case(S: "sle", Value: ICmpInst::ICMP_SLE)
5732 .Default(Value: ICmpInst::BAD_ICMP_PREDICATE);
5733}
5734
5735static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5736 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5737 if (!MD || !isa<MDString>(Val: MD))
5738 return FCmpInst::BAD_FCMP_PREDICATE;
5739 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5740 .Case(S: "oeq", Value: FCmpInst::FCMP_OEQ)
5741 .Case(S: "ogt", Value: FCmpInst::FCMP_OGT)
5742 .Case(S: "oge", Value: FCmpInst::FCMP_OGE)
5743 .Case(S: "olt", Value: FCmpInst::FCMP_OLT)
5744 .Case(S: "ole", Value: FCmpInst::FCMP_OLE)
5745 .Case(S: "one", Value: FCmpInst::FCMP_ONE)
5746 .Case(S: "ord", Value: FCmpInst::FCMP_ORD)
5747 .Case(S: "uno", Value: FCmpInst::FCMP_UNO)
5748 .Case(S: "ueq", Value: FCmpInst::FCMP_UEQ)
5749 .Case(S: "ugt", Value: FCmpInst::FCMP_UGT)
5750 .Case(S: "uge", Value: FCmpInst::FCMP_UGE)
5751 .Case(S: "ult", Value: FCmpInst::FCMP_ULT)
5752 .Case(S: "ule", Value: FCmpInst::FCMP_ULE)
5753 .Case(S: "une", Value: FCmpInst::FCMP_UNE)
5754 .Default(Value: FCmpInst::BAD_FCMP_PREDICATE);
5755}
5756
5757static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5758 IRBuilder<> &Builder) {
5759 Value *Rep;
5760 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5761 if (Opcode && Instruction::isUnaryOp(Opcode))
5762 Rep =
5763 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)Opcode, V: CI->getArgOperand(i: 0));
5764 else if (Opcode && Instruction::isBinaryOp(Opcode))
5765 Rep = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opcode,
5766 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5767 else if (Opcode && Instruction::isCast(Opcode))
5768 Rep = Builder.CreateCast(Op: (Instruction::CastOps)Opcode, V: CI->getArgOperand(i: 0),
5769 DestTy: CI->getType());
5770 else if (Opcode == Instruction::ICmp)
5771 Rep = Builder.CreateICmp(P: getVPIntPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5772 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5773 else if (Opcode == Instruction::FCmp)
5774 Rep = Builder.CreateFCmp(P: getVPFPPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5775 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5776 else if (Opcode == Instruction::Select)
5777 Rep = Builder.CreateSelect(C: CI->getArgOperand(i: 0), True: CI->getArgOperand(i: 1),
5778 False: CI->getArgOperand(i: 2));
5779 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5780 SmallVector<Value *, 2> Args(drop_end(RangeOrContainer: CI->args(), N: 2));
5781 Rep = Builder.CreateIntrinsic(RetTy: CI->getType(), ID: IntrinsicID, Args, FMFSource: {});
5782 } else
5783 llvm_unreachable("Unexpected vp intrinsic");
5784 Rep->takeName(V: CI);
5785 return Rep;
5786}
5787
5788static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5789 IRBuilder<> &Builder) {
5790 Intrinsic::ID IID = NewFn->getIntrinsicID();
5791
5792 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5793 if (Defaults.empty())
5794 return false;
5795
5796 unsigned OldArgCount = CI->arg_size();
5797 unsigned NewArgCount = NewFn->arg_size();
5798
5799 if (OldArgCount < FirstDefault)
5800 return false;
5801
5802 // More arguments than the new intrinsic accepts, cannot upgrade.
5803 if (OldArgCount > NewArgCount)
5804 return false;
5805
5806 // The call already passes the defaulted arguments explicitly; only the
5807 // callee is still the old, shorter declaration, so retarget it.
5808 if (OldArgCount == NewArgCount) {
5809 if (CI->getFunctionType() != NewFn->getFunctionType())
5810 return false;
5811 CI->setCalledFunction(NewFn);
5812 return true;
5813 }
5814
5815 // OldArgCount < NewArgCount: Fill in each missing trailing default
5816 // argument from the table.
5817 SmallVector<Value *, 8> NewArgs(CI->args());
5818
5819 FunctionType *NewFT = NewFn->getFunctionType();
5820 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5821 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5822 "missing argument outside the default range");
5823 Type *ParamTy = NewFT->getParamType(i: Idx);
5824
5825 // Only integer types are supported (i1, i8, i16, i32, i64).
5826 if (!ParamTy->isIntegerTy())
5827 return false;
5828 NewArgs.push_back(Elt: ConstantInt::get(Ty: ParamTy, V: Defaults[Idx - FirstDefault]));
5829 }
5830
5831 // Preserve operand bundles by creating the call with them.
5832 SmallVector<OperandBundleDef, 1> OpBundles;
5833 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
5834 CallInst *NewCall = Builder.CreateCall(Callee: NewFn, Args: NewArgs, OpBundles);
5835
5836 NewCall->takeName(V: CI);
5837 NewCall->setCallingConv(CI->getCallingConv());
5838 NewCall->copyMetadata(SrcInst: *CI);
5839 if (auto *OldCI = dyn_cast<CallInst>(Val: CI))
5840 NewCall->setTailCallKind(OldCI->getTailCallKind());
5841
5842 CI->replaceAllUsesWith(V: NewCall);
5843 CI->eraseFromParent();
5844 return true;
5845}
5846
5847/// Upgrade a call to an old intrinsic. All argument and return casting must be
5848/// provided to seamlessly integrate with existing context.
5849void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5850 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5851 // checks the callee's function type matches. It's likely we need to handle
5852 // type changes here.
5853 Function *F = dyn_cast<Function>(Val: CI->getCalledOperand());
5854 if (!F)
5855 return;
5856
5857 LLVMContext &C = CI->getContext();
5858 IRBuilder<> Builder(CI->getIterator());
5859 if (isa<FPMathOperator>(Val: CI))
5860 Builder.setFastMathFlags(CI->getFastMathFlags());
5861
5862 if (!NewFn) {
5863 // Get the Function's name.
5864 StringRef Name = F->getName();
5865 if (!Name.consume_front(Prefix: "llvm."))
5866 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5867
5868 bool IsX86 = Name.consume_front(Prefix: "x86.");
5869 bool IsNVVM = Name.consume_front(Prefix: "nvvm.");
5870 bool IsAArch64 = Name.consume_front(Prefix: "aarch64.");
5871 bool IsARM = Name.consume_front(Prefix: "arm.");
5872 bool IsAMDGCN = Name.consume_front(Prefix: "amdgcn.");
5873 bool IsDbg = Name.consume_front(Prefix: "dbg.");
5874 bool IsOldSplice =
5875 (Name.consume_front(Prefix: "experimental.vector.splice") ||
5876 Name.consume_front(Prefix: "vector.splice")) &&
5877 !(Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"));
5878 Value *Rep = nullptr;
5879
5880 if (!IsX86 && Name == "stackprotectorcheck") {
5881 Rep = nullptr;
5882 } else if (IsNVVM) {
5883 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5884 } else if (IsX86) {
5885 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5886 } else if (IsAArch64) {
5887 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5888 } else if (IsARM) {
5889 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5890 } else if (IsAMDGCN) {
5891 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5892 } else if (IsDbg) {
5893 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5894 } else if (IsOldSplice) {
5895 Rep = upgradeVectorSplice(CI, Builder);
5896 } else if (Name.consume_front(Prefix: "convert.")) {
5897 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5898 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5899 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5900 Rep = nullptr;
5901 } else if (shouldUpgradeVPIntrinsic(Name)) {
5902 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5903 } else {
5904 llvm_unreachable("Unknown function for CallBase upgrade.");
5905 }
5906
5907 if (Rep)
5908 CI->replaceAllUsesWith(V: Rep);
5909 CI->eraseFromParent();
5910 return;
5911 }
5912
5913 const auto &DefaultCase = [&]() -> void {
5914 if (F == NewFn)
5915 return;
5916
5917 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5918 // Handle generic mangling change.
5919 assert(
5920 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5921 "Unknown function for CallBase upgrade and isn't just a name change");
5922 CI->setCalledFunction(NewFn);
5923 return;
5924 }
5925
5926 // This must be an upgrade from a named to a literal struct.
5927 if (auto *OldST = dyn_cast<StructType>(Val: CI->getType())) {
5928 assert(OldST != NewFn->getReturnType() &&
5929 "Return type must have changed");
5930 assert(OldST->getNumElements() ==
5931 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5932 "Must have same number of elements");
5933
5934 SmallVector<Value *> Args(CI->args());
5935 CallInst *NewCI = Builder.CreateCall(Callee: NewFn, Args);
5936 NewCI->setAttributes(CI->getAttributes());
5937 Value *Res = PoisonValue::get(T: OldST);
5938 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5939 Value *Elem = Builder.CreateExtractValue(Agg: NewCI, Idxs: Idx);
5940 Res = Builder.CreateInsertValue(Agg: Res, Val: Elem, Idxs: Idx);
5941 }
5942 CI->replaceAllUsesWith(V: Res);
5943 CI->eraseFromParent();
5944 return;
5945 }
5946
5947 // We're probably about to produce something invalid. Let the verifier catch
5948 // it instead of dying here.
5949 CI->setCalledOperand(
5950 ConstantExpr::getPointerCast(C: NewFn, Ty: CI->getCalledOperand()->getType()));
5951 return;
5952 };
5953 CallInst *NewCall = nullptr;
5954 switch (NewFn->getIntrinsicID()) {
5955 default: {
5956 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5957 return;
5958 DefaultCase();
5959 return;
5960 }
5961 case Intrinsic::arm_neon_vst1:
5962 case Intrinsic::arm_neon_vst2:
5963 case Intrinsic::arm_neon_vst3:
5964 case Intrinsic::arm_neon_vst4:
5965 case Intrinsic::arm_neon_vst2lane:
5966 case Intrinsic::arm_neon_vst3lane:
5967 case Intrinsic::arm_neon_vst4lane: {
5968 SmallVector<Value *, 4> Args(CI->args());
5969 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5970 break;
5971 }
5972 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5973 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5974 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5975 LLVMContext &Ctx = F->getParent()->getContext();
5976 SmallVector<Value *, 4> Args(CI->args());
5977 Args[3] = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx),
5978 V: cast<ConstantInt>(Val: Args[3])->getZExtValue());
5979 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5980 break;
5981 }
5982 case Intrinsic::aarch64_sve_ld3_sret:
5983 case Intrinsic::aarch64_sve_ld4_sret:
5984 case Intrinsic::aarch64_sve_ld2_sret: {
5985 // Is this a trivial remangle of the name to support ptr address spaces?
5986 if (isa<StructType>(Val: F->getReturnType())) {
5987 DefaultCase();
5988 return;
5989 }
5990
5991 StringRef Name = F->getName();
5992 Name = Name.substr(Start: 5);
5993 unsigned N = StringSwitch<unsigned>(Name)
5994 .StartsWith(S: "aarch64.sve.ld2", Value: 2)
5995 .StartsWith(S: "aarch64.sve.ld3", Value: 3)
5996 .StartsWith(S: "aarch64.sve.ld4", Value: 4)
5997 .Default(Value: 0);
5998 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5999 unsigned MinElts = RetTy->getMinNumElements() / N;
6000 SmallVector<Value *, 2> Args(CI->args());
6001 Value *NewLdCall = Builder.CreateCall(Callee: NewFn, Args);
6002 Value *Ret = llvm::PoisonValue::get(T: RetTy);
6003 for (unsigned I = 0; I < N; I++) {
6004 Value *SRet = Builder.CreateExtractValue(Agg: NewLdCall, Idxs: I);
6005 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: SRet, Idx: I * MinElts);
6006 }
6007 NewCall = dyn_cast<CallInst>(Val: Ret);
6008 break;
6009 }
6010
6011 case Intrinsic::coro_end_async:
6012 case Intrinsic::coro_end: {
6013 SmallVector<Value *, 3> Args(CI->args());
6014 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
6015 Args.push_back(Elt: ConstantTokenNone::get(Context&: CI->getContext()));
6016 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6017
6018 if (!CI->getType()->isVoidTy()) {
6019 if (!CI->use_empty()) {
6020 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
6021 M: CI->getModule(), id: Intrinsic::coro_is_in_ramp);
6022 Value *InRamp = Builder.CreateCall(Callee: IsInRamp);
6023 CI->replaceAllUsesWith(V: Builder.CreateNot(V: InRamp));
6024 }
6025 CI->eraseFromParent();
6026 return;
6027 }
6028
6029 break;
6030 }
6031
6032 case Intrinsic::vector_extract: {
6033 StringRef Name = F->getName();
6034 Name = Name.substr(Start: 5); // Strip llvm
6035 if (!Name.starts_with(Prefix: "aarch64.sve.tuple.get")) {
6036 DefaultCase();
6037 return;
6038 }
6039 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
6040 unsigned MinElts = RetTy->getMinNumElements();
6041 uint64_t I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
6042 Value *NewIdx = ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * MinElts);
6043 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), NewIdx});
6044 break;
6045 }
6046
6047 case Intrinsic::vector_insert: {
6048 StringRef Name = F->getName();
6049 Name = Name.substr(Start: 5);
6050 if (!Name.starts_with(Prefix: "aarch64.sve.tuple")) {
6051 DefaultCase();
6052 return;
6053 }
6054 if (Name.starts_with(Prefix: "aarch64.sve.tuple.set")) {
6055 uint64_t I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
6056 auto *Ty = cast<ScalableVectorType>(Val: CI->getArgOperand(i: 2)->getType());
6057 Value *NewIdx =
6058 ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * Ty->getMinNumElements());
6059 NewCall = Builder.CreateCall(
6060 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), NewIdx});
6061 break;
6062 }
6063 if (Name.starts_with(Prefix: "aarch64.sve.tuple.create")) {
6064 unsigned N = StringSwitch<unsigned>(Name)
6065 .StartsWith(S: "aarch64.sve.tuple.create2", Value: 2)
6066 .StartsWith(S: "aarch64.sve.tuple.create3", Value: 3)
6067 .StartsWith(S: "aarch64.sve.tuple.create4", Value: 4)
6068 .Default(Value: 0);
6069 assert(N > 1 && "Create is expected to be between 2-4");
6070 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
6071 Value *Ret = llvm::PoisonValue::get(T: RetTy);
6072 unsigned MinElts = RetTy->getMinNumElements() / N;
6073 for (unsigned I = 0; I < N; I++) {
6074 Value *V = CI->getArgOperand(i: I);
6075 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: V, Idx: I * MinElts);
6076 }
6077 NewCall = dyn_cast<CallInst>(Val: Ret);
6078 }
6079 break;
6080 }
6081
6082 case Intrinsic::arm_neon_bfdot:
6083 case Intrinsic::arm_neon_bfmmla:
6084 case Intrinsic::arm_neon_bfmlalb:
6085 case Intrinsic::arm_neon_bfmlalt:
6086 case Intrinsic::aarch64_neon_bfdot:
6087 case Intrinsic::aarch64_neon_bfmmla:
6088 case Intrinsic::aarch64_neon_bfmlalb:
6089 case Intrinsic::aarch64_neon_bfmlalt: {
6090 SmallVector<Value *, 3> Args;
6091 assert(CI->arg_size() == 3 &&
6092 "Mismatch between function args and call args");
6093 size_t OperandWidth =
6094 CI->getArgOperand(i: 1)->getType()->getPrimitiveSizeInBits();
6095 assert((OperandWidth == 64 || OperandWidth == 128) &&
6096 "Unexpected operand width");
6097 Type *NewTy = FixedVectorType::get(ElementType: Type::getBFloatTy(C), NumElts: OperandWidth / 16);
6098 auto Iter = CI->args().begin();
6099 Args.push_back(Elt: *Iter++);
6100 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
6101 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
6102 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6103 break;
6104 }
6105
6106 case Intrinsic::bitreverse:
6107 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
6108 break;
6109
6110 case Intrinsic::ctlz:
6111 case Intrinsic::cttz: {
6112 if (CI->arg_size() != 1) {
6113 DefaultCase();
6114 return;
6115 }
6116
6117 NewCall =
6118 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), Builder.getFalse()});
6119 break;
6120 }
6121
6122 case Intrinsic::objectsize: {
6123 Value *NullIsUnknownSize =
6124 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(i: 2);
6125 Value *Dynamic =
6126 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(i: 3);
6127 NewCall = Builder.CreateCall(
6128 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), NullIsUnknownSize, Dynamic});
6129 break;
6130 }
6131
6132 case Intrinsic::ctpop:
6133 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
6134 break;
6135 case Intrinsic::dbg_value: {
6136 StringRef Name = F->getName();
6137 Name = Name.substr(Start: 5); // Strip llvm.
6138 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6139 if (Name.starts_with(Prefix: "dbg.addr")) {
6140 DIExpression *Expr = cast<DIExpression>(
6141 Val: cast<MetadataAsValue>(Val: CI->getArgOperand(i: 2))->getMetadata());
6142 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
6143 NewCall =
6144 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6145 MetadataAsValue::get(Context&: C, MD: Expr)});
6146 break;
6147 }
6148
6149 // Upgrade from the old version that had an extra offset argument.
6150 assert(CI->arg_size() == 4);
6151 // Drop nonzero offsets instead of attempting to upgrade them.
6152 if (auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1)))
6153 if (Offset->isNullValue()) {
6154 NewCall = Builder.CreateCall(
6155 Callee: NewFn,
6156 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)});
6157 break;
6158 }
6159 CI->eraseFromParent();
6160 return;
6161 }
6162
6163 case Intrinsic::ptr_annotation:
6164 // Upgrade from versions that lacked the annotation attribute argument.
6165 if (CI->arg_size() != 4) {
6166 DefaultCase();
6167 return;
6168 }
6169
6170 // Create a new call with an added null annotation attribute argument.
6171 NewCall = Builder.CreateCall(
6172 Callee: NewFn,
6173 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
6174 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
6175 NewCall->takeName(V: CI);
6176 CI->replaceAllUsesWith(V: NewCall);
6177 CI->eraseFromParent();
6178 return;
6179
6180 case Intrinsic::var_annotation:
6181 // Upgrade from versions that lacked the annotation attribute argument.
6182 if (CI->arg_size() != 4) {
6183 DefaultCase();
6184 return;
6185 }
6186 // Create a new call with an added null annotation attribute argument.
6187 NewCall = Builder.CreateCall(
6188 Callee: NewFn,
6189 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
6190 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
6191 NewCall->takeName(V: CI);
6192 CI->replaceAllUsesWith(V: NewCall);
6193 CI->eraseFromParent();
6194 return;
6195
6196 case Intrinsic::riscv_aes32dsi:
6197 case Intrinsic::riscv_aes32dsmi:
6198 case Intrinsic::riscv_aes32esi:
6199 case Intrinsic::riscv_aes32esmi:
6200 case Intrinsic::riscv_sm4ks:
6201 case Intrinsic::riscv_sm4ed: {
6202 // The last argument to these intrinsics used to be i8 and changed to i32.
6203 // The type overload for sm4ks and sm4ed was removed.
6204 Value *Arg2 = CI->getArgOperand(i: 2);
6205 if (Arg2->getType()->isIntegerTy(BitWidth: 32) && !CI->getType()->isIntegerTy(BitWidth: 64))
6206 return;
6207
6208 Value *Arg0 = CI->getArgOperand(i: 0);
6209 Value *Arg1 = CI->getArgOperand(i: 1);
6210 if (CI->getType()->isIntegerTy(BitWidth: 64)) {
6211 Arg0 = Builder.CreateTrunc(V: Arg0, DestTy: Builder.getInt32Ty());
6212 Arg1 = Builder.CreateTrunc(V: Arg1, DestTy: Builder.getInt32Ty());
6213 }
6214
6215 Arg2 = ConstantInt::get(Ty: Type::getInt32Ty(C),
6216 V: cast<ConstantInt>(Val: Arg2)->getZExtValue());
6217
6218 NewCall = Builder.CreateCall(Callee: NewFn, Args: {Arg0, Arg1, Arg2});
6219 Value *Res = NewCall;
6220 if (Res->getType() != CI->getType())
6221 Res = Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
6222 NewCall->takeName(V: CI);
6223 CI->replaceAllUsesWith(V: Res);
6224 CI->eraseFromParent();
6225 return;
6226 }
6227 case Intrinsic::nvvm_mapa_shared_cluster: {
6228 // Create a new call with the correct address space.
6229 NewCall =
6230 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
6231 Value *Res = NewCall;
6232 Res = Builder.CreateAddrSpaceCast(
6233 V: Res, DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED));
6234 NewCall->takeName(V: CI);
6235 CI->replaceAllUsesWith(V: Res);
6236 CI->eraseFromParent();
6237 return;
6238 }
6239 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6240 SmallVector<Value *, 4> Args(CI->args());
6241 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6242 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6243 Args[0] = Builder.CreateAddrSpaceCast(
6244 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6245
6246 // Append the missing trailing flag_valid_pattern (0 = disabled).
6247 Args.push_back(Elt: Builder.getInt32(C: 0));
6248
6249 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6250 NewCall->takeName(V: CI);
6251 CI->replaceAllUsesWith(V: NewCall);
6252 CI->eraseFromParent();
6253 return;
6254 }
6255 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6256 // (dst, mbar, src, size, ch, flag_ch)
6257 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6258 // i32 0 /* flag_valid_pattern=disabled */)
6259 SmallVector<Value *, 10> Args;
6260 for (unsigned I = 0; I < 4; ++I)
6261 Args.push_back(Elt: CI->getArgOperand(i: I));
6262 Args.push_back(Elt: Builder.getInt32(C: 0)); // ignore_bytes_left
6263 Args.push_back(Elt: Builder.getInt32(C: 0)); // ignore_bytes_right
6264 Args.push_back(Elt: CI->getArgOperand(i: 4)); // cache_hint
6265 Args.push_back(Elt: CI->getArgOperand(i: 5)); // flag_ch
6266 Args.push_back(Elt: Builder.getInt1(V: false)); // flag_oob
6267 Args.push_back(Elt: Builder.getInt32(C: 0)); // flag_valid_pattern
6268
6269 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6270 NewCall->takeName(V: CI);
6271 CI->replaceAllUsesWith(V: NewCall);
6272 CI->eraseFromParent();
6273 return;
6274 }
6275 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6276 // Create a new call with the correct address space.
6277 SmallVector<Value *, 4> Args(CI->args());
6278 Args[0] = Builder.CreateAddrSpaceCast(
6279 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6280
6281 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6282 NewCall->takeName(V: CI);
6283 CI->replaceAllUsesWith(V: NewCall);
6284 CI->eraseFromParent();
6285 return;
6286 }
6287 // clang-format off
6288#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6289 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6290 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6291#undef G2S_CLUSTER_CASE
6292 {
6293 SmallVector<Value *, 16> Args(CI->args());
6294 unsigned AS = CI->getArgOperand(i: 0)->getType()->getPointerAddressSpace();
6295 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6296 Args[0] = Builder.CreateAddrSpaceCast(
6297 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6298
6299 // Append the missing trailing arguments with default values (cta_group,
6300 // flag_valid_pattern).
6301 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6302 Args.push_back(Elt: Builder.getInt32(C: 0));
6303
6304 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6305 NewCall->takeName(V: CI);
6306 CI->replaceAllUsesWith(V: NewCall);
6307 CI->eraseFromParent();
6308 return;
6309 }
6310
6311#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6312 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6313 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6314#undef G2S_CTA_CASE
6315 {
6316 SmallVector<Value *, 16> Args(CI->args());
6317 // Append the missing trailing flag_valid_pattern argument with default
6318 // value 0.
6319 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6320 "expected only the trailing flag_valid_pattern to be missing");
6321 Args.push_back(Elt: Builder.getInt32(C: 0));
6322
6323 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6324 NewCall->takeName(V: CI);
6325 CI->replaceAllUsesWith(V: NewCall);
6326 CI->eraseFromParent();
6327 return;
6328 }
6329#undef NVVM_TMA_G2S_MODES
6330 // clang-format on
6331
6332 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6333 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6334 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6335 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6336 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6337 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6338 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6339 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6340 StringRef Name = F->getName();
6341 Name.consume_front(Prefix: "llvm.nvvm.cp.async.bulk.tensor.reduce.");
6342 auto RedOp = getNVPTXTMAReductionOp(Name: Name.split(Separator: '.').first);
6343
6344 SmallVector<Value *, 16> Args(CI->args());
6345 Args.insert(I: Args.end() - 1, Elt: Builder.getInt32(C: *RedOp));
6346 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6347 break;
6348 }
6349 // clang-format off
6350 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cta_space_cluster:
6351 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cluster_space_cluster:
6352 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cta_space_cluster:
6353 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cluster_space_cluster:
6354 case Intrinsic::nvvm_mbarrier_arrive_scope_cta_space_cluster:
6355 case Intrinsic::nvvm_mbarrier_arrive_scope_cluster_space_cluster:
6356 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cta_space_cluster:
6357 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cluster_space_cluster:
6358 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cta_space_cluster:
6359 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cluster_space_cluster:
6360 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cta_space_cluster:
6361 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cluster_space_cluster:
6362 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cta_space_cluster:
6363 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cluster_space_cluster:
6364 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cta_space_cluster:
6365 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cluster_space_cluster:
6366 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cta_space_cluster:
6367 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cluster_space_cluster:
6368 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cta_space_cluster:
6369 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cluster_space_cluster: {
6370 // clang-format on
6371 SmallVector<Value *, 8> Args(CI->args());
6372 Args.push_back(Elt: Builder.getInt32(C: 0)); // cta-mask
6373 Args.push_back(Elt: Builder.getFalse()); // is-multicast
6374 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6375 NewCall->takeName(V: CI);
6376 CI->replaceAllUsesWith(V: NewCall);
6377 CI->eraseFromParent();
6378 return;
6379 }
6380 case Intrinsic::nvvm_mbarrier_init: {
6381 SmallVector<Value *, 3> Args(CI->args());
6382 // The .shared variant folded into the overloaded form without gaining an
6383 // operand, so only the pre-layout two-argument form needs one appended.
6384 if (Args.size() == 2)
6385 Args.push_back(Elt: Builder.getInt32(C: 0)); // layout = default(0)
6386 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6387 break;
6388 }
6389 case Intrinsic::riscv_sha256sig0:
6390 case Intrinsic::riscv_sha256sig1:
6391 case Intrinsic::riscv_sha256sum0:
6392 case Intrinsic::riscv_sha256sum1:
6393 case Intrinsic::riscv_sm3p0:
6394 case Intrinsic::riscv_sm3p1: {
6395 // The last argument to these intrinsics used to be i8 and changed to i32.
6396 // The type overload for sm4ks and sm4ed was removed.
6397 if (!CI->getType()->isIntegerTy(BitWidth: 64))
6398 return;
6399
6400 Value *Arg =
6401 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getInt32Ty());
6402
6403 NewCall = Builder.CreateCall(Callee: NewFn, Args: Arg);
6404 Value *Res =
6405 Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
6406 NewCall->takeName(V: CI);
6407 CI->replaceAllUsesWith(V: Res);
6408 CI->eraseFromParent();
6409 return;
6410 }
6411
6412 case Intrinsic::x86_xop_vfrcz_ss:
6413 case Intrinsic::x86_xop_vfrcz_sd:
6414 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 1)});
6415 break;
6416
6417 case Intrinsic::x86_xop_vpermil2pd:
6418 case Intrinsic::x86_xop_vpermil2ps:
6419 case Intrinsic::x86_xop_vpermil2pd_256:
6420 case Intrinsic::x86_xop_vpermil2ps_256: {
6421 SmallVector<Value *, 4> Args(CI->args());
6422 VectorType *FltIdxTy = cast<VectorType>(Val: Args[2]->getType());
6423 VectorType *IntIdxTy = VectorType::getInteger(VTy: FltIdxTy);
6424 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: IntIdxTy);
6425 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6426 break;
6427 }
6428
6429 case Intrinsic::x86_sse41_ptestc:
6430 case Intrinsic::x86_sse41_ptestz:
6431 case Intrinsic::x86_sse41_ptestnzc: {
6432 // The arguments for these intrinsics used to be v4f32, and changed
6433 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6434 // So, the only thing required is a bitcast for both arguments.
6435 // First, check the arguments have the old type.
6436 Value *Arg0 = CI->getArgOperand(i: 0);
6437 if (Arg0->getType() != FixedVectorType::get(ElementType: Type::getFloatTy(C), NumElts: 4))
6438 return;
6439
6440 // Old intrinsic, add bitcasts
6441 Value *Arg1 = CI->getArgOperand(i: 1);
6442
6443 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
6444
6445 Value *BC0 = Builder.CreateBitCast(V: Arg0, DestTy: NewVecTy, Name: "cast");
6446 Value *BC1 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
6447
6448 NewCall = Builder.CreateCall(Callee: NewFn, Args: {BC0, BC1});
6449 break;
6450 }
6451
6452 case Intrinsic::x86_rdtscp: {
6453 // This used to take 1 arguments. If we have no arguments, it is already
6454 // upgraded.
6455 if (CI->getNumOperands() == 0)
6456 return;
6457
6458 NewCall = Builder.CreateCall(Callee: NewFn);
6459 // Extract the second result and store it.
6460 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
6461 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
6462 // Replace the original call result with the first result of the new call.
6463 Value *TSC = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
6464
6465 NewCall->takeName(V: CI);
6466 CI->replaceAllUsesWith(V: TSC);
6467 CI->eraseFromParent();
6468 return;
6469 }
6470
6471 case Intrinsic::x86_sse41_insertps:
6472 case Intrinsic::x86_sse41_dppd:
6473 case Intrinsic::x86_sse41_dpps:
6474 case Intrinsic::x86_sse41_mpsadbw:
6475 case Intrinsic::x86_avx_dp_ps_256:
6476 case Intrinsic::x86_avx2_mpsadbw: {
6477 // Need to truncate the last argument from i32 to i8 -- this argument models
6478 // an inherently 8-bit immediate operand to these x86 instructions.
6479 SmallVector<Value *, 4> Args(CI->args());
6480
6481 // Replace the last argument with a trunc.
6482 Args.back() = Builder.CreateTrunc(V: Args.back(), DestTy: Type::getInt8Ty(C), Name: "trunc");
6483 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6484 break;
6485 }
6486
6487 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6488 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6489 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6490 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6491 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6492 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6493 SmallVector<Value *, 4> Args(CI->args());
6494 unsigned NumElts =
6495 cast<FixedVectorType>(Val: Args[0]->getType())->getNumElements();
6496 Args[3] = getX86MaskVec(Builder, Mask: Args[3], NumElts);
6497
6498 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6499 Value *Res = applyX86MaskOn1BitsVec(Builder, Vec: NewCall, Mask: nullptr);
6500
6501 NewCall->takeName(V: CI);
6502 CI->replaceAllUsesWith(V: Res);
6503 CI->eraseFromParent();
6504 return;
6505 }
6506
6507 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6508 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6509 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6510 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6511 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6512 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6513 SmallVector<Value *, 4> Args(CI->args());
6514 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
6515 if (NewFn->getIntrinsicID() ==
6516 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6517 Args[1] = Builder.CreateBitCast(
6518 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6519
6520 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6521 Value *Res = Builder.CreateBitCast(
6522 V: NewCall, DestTy: FixedVectorType::get(ElementType: Builder.getInt16Ty(), NumElts));
6523
6524 NewCall->takeName(V: CI);
6525 CI->replaceAllUsesWith(V: Res);
6526 CI->eraseFromParent();
6527 return;
6528 }
6529 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6530 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6531 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6532 SmallVector<Value *, 4> Args(CI->args());
6533 unsigned NumElts =
6534 cast<FixedVectorType>(Val: CI->getType())->getNumElements() * 2;
6535 Args[1] = Builder.CreateBitCast(
6536 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6537 Args[2] = Builder.CreateBitCast(
6538 V: Args[2], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6539
6540 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6541 break;
6542 }
6543
6544 case Intrinsic::amdgcn_asyncmark:
6545 case Intrinsic::amdgcn_wait_asyncmark: {
6546 // Legacy asyncmark intrinsics missed the stage mask operand. Append a zero
6547 // mask, which names every stage and so preserves the behavior they had.
6548 SmallVector<Value *, 2> Args(CI->args());
6549 Args.push_back(Elt: Builder.getInt32(C: 0));
6550
6551 SmallVector<OperandBundleDef, 1> Bundles;
6552 CI->getOperandBundlesAsDefs(Defs&: Bundles);
6553
6554 NewCall = Builder.CreateCall(Callee: NewFn, Args, OpBundles: Bundles);
6555 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6556 NewCall->setCallingConv(CI->getCallingConv());
6557 NewCall->setAttributes(CI->getAttributes());
6558 NewCall->copyMetadata(SrcInst: *CI);
6559 break;
6560 }
6561
6562 case Intrinsic::thread_pointer: {
6563 NewCall = Builder.CreateCall(Callee: NewFn, Args: {});
6564 break;
6565 }
6566
6567 case Intrinsic::memcpy:
6568 case Intrinsic::memmove:
6569 case Intrinsic::memset: {
6570 // We have to make sure that the call signature is what we're expecting.
6571 // We only want to change the old signatures by removing the alignment arg:
6572 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6573 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6574 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6575 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6576 // Note: i8*'s in the above can be any pointer type
6577 if (CI->arg_size() != 5) {
6578 DefaultCase();
6579 return;
6580 }
6581 // Remove alignment argument (3), and add alignment attributes to the
6582 // dest/src pointers.
6583 Value *Args[4] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6584 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
6585 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6586 AttributeList OldAttrs = CI->getAttributes();
6587 AttributeList NewAttrs = AttributeList::get(
6588 C, FnAttrs: OldAttrs.getFnAttrs(), RetAttrs: OldAttrs.getRetAttrs(),
6589 ArgAttrs: {OldAttrs.getParamAttrs(ArgNo: 0), OldAttrs.getParamAttrs(ArgNo: 1),
6590 OldAttrs.getParamAttrs(ArgNo: 2), OldAttrs.getParamAttrs(ArgNo: 4)});
6591 NewCall->setAttributes(NewAttrs);
6592 auto *MemCI = cast<MemIntrinsic>(Val: NewCall);
6593 // All mem intrinsics support dest alignment.
6594 const ConstantInt *Align = cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
6595 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6596 // Memcpy/Memmove also support source alignment.
6597 if (auto *MTI = dyn_cast<MemTransferInst>(Val: MemCI))
6598 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6599 break;
6600 }
6601
6602 case Intrinsic::masked_load:
6603 case Intrinsic::masked_gather:
6604 case Intrinsic::masked_store:
6605 case Intrinsic::masked_scatter: {
6606 if (CI->arg_size() != 4) {
6607 DefaultCase();
6608 return;
6609 }
6610
6611 auto GetMaybeAlign = [](Value *Op) {
6612 if (auto *CI = dyn_cast<ConstantInt>(Val: Op)) {
6613 uint64_t Val = CI->getZExtValue();
6614 if (Val == 0)
6615 return MaybeAlign();
6616 if (isPowerOf2_64(Value: Val))
6617 return MaybeAlign(Val);
6618 }
6619 reportFatalUsageError(reason: "Invalid alignment argument");
6620 };
6621 auto GetAlign = [&](Value *Op) {
6622 MaybeAlign Align = GetMaybeAlign(Op);
6623 if (Align)
6624 return *Align;
6625 reportFatalUsageError(reason: "Invalid zero alignment argument");
6626 };
6627
6628 const DataLayout &DL = CI->getDataLayout();
6629 switch (NewFn->getIntrinsicID()) {
6630 case Intrinsic::masked_load:
6631 NewCall = Builder.CreateMaskedLoad(
6632 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0), Alignment: GetAlign(CI->getArgOperand(i: 1)),
6633 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6634 break;
6635 case Intrinsic::masked_gather:
6636 NewCall = Builder.CreateMaskedGather(
6637 Ty: CI->getType(), Ptrs: CI->getArgOperand(i: 0),
6638 Alignment: DL.getValueOrABITypeAlignment(Alignment: GetMaybeAlign(CI->getArgOperand(i: 1)),
6639 Ty: CI->getType()->getScalarType()),
6640 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6641 break;
6642 case Intrinsic::masked_store:
6643 NewCall = Builder.CreateMaskedStore(
6644 Val: CI->getArgOperand(i: 0), Ptr: CI->getArgOperand(i: 1),
6645 Alignment: GetAlign(CI->getArgOperand(i: 2)), Mask: CI->getArgOperand(i: 3));
6646 break;
6647 case Intrinsic::masked_scatter:
6648 NewCall = Builder.CreateMaskedScatter(
6649 Val: CI->getArgOperand(i: 0), Ptrs: CI->getArgOperand(i: 1),
6650 Alignment: DL.getValueOrABITypeAlignment(
6651 Alignment: GetMaybeAlign(CI->getArgOperand(i: 2)),
6652 Ty: CI->getArgOperand(i: 0)->getType()->getScalarType()),
6653 Mask: CI->getArgOperand(i: 3));
6654 break;
6655 default:
6656 llvm_unreachable("Unexpected intrinsic ID");
6657 }
6658 // Previous metadata is still valid.
6659 NewCall->copyMetadata(SrcInst: *CI);
6660 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6661 break;
6662 }
6663
6664 case Intrinsic::lifetime_start:
6665 case Intrinsic::lifetime_end: {
6666 if (CI->arg_size() != 2) {
6667 DefaultCase();
6668 return;
6669 }
6670
6671 Value *Ptr = CI->getArgOperand(i: 1);
6672 // Try to strip pointer casts, such that the lifetime works on an alloca.
6673 Ptr = Ptr->stripPointerCasts();
6674 if (isa<AllocaInst>(Val: Ptr)) {
6675 // Don't use NewFn, as we might have looked through an addrspacecast.
6676 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6677 NewCall = Builder.CreateLifetimeStart(Ptr);
6678 else
6679 NewCall = Builder.CreateLifetimeEnd(Ptr);
6680 break;
6681 }
6682
6683 // Otherwise remove the lifetime marker.
6684 CI->eraseFromParent();
6685 return;
6686 }
6687
6688 case Intrinsic::x86_avx512_vpdpbusd_128:
6689 case Intrinsic::x86_avx512_vpdpbusd_256:
6690 case Intrinsic::x86_avx512_vpdpbusd_512:
6691 case Intrinsic::x86_avx512_vpdpbusds_128:
6692 case Intrinsic::x86_avx512_vpdpbusds_256:
6693 case Intrinsic::x86_avx512_vpdpbusds_512:
6694 case Intrinsic::x86_avx2_vpdpbssd_128:
6695 case Intrinsic::x86_avx2_vpdpbssd_256:
6696 case Intrinsic::x86_avx10_vpdpbssd_512:
6697 case Intrinsic::x86_avx2_vpdpbssds_128:
6698 case Intrinsic::x86_avx2_vpdpbssds_256:
6699 case Intrinsic::x86_avx10_vpdpbssds_512:
6700 case Intrinsic::x86_avx2_vpdpbsud_128:
6701 case Intrinsic::x86_avx2_vpdpbsud_256:
6702 case Intrinsic::x86_avx10_vpdpbsud_512:
6703 case Intrinsic::x86_avx2_vpdpbsuds_128:
6704 case Intrinsic::x86_avx2_vpdpbsuds_256:
6705 case Intrinsic::x86_avx10_vpdpbsuds_512:
6706 case Intrinsic::x86_avx2_vpdpbuud_128:
6707 case Intrinsic::x86_avx2_vpdpbuud_256:
6708 case Intrinsic::x86_avx10_vpdpbuud_512:
6709 case Intrinsic::x86_avx2_vpdpbuuds_128:
6710 case Intrinsic::x86_avx2_vpdpbuuds_256:
6711 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6712 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6713 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6714 CI->getArgOperand(i: 2)};
6715 Type *NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: NumElts, Scalable: false);
6716 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6717 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6718
6719 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6720 break;
6721 }
6722 case Intrinsic::x86_avx512_vpdpwssd_128:
6723 case Intrinsic::x86_avx512_vpdpwssd_256:
6724 case Intrinsic::x86_avx512_vpdpwssd_512:
6725 case Intrinsic::x86_avx512_vpdpwssds_128:
6726 case Intrinsic::x86_avx512_vpdpwssds_256:
6727 case Intrinsic::x86_avx512_vpdpwssds_512:
6728 case Intrinsic::x86_avx2_vpdpwsud_128:
6729 case Intrinsic::x86_avx2_vpdpwsud_256:
6730 case Intrinsic::x86_avx10_vpdpwsud_512:
6731 case Intrinsic::x86_avx2_vpdpwsuds_128:
6732 case Intrinsic::x86_avx2_vpdpwsuds_256:
6733 case Intrinsic::x86_avx10_vpdpwsuds_512:
6734 case Intrinsic::x86_avx2_vpdpwusd_128:
6735 case Intrinsic::x86_avx2_vpdpwusd_256:
6736 case Intrinsic::x86_avx10_vpdpwusd_512:
6737 case Intrinsic::x86_avx2_vpdpwusds_128:
6738 case Intrinsic::x86_avx2_vpdpwusds_256:
6739 case Intrinsic::x86_avx10_vpdpwusds_512:
6740 case Intrinsic::x86_avx2_vpdpwuud_128:
6741 case Intrinsic::x86_avx2_vpdpwuud_256:
6742 case Intrinsic::x86_avx10_vpdpwuud_512:
6743 case Intrinsic::x86_avx2_vpdpwuuds_128:
6744 case Intrinsic::x86_avx2_vpdpwuuds_256:
6745 case Intrinsic::x86_avx10_vpdpwuuds_512:
6746 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6747 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6748 CI->getArgOperand(i: 2)};
6749 Type *NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: NumElts, Scalable: false);
6750 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6751 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6752
6753 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6754 break;
6755 }
6756 assert(NewCall && "Should have either set this variable or returned through "
6757 "the default case");
6758 NewCall->takeName(V: CI);
6759 CI->replaceAllUsesWith(V: NewCall);
6760 CI->eraseFromParent();
6761}
6762
6763void llvm::UpgradeCallsToIntrinsic(Function *F) {
6764 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6765
6766 // Check if this function should be upgraded and get the replacement function
6767 // if there is one.
6768 Function *NewFn;
6769 if (UpgradeIntrinsicFunction(F, NewFn)) {
6770 // Replace all users of the old function with the new function or new
6771 // instructions. This is not a range loop because the call is deleted.
6772 for (User *U : make_early_inc_range(Range: F->users()))
6773 if (CallBase *CB = dyn_cast<CallBase>(Val: U))
6774 UpgradeIntrinsicCall(CI: CB, NewFn);
6775
6776 // Remove old function, no longer used, from the module.
6777 if (F != NewFn)
6778 F->eraseFromParent();
6779 }
6780}
6781
6782MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6783 const unsigned NumOperands = MD.getNumOperands();
6784 if (NumOperands == 0)
6785 return &MD; // Invalid, punt to a verifier error.
6786
6787 // Check if the tag uses struct-path aware TBAA format.
6788 if (isa<MDNode>(Val: MD.getOperand(I: 0)) && NumOperands >= 3)
6789 return &MD;
6790
6791 auto &Context = MD.getContext();
6792 if (NumOperands == 3) {
6793 Metadata *Elts[] = {MD.getOperand(I: 0), MD.getOperand(I: 1)};
6794 MDNode *ScalarType = MDNode::get(Context, MDs: Elts);
6795 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6796 Metadata *Elts2[] = {ScalarType, ScalarType,
6797 ConstantAsMetadata::get(
6798 C: Constant::getNullValue(Ty: Type::getInt64Ty(C&: Context))),
6799 MD.getOperand(I: 2)};
6800 return MDNode::get(Context, MDs: Elts2);
6801 }
6802 // Create a MDNode <MD, MD, offset 0>
6803 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(C: Constant::getNullValue(
6804 Ty: Type::getInt64Ty(C&: Context)))};
6805 return MDNode::get(Context, MDs: Elts);
6806}
6807
6808MDNode *llvm::UpgradeTBAAStructNode(MDNode &MD) {
6809 // !tbaa.struct is a list of (offset, size, tag) triples. Upgrade any
6810 // old-style scalar field tag to struct-path form via UpgradeTBAANode.
6811 unsigned NumOperands = MD.getNumOperands();
6812 if (NumOperands == 0 || NumOperands % 3 != 0)
6813 return &MD; // Malformed; leave it for the verifier to reject.
6814
6815 SmallVector<Metadata *, 12> Elts(MD.op_begin(), MD.op_end());
6816 bool Changed = false;
6817 for (unsigned I = 2; I < NumOperands; I += 3) {
6818 auto *Tag = dyn_cast_or_null<MDNode>(Val: Elts[I]);
6819 if (!Tag)
6820 continue;
6821 MDNode *Upgraded = UpgradeTBAANode(MD&: *Tag);
6822 if (Upgraded == Tag)
6823 continue;
6824 Elts[I] = Upgraded;
6825 Changed = true;
6826 }
6827 return Changed ? MDNode::get(Context&: MD.getContext(), MDs: Elts) : &MD;
6828}
6829
6830/// Rebuild the alias scope or domain \p Old with \p Ops, keeping its identity:
6831/// a self reference has to point at the replacement, and a distinct node must
6832/// not be uniqued. The first operand of both a scope and a domain is its name,
6833/// which is either a string or a self reference.
6834static MDNode *rebuildScopeOrDomainNode(MDNode *Old,
6835 SmallVectorImpl<Metadata *> &Ops) {
6836 LLVMContext &Context = Old->getContext();
6837 if (Ops[0] != Old)
6838 return Old->isDistinct() ? MDNode::getDistinct(Context, MDs: Ops)
6839 : MDNode::get(Context, MDs: Ops);
6840
6841 Ops[0] = nullptr;
6842 MDNode *New = MDNode::getDistinct(Context, MDs: Ops);
6843 New->replaceOperandWith(I: 0, New);
6844 return New;
6845}
6846
6847static MDNode *upgradeAliasScopeDomain(MDNode *Domain,
6848 DenseMap<MDNode *, MDNode *> &Upgraded) {
6849 unsigned NumOperands = Domain->getNumOperands();
6850 bool HadDescription = NumOperands == 2;
6851 // Already upgraded, or invalid and left to the verifier.
6852 if (NumOperands == 0 || NumOperands > 2 ||
6853 (HadDescription && mdconst::hasa<ConstantInt>(MD: Domain->getOperand(I: 1))))
6854 return Domain;
6855
6856 MDNode *&Upgrade = Upgraded[Domain];
6857 if (Upgrade)
6858 return Upgrade;
6859
6860 LLVMContext &Context = Domain->getContext();
6861 SmallVector<Metadata *, 3> Ops = {
6862 Domain->getOperand(I: 0),
6863 ConstantAsMetadata::get(C: ConstantInt::getFalse(Context))};
6864 if (HadDescription)
6865 Ops.push_back(Elt: Domain->getOperand(I: 1));
6866
6867 Upgrade = rebuildScopeOrDomainNode(Old: Domain, Ops);
6868 return Upgrade;
6869}
6870
6871static MDNode *upgradeAliasScope(MDNode *Scope,
6872 DenseMap<MDNode *, MDNode *> &Upgraded) {
6873 if (MDNode *Upgrade = Upgraded.lookup(Val: Scope))
6874 return Upgrade;
6875
6876 // Extract domain, or leave invalid scope to the verifier.
6877 if (Scope->getNumOperands() < 2 || !isa<MDNode>(Val: Scope->getOperand(I: 1)))
6878 return Scope;
6879 auto *Domain = cast<MDNode>(Val: Scope->getOperand(I: 1));
6880
6881 MDNode *UpgradedDomain = upgradeAliasScopeDomain(Domain, Upgraded);
6882 if (UpgradedDomain == Domain)
6883 return Scope;
6884
6885 SmallVector<Metadata *, 3> Ops(Scope->op_begin(), Scope->op_end());
6886 Ops[1] = UpgradedDomain;
6887 MDNode *Upgrade = rebuildScopeOrDomainNode(Old: Scope, Ops);
6888 Upgraded[Scope] = Upgrade;
6889 return Upgrade;
6890}
6891
6892MDNode *
6893llvm::UpgradeAliasScopeList(MDNode &ScopeList,
6894 DenseMap<MDNode *, MDNode *> &UpgradedAliasScopes) {
6895 if (MDNode *Upgrade = UpgradedAliasScopes.lookup(Val: &ScopeList))
6896 return Upgrade;
6897
6898 SmallVector<Metadata *, 4> Ops;
6899 bool Changed = false;
6900 for (const MDOperand &Op : ScopeList.operands()) {
6901 Metadata *Scope = Op;
6902 if (auto *ScopeNode = dyn_cast<MDNode>(Val: Op))
6903 Scope = upgradeAliasScope(Scope: ScopeNode, Upgraded&: UpgradedAliasScopes);
6904 Changed |= Scope != Op.get();
6905 Ops.push_back(Elt: Scope);
6906 }
6907 if (!Changed)
6908 return &ScopeList;
6909
6910 MDNode *Upgrade = MDNode::get(Context&: ScopeList.getContext(), MDs: Ops);
6911 UpgradedAliasScopes[&ScopeList] = Upgrade;
6912 return Upgrade;
6913}
6914
6915Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6916 Instruction *&Temp) {
6917 if (Opc != Instruction::BitCast)
6918 return nullptr;
6919
6920 Temp = nullptr;
6921 Type *SrcTy = V->getType();
6922 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6923 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6924 LLVMContext &Context = V->getContext();
6925
6926 // We have no information about target data layout, so we assume that
6927 // the maximum pointer size is 64bit.
6928 Type *MidTy = Type::getInt64Ty(C&: Context);
6929 Temp = CastInst::Create(Instruction::PtrToInt, S: V, Ty: MidTy);
6930
6931 return CastInst::Create(Instruction::IntToPtr, S: Temp, Ty: DestTy);
6932 }
6933
6934 return nullptr;
6935}
6936
6937Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6938 if (Opc != Instruction::BitCast)
6939 return nullptr;
6940
6941 Type *SrcTy = C->getType();
6942 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6943 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6944 LLVMContext &Context = C->getContext();
6945
6946 // We have no information about target data layout, so we assume that
6947 // the maximum pointer size is 64bit.
6948 Type *MidTy = Type::getInt64Ty(C&: Context);
6949
6950 return ConstantExpr::getIntToPtr(C: ConstantExpr::getPtrToInt(C, Ty: MidTy),
6951 Ty: DestTy);
6952 }
6953
6954 return nullptr;
6955}
6956
6957static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6958 if (Flag.getNumOperands() < 3)
6959 return std::nullopt;
6960 if (MDString *Name = dyn_cast_or_null<MDString>(Val: Flag.getOperand(I: 1)))
6961 return Name->getString();
6962 return std::nullopt;
6963}
6964
6965/// Check the debug info version number, if it is out-dated, drop the debug
6966/// info. Return true if module is modified.
6967bool llvm::UpgradeDebugInfo(Module &M) {
6968 if (!getIROptions(C: M.getContext()).auto_upgrade_debug_info)
6969 return false;
6970
6971 llvm::TimeTraceScope timeScope("Upgrade debug info");
6972 // We need to get metadata before the module is verified (i.e., getModuleFlag
6973 // makes assumptions that we haven't verified yet). Carefully extract the flag
6974 // from the metadata.
6975 unsigned Version = 0;
6976 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6977 auto OpIt = find_if(Range: ModFlags->operands(), P: [](const MDNode *Flag) {
6978 if (auto Name = getModuleFlagNameSafely(Flag: *Flag))
6979 return *Name == "Debug Info Version";
6980 return false;
6981 });
6982 if (OpIt != ModFlags->op_end()) {
6983 const MDOperand &ValOp = (*OpIt)->getOperand(I: 2);
6984 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(MD: ValOp))
6985 Version = CI->getZExtValue();
6986 }
6987 }
6988
6989 if (Version == DEBUG_METADATA_VERSION) {
6990 bool BrokenDebugInfo = false;
6991 if (verifyModule(M, OS: &llvm::errs(), BrokenDebugInfo: &BrokenDebugInfo))
6992 report_fatal_error(reason: "Broken module found, compilation aborted!");
6993 if (!BrokenDebugInfo)
6994 // Everything is ok.
6995 return false;
6996 else {
6997 // Diagnose malformed debug info.
6998 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6999 M.getContext().diagnose(DI: Diag);
7000 }
7001 }
7002 bool Modified = StripDebugInfo(M);
7003 if (Modified && Version != DEBUG_METADATA_VERSION) {
7004 // Diagnose a version mismatch.
7005 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
7006 M.getContext().diagnose(DI: DiagVersion);
7007 }
7008 return Modified;
7009}
7010
7011static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
7012 GlobalValue *GV, const Metadata *V) {
7013 Function *F = cast<Function>(Val: GV);
7014
7015 constexpr StringLiteral DefaultValue = "1";
7016 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
7017 unsigned Length = 0;
7018
7019 if (F->hasFnAttribute(Kind: Attr)) {
7020 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
7021 // parse these elements placing them into Vect3
7022 StringRef S = F->getFnAttribute(Kind: Attr).getValueAsString();
7023 for (; Length < 3 && !S.empty(); Length++) {
7024 auto [Part, Rest] = S.split(Separator: ',');
7025 Vect3[Length] = Part.trim();
7026 S = Rest;
7027 }
7028 }
7029
7030 const unsigned Dim = DimC - 'x';
7031 assert(Dim < 3 && "Unexpected dim char");
7032
7033 const uint64_t VInt = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
7034
7035 // local variable required for StringRef in Vect3 to point to.
7036 const std::string VStr = llvm::utostr(X: VInt);
7037 Vect3[Dim] = VStr;
7038 Length = std::max(a: Length, b: Dim + 1);
7039
7040 const std::string NewAttr = llvm::join(R: ArrayRef(Vect3, Length), Separator: ",");
7041 F->addFnAttr(Kind: Attr, Val: NewAttr);
7042}
7043
7044static inline bool isXYZ(StringRef S) {
7045 return S == "x" || S == "y" || S == "z";
7046}
7047
7048bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
7049 const Metadata *V) {
7050 if (K == "kernel") {
7051 if (!mdconst::extract<ConstantInt>(MD&: V)->isZero())
7052 cast<Function>(Val: GV)->setCallingConv(CallingConv::PTX_Kernel);
7053 return true;
7054 }
7055 if (K == "align") {
7056 // V is a bitfeild specifying two 16-bit values. The alignment value is
7057 // specfied in low 16-bits, The index is specified in the high bits. For the
7058 // index, 0 indicates the return value while higher values correspond to
7059 // each parameter (idx = param + 1).
7060 const uint64_t AlignIdxValuePair =
7061 mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
7062 const unsigned Idx = (AlignIdxValuePair >> 16);
7063 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
7064 cast<Function>(Val: GV)->addAttributeAtIndex(
7065 i: Idx, Attr: Attribute::getWithStackAlignment(Context&: GV->getContext(), Alignment: StackAlign));
7066 return true;
7067 }
7068 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
7069 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
7070 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: CV));
7071 return true;
7072 }
7073 if (K == "minctasm") {
7074 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
7075 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MinCTASm, Val: llvm::utostr(X: CV));
7076 return true;
7077 }
7078 if (K == "maxnreg") {
7079 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
7080 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxNReg, Val: llvm::utostr(X: CV));
7081 return true;
7082 }
7083 if (K.consume_front(Prefix: "maxntid") && isXYZ(S: K)) {
7084 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::MaxNTID, DimC: K[0], GV, V);
7085 return true;
7086 }
7087 if (K.consume_front(Prefix: "reqntid") && isXYZ(S: K)) {
7088 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ReqNTID, DimC: K[0], GV, V);
7089 return true;
7090 }
7091 if (K.consume_front(Prefix: "cluster_dim_") && isXYZ(S: K)) {
7092 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ClusterDim, DimC: K[0], GV, V);
7093 return true;
7094 }
7095 if (K == "grid_constant") {
7096 const auto Attr = Attribute::get(Context&: GV->getContext(), Kind: NVVMAttr::GridConstant);
7097 for (const auto &Op : cast<MDNode>(Val: V)->operands()) {
7098 // For some reason, the index is 1-based in the metadata. Good thing we're
7099 // able to auto-upgrade it!
7100 const auto Index = mdconst::extract<ConstantInt>(MD: Op)->getZExtValue() - 1;
7101 cast<Function>(Val: GV)->addParamAttr(ArgNo: Index, Attr);
7102 }
7103 return true;
7104 }
7105
7106 return false;
7107}
7108
7109void llvm::UpgradeNVVMAnnotations(Module &M) {
7110 NamedMDNode *NamedMD = M.getNamedMetadata(Name: "nvvm.annotations");
7111 if (!NamedMD)
7112 return;
7113
7114 SmallVector<MDNode *, 8> NewNodes;
7115 SmallPtrSet<const MDNode *, 8> SeenNodes;
7116 for (MDNode *MD : NamedMD->operands()) {
7117 if (!SeenNodes.insert(Ptr: MD).second)
7118 continue;
7119
7120 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD: MD->getOperand(I: 0));
7121 if (!GV)
7122 continue;
7123
7124 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
7125
7126 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(I: 0)};
7127 // Each nvvm.annotations metadata entry will be of the following form:
7128 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
7129 // start index = 1, to skip the global variable key
7130 // increment = 2, to skip the value for each property-value pairs
7131 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
7132 MDString *K = cast<MDString>(Val: MD->getOperand(I: j));
7133 const MDOperand &V = MD->getOperand(I: j + 1);
7134 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K: K->getString(), V);
7135 if (!Upgraded)
7136 NewOperands.append(IL: {K, V});
7137 }
7138
7139 if (NewOperands.size() > 1)
7140 NewNodes.push_back(Elt: MDNode::get(Context&: M.getContext(), MDs: NewOperands));
7141 }
7142
7143 NamedMD->clearOperands();
7144 for (MDNode *N : NewNodes)
7145 NamedMD->addOperand(M: N);
7146}
7147
7148/// This checks for objc retain release marker which should be upgraded. It
7149/// returns true if module is modified.
7150static bool upgradeRetainReleaseMarker(Module &M) {
7151 bool Changed = false;
7152 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
7153 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(Name: MarkerKey);
7154 if (ModRetainReleaseMarker) {
7155 MDNode *Op = ModRetainReleaseMarker->getOperand(i: 0);
7156 if (Op) {
7157 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 0));
7158 if (ID) {
7159 SmallVector<StringRef, 4> ValueComp;
7160 ID->getString().split(A&: ValueComp, Separator: "#");
7161 if (ValueComp.size() == 2) {
7162 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
7163 ID = MDString::get(Context&: M.getContext(), Str: NewValue);
7164 }
7165 M.addModuleFlag(Behavior: Module::Error, Key: MarkerKey, Val: ID);
7166 M.eraseNamedMetadata(NMD: ModRetainReleaseMarker);
7167 Changed = true;
7168 }
7169 }
7170 }
7171 return Changed;
7172}
7173
7174void llvm::UpgradeARCRuntime(Module &M) {
7175 // This lambda converts normal function calls to ARC runtime functions to
7176 // intrinsic calls.
7177 auto UpgradeToIntrinsic = [&](const char *OldFunc,
7178 llvm::Intrinsic::ID IntrinsicFunc) {
7179 Function *Fn = M.getFunction(Name: OldFunc);
7180
7181 if (!Fn)
7182 return;
7183
7184 Function *NewFn =
7185 llvm::Intrinsic::getOrInsertDeclaration(M: &M, id: IntrinsicFunc);
7186
7187 for (User *U : make_early_inc_range(Range: Fn->users())) {
7188 CallInst *CI = dyn_cast<CallInst>(Val: U);
7189 if (!CI || CI->getCalledFunction() != Fn)
7190 continue;
7191
7192 IRBuilder<> Builder(CI->getIterator());
7193 FunctionType *NewFuncTy = NewFn->getFunctionType();
7194 SmallVector<Value *, 2> Args;
7195
7196 // Don't upgrade the intrinsic if it's not valid to bitcast the return
7197 // value to the return type of the old function.
7198 if (NewFuncTy->getReturnType() != CI->getType() &&
7199 !CastInst::castIsValid(op: Instruction::BitCast, S: CI,
7200 DstTy: NewFuncTy->getReturnType()))
7201 continue;
7202
7203 bool InvalidCast = false;
7204
7205 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
7206 Value *Arg = CI->getArgOperand(i: I);
7207
7208 // Bitcast argument to the parameter type of the new function if it's
7209 // not a variadic argument.
7210 if (I < NewFuncTy->getNumParams()) {
7211 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
7212 // to the parameter type of the new function.
7213 if (!CastInst::castIsValid(op: Instruction::BitCast, S: Arg,
7214 DstTy: NewFuncTy->getParamType(i: I))) {
7215 InvalidCast = true;
7216 break;
7217 }
7218 Arg = Builder.CreateBitCast(V: Arg, DestTy: NewFuncTy->getParamType(i: I));
7219 }
7220 Args.push_back(Elt: Arg);
7221 }
7222
7223 if (InvalidCast)
7224 continue;
7225
7226 // Create a call instruction that calls the new function.
7227 CallInst *NewCall = Builder.CreateCall(FTy: NewFuncTy, Callee: NewFn, Args);
7228 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
7229 NewCall->takeName(V: CI);
7230
7231 // Bitcast the return value back to the type of the old call.
7232 Value *NewRetVal = Builder.CreateBitCast(V: NewCall, DestTy: CI->getType());
7233
7234 if (!CI->use_empty())
7235 CI->replaceAllUsesWith(V: NewRetVal);
7236 CI->eraseFromParent();
7237 }
7238
7239 if (Fn->use_empty())
7240 Fn->eraseFromParent();
7241 };
7242
7243 // Unconditionally convert a call to "clang.arc.use" to a call to
7244 // "llvm.objc.clang.arc.use".
7245 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7246
7247 // Upgrade the retain release marker. If there is no need to upgrade
7248 // the marker, that means either the module is already new enough to contain
7249 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7250 if (!upgradeRetainReleaseMarker(M))
7251 return;
7252
7253 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7254 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7255 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7256 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7257 {"objc_autoreleaseReturnValue",
7258 llvm::Intrinsic::objc_autoreleaseReturnValue},
7259 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7260 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7261 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7262 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7263 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7264 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7265 {"objc_release", llvm::Intrinsic::objc_release},
7266 {"objc_retain", llvm::Intrinsic::objc_retain},
7267 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7268 {"objc_retainAutoreleaseReturnValue",
7269 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7270 {"objc_retainAutoreleasedReturnValue",
7271 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7272 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7273 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7274 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7275 {"objc_unsafeClaimAutoreleasedReturnValue",
7276 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7277 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7278 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7279 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7280 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7281 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7282 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7283 {"objc_arc_annotation_topdown_bbstart",
7284 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7285 {"objc_arc_annotation_topdown_bbend",
7286 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7287 {"objc_arc_annotation_bottomup_bbstart",
7288 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7289 {"objc_arc_annotation_bottomup_bbend",
7290 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7291
7292 for (auto &I : RuntimeFuncs)
7293 UpgradeToIntrinsic(I.first, I.second);
7294}
7295
7296// Upgrade the way signing of pointers to init/fini functions is described.
7297//
7298// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7299// constants, if signing was requested. After the upgrade, these arrays contain
7300// plain function pointers and the desired signing schema is described via a
7301// pair of module flags.
7302//
7303// Note that the upgrade is only performed if all elements of *both* arrays
7304// agree on a common signing schema.
7305static bool upgradePtrauthInitFiniArrays(Module &M) {
7306 // As we cannot always decide whether the particular module should have
7307 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7308 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7309 // returns as soon as it spots any non-signed init/fini pointer: either we
7310 // should request non-signed pointers (safe to omit both flags) or there is
7311 // no common schema (and thus we do not modify anything).
7312 //
7313 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7314 // or whether we should request address diversity in addition to the basic
7315 // constant diversity. There is no value representing "decided not to sign"
7316 // for the reasons explained above.
7317 std::optional<bool> UseAddressDisc;
7318
7319 // Do not attempt upgrading if the new module flags already exist.
7320 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7321 for (const MDNode *Flag : ModFlags->operands()) {
7322 std::optional<StringRef> Name = getModuleFlagNameSafely(Flag: *Flag);
7323 if (Name && (*Name == "ptrauth-init-fini" ||
7324 *Name == "ptrauth-init-fini-address-discrimination"))
7325 return false;
7326 }
7327 }
7328
7329 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7330 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7331 constexpr unsigned ExpectedAddressMarker = 1;
7332
7333 auto *CPA = dyn_cast<ConstantPtrAuth>(Val: CV);
7334 if (!CPA || !CPA->getDiscriminator()->equalsInt(V: ExpectedConstDisc))
7335 return nullptr; // Nothing to upgrade or unknown pattern found.
7336
7337 bool HasAddressDisc;
7338 if (!CPA->hasAddressDiscriminator())
7339 HasAddressDisc = false;
7340 else if (CPA->hasSpecialAddressDiscriminator(Value: ExpectedAddressMarker))
7341 HasAddressDisc = true;
7342 else
7343 return nullptr; // Unknown pattern.
7344
7345 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7346 return nullptr; // Disagreement with the decided mode.
7347
7348 UseAddressDisc = HasAddressDisc;
7349 return CPA->getPointer();
7350 };
7351
7352 // Do not apply any changes until we know the upgrade is non-ambiguous.
7353 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7354 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7355
7356 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7357 auto *GV = dyn_cast_if_present<GlobalVariable>(Val: M.getNamedValue(Name));
7358 if (!GV || !GV->hasInitializer())
7359 continue; // Skip, but it is okay to upgrade the other variable.
7360
7361 auto *OldStructorsArray = dyn_cast<ConstantArray>(Val: GV->getInitializer());
7362 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7363 return false;
7364
7365 std::vector<Constant *> NewStructors;
7366 NewStructors.reserve(n: OldStructorsArray->getNumOperands());
7367
7368 for (Use &U : OldStructorsArray->operands()) {
7369 ConstantStruct *Structor = dyn_cast<ConstantStruct>(Val: U.get());
7370 if (!Structor || Structor->getNumOperands() != 3)
7371 return false;
7372
7373 Constant *Prio = Structor->getOperand(i_nocapture: 0);
7374 Constant *Func = Structor->getOperand(i_nocapture: 1);
7375 Constant *Arg = Structor->getOperand(i_nocapture: 2);
7376
7377 Func = UpgradeSinglePointer(Func);
7378 if (!Func)
7379 return false;
7380
7381 NewStructors.push_back(
7382 x: ConstantStruct::get(T: Structor->getType(), V: {Prio, Func, Arg}));
7383 }
7384
7385 Constant *NewInit =
7386 ConstantArray::get(T: OldStructorsArray->getType(), V: NewStructors);
7387 GlobalArraysToUpgrade.emplace_back(Args&: GV, Args&: NewInit);
7388 }
7389
7390 if (GlobalArraysToUpgrade.empty())
7391 return false;
7392 assert(UseAddressDisc.has_value());
7393
7394 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7395 GV->setInitializer(NewInit);
7396
7397 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini", Val: 1);
7398 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini-address-discrimination",
7399 Val: *UseAddressDisc);
7400
7401 return true;
7402}
7403
7404bool llvm::UpgradeModuleFlags(Module &M) {
7405 bool Changed = false;
7406 Changed |= upgradePtrauthInitFiniArrays(M);
7407
7408 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7409 if (!ModFlags)
7410 return Changed;
7411
7412 bool HasObjCFlag = false, HasClassProperties = false;
7413 bool HasClassROSigning = false;
7414 bool HasSwiftVersionFlag = false;
7415 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7416 uint32_t SwiftABIVersion;
7417 auto Int8Ty = Type::getInt8Ty(C&: M.getContext());
7418 auto Int32Ty = Type::getInt32Ty(C&: M.getContext());
7419
7420 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7421 MDNode *Op = ModFlags->getOperand(i: I);
7422 if (Op->getNumOperands() != 3)
7423 continue;
7424 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
7425 if (!ID)
7426 continue;
7427 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7428 Metadata *Ops[3] = {ConstantAsMetadata::get(C: ConstantInt::get(
7429 Ty: Type::getInt32Ty(C&: M.getContext()), V: B)),
7430 MDString::get(Context&: M.getContext(), Str: ID->getString()),
7431 Op->getOperand(I: 2)};
7432 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7433 Changed = true;
7434 };
7435
7436 if (ID->getString() == "Objective-C Image Info Version")
7437 HasObjCFlag = true;
7438 if (ID->getString() == "Objective-C Class Properties")
7439 HasClassProperties = true;
7440 if (ID->getString() == "Objective-C Enforce ClassRO Pointer Signing")
7441 HasClassROSigning = true;
7442 // Upgrade PIC from Error/Max to Min.
7443 if (ID->getString() == "PIC Level") {
7444 if (auto *Behavior =
7445 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
7446 uint64_t V = Behavior->getLimitedValue();
7447 if (V == Module::Error || V == Module::Max)
7448 SetBehavior(Module::Min);
7449 }
7450 }
7451 // Upgrade "PIE Level" from Error to Max.
7452 if (ID->getString() == "PIE Level")
7453 if (auto *Behavior =
7454 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0)))
7455 if (Behavior->getLimitedValue() == Module::Error)
7456 SetBehavior(Module::Max);
7457
7458 // Upgrade branch protection and return address signing module flags. The
7459 // module flag behavior for these fields were Error and now they are Min.
7460 // The one exception is "sign-return-address-harden".
7461 if (ID->getString() == "branch-target-enforcement" ||
7462 (ID->getString().starts_with(Prefix: "sign-return-address") &&
7463 ID->getString() != "sign-return-address-harden")) {
7464 if (auto *Behavior =
7465 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
7466 if (Behavior->getLimitedValue() == Module::Error) {
7467 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
7468 Metadata *Ops[3] = {
7469 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Module::Min)),
7470 Op->getOperand(I: 1), Op->getOperand(I: 2)};
7471 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7472 Changed = true;
7473 }
7474 }
7475 }
7476
7477 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7478 // section name so that llvm-lto will not complain about mismatching
7479 // module flags that is functionally the same.
7480 if (ID->getString() == "Objective-C Image Info Section") {
7481 if (auto *Value = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2))) {
7482 SmallVector<StringRef, 4> ValueComp;
7483 Value->getString().split(A&: ValueComp, Separator: " ");
7484 if (ValueComp.size() != 1) {
7485 std::string NewValue;
7486 for (auto &S : ValueComp)
7487 NewValue += S.str();
7488 Metadata *Ops[3] = {Op->getOperand(I: 0), Op->getOperand(I: 1),
7489 MDString::get(Context&: M.getContext(), Str: NewValue)};
7490 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7491 Changed = true;
7492 }
7493 }
7494 }
7495
7496 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7497 // If the higher bits are set, it adds new module flag for swift info.
7498 if (ID->getString() == "Objective-C Garbage Collection") {
7499 auto Md = dyn_cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2));
7500 if (Md) {
7501 assert(Md->getValue() && "Expected non-empty metadata");
7502 auto Type = Md->getValue()->getType();
7503 if (Type == Int8Ty)
7504 continue;
7505 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7506 if ((Val & 0xff) != Val) {
7507 HasSwiftVersionFlag = true;
7508 SwiftABIVersion = (Val & 0xff00) >> 8;
7509 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7510 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7511 }
7512 Metadata *Ops[3] = {
7513 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty,V: Module::Error)),
7514 Op->getOperand(I: 1),
7515 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int8Ty,V: Val & 0xff))};
7516 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7517 Changed = true;
7518 }
7519 }
7520
7521 if (ID->getString() == "amdgpu_code_object_version") {
7522 Metadata *Ops[3] = {
7523 Op->getOperand(I: 0),
7524 MDString::get(Context&: M.getContext(), Str: "amdhsa_code_object_version"),
7525 Op->getOperand(I: 2)};
7526 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7527 Changed = true;
7528 }
7529
7530 // clang/PowerPC used to use "float-abi" to describe the long double format;
7531 // it has been renamed to "long-double-type", with its values changed to the
7532 // corresponding IR floating-point type names.
7533 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7534 StringRef Format;
7535 if (auto *S = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2)))
7536 Format = S->getString();
7537
7538 // The "float-abi" key is now reserved for the target-independent
7539 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7540 // (including unrecognized ones, which were never valid) to the default.
7541 if (!FloatABI::parseABIType(S: Format)) {
7542 LongDoubleFormat NewFormat =
7543 StringSwitch<LongDoubleFormat>(Format)
7544 .Case(S: "ieeequad", Value: LongDoubleFormat::IEEEquad)
7545 .Case(S: "ieeedouble", Value: LongDoubleFormat::IEEEdouble)
7546 .Default(Value: LongDoubleFormat::PPCDoubleDouble);
7547 Metadata *Ops[3] = {
7548 Op->getOperand(I: 0),
7549 MDString::get(Context&: M.getContext(), Str: "long-double-type"),
7550 MDString::get(Context&: M.getContext(), Str: getLongDoubleFormatName(Format: NewFormat))};
7551 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7552 Changed = true;
7553 }
7554 }
7555 }
7556
7557 // "Objective-C Class Properties" is recently added for Objective-C. We
7558 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7559 // flag of value 0, so we can correclty downgrade this flag when trying to
7560 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7561 // this module flag.
7562 if (HasObjCFlag && !HasClassProperties) {
7563 M.addModuleFlag(Behavior: llvm::Module::Override, Key: "Objective-C Class Properties",
7564 Val: (uint32_t)0);
7565 Changed = true;
7566 }
7567
7568 // Likewise, ObjC bitcode from before "Objective-C Enforce ClassRO Pointer
7569 // Signing" was added does not sign class_ro_t pointers. Give it the flag with
7570 // value 0, so that linking it with bitcode that signs them does not set
7571 // SignedClassRO.
7572 if (HasObjCFlag && !HasClassROSigning) {
7573 M.addModuleFlag(Behavior: Module::Override,
7574 Key: "Objective-C Enforce ClassRO Pointer Signing", Val: (uint32_t)0);
7575 Changed = true;
7576 }
7577
7578 if (HasSwiftVersionFlag) {
7579 M.addModuleFlag(Behavior: Module::Error, Key: "Swift ABI Version",
7580 Val: SwiftABIVersion);
7581 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Major Version",
7582 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMajorVersion));
7583 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Minor Version",
7584 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMinorVersion));
7585 Changed = true;
7586 }
7587
7588 return Changed;
7589}
7590
7591bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7592 NamedMDNode *CFIConsts = M.getNamedMetadata(Name: "cfi.functions");
7593 // If this metadata has operands, we expect all of them to be either from
7594 // before or from after the format change handled here, so we can bail out
7595 // fast if the first (if any) operands is of the new format.
7596 auto MatchesVersion = [](const MDNode *Op) {
7597 return Op->getNumOperands() >= 3 &&
7598 isa<ConstantAsMetadata>(Val: Op->getOperand(I: 2)) &&
7599 cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2))
7600 ->getType()
7601 ->isIntegerTy(BitWidth: 64);
7602 };
7603
7604 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7605 MatchesVersion(CFIConsts->getOperand(i: 0)))
7606 return false;
7607
7608 bool Changed = false;
7609 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7610 MDNode *Op = CFIConsts->getOperand(i: I);
7611 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7612 assert(Op->getNumOperands() >= 2 &&
7613 "Expected at least 2 operands - name and linkage type");
7614 MDString *NameMD = dyn_cast<MDString>(Val: Op->getOperand(I: 0));
7615 StringRef Name = NameMD->getString();
7616 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7617 GlobalName: GlobalValue::dropLLVMManglingEscape(Name));
7618
7619 SmallVector<Metadata *, 4> Elts;
7620 Elts.push_back(Elt: Op->getOperand(I: 0));
7621 Elts.push_back(Elt: Op->getOperand(I: 1));
7622 Elts.push_back(Elt: ConstantAsMetadata::get(
7623 C: ConstantInt::get(Ty: Type::getInt64Ty(C&: M.getContext()), V: GUID)));
7624
7625 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7626 Elts.push_back(Elt: Op->getOperand(I: J));
7627
7628 CFIConsts->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Elts));
7629 Changed = true;
7630 }
7631
7632 return Changed;
7633}
7634
7635void llvm::UpgradeSectionAttributes(Module &M) {
7636 auto TrimSpaces = [](StringRef Section) -> std::string {
7637 SmallVector<StringRef, 5> Components;
7638 Section.split(A&: Components, Separator: ',');
7639
7640 SmallString<32> Buffer;
7641 raw_svector_ostream OS(Buffer);
7642
7643 for (auto Component : Components)
7644 OS << ',' << Component.trim();
7645
7646 return std::string(OS.str().substr(Start: 1));
7647 };
7648
7649 for (auto &GV : M.globals()) {
7650 if (!GV.hasSection())
7651 continue;
7652
7653 StringRef Section = GV.getSection();
7654
7655 if (!Section.starts_with(Prefix: "__DATA, __objc_catlist"))
7656 continue;
7657
7658 // __DATA, __objc_catlist, regular, no_dead_strip
7659 // __DATA,__objc_catlist,regular,no_dead_strip
7660 GV.setSection(TrimSpaces(Section));
7661 }
7662}
7663
7664void llvm::UpgradeInstructionMetadata(Module &M) {
7665 DenseMap<MDNode *, MDNode *> UpgradedAliasScopes;
7666
7667 for (Function &F : M)
7668 for (Instruction &I : instructions(F)) {
7669 for (auto Kind : {LLVMContext::MD_alias_scope, LLVMContext::MD_noalias}) {
7670 if (MDNode *Old = I.getMetadata(KindID: Kind)) {
7671 MDNode *New = UpgradeAliasScopeList(ScopeList&: *Old, UpgradedAliasScopes);
7672 if (New != Old)
7673 I.setMetadata(KindID: Kind, Node: New);
7674 }
7675 }
7676 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(Val: &I)) {
7677 unsigned ArgNo = Intrinsic::NoAliasScopeDeclScopeArg;
7678 if (Decl->getNumOperands() > ArgNo)
7679 if (auto *ListAsValue =
7680 dyn_cast<MetadataAsValue>(Val: Decl->getOperand(i_nocapture: ArgNo)))
7681 if (auto *List = dyn_cast<MDNode>(Val: ListAsValue->getMetadata())) {
7682 MDNode *New = UpgradeAliasScopeList(ScopeList&: *List, UpgradedAliasScopes);
7683 if (New != List)
7684 Decl->setOperand(i_nocapture: ArgNo,
7685 Val_nocapture: MetadataAsValue::get(Context&: Decl->getContext(), MD: New));
7686 }
7687 }
7688 }
7689}
7690
7691namespace {
7692// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7693// callsites within a function that did not also have the strictfp attribute.
7694// Since 10.0, if strict FP semantics are needed within a function, the
7695// function must have the strictfp attribute and all calls within the function
7696// must also have the strictfp attribute. This latter restriction is
7697// necessary to prevent unwanted libcall simplification when a function is
7698// being cloned (such as for inlining).
7699//
7700// The "dangling" strictfp attribute usage was only used to prevent constant
7701// folding and other libcall simplification. The nobuiltin attribute on the
7702// callsite has the same effect.
7703struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7704 StrictFPUpgradeVisitor() = default;
7705
7706 void visitCallBase(CallBase &Call) {
7707 if (!Call.isStrictFP())
7708 return;
7709 if (isa<ConstrainedFPIntrinsic>(Val: &Call))
7710 return;
7711 // If we get here, the caller doesn't have the strictfp attribute
7712 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7713 Call.removeFnAttr(Kind: Attribute::StrictFP);
7714 Call.addFnAttr(Kind: Attribute::NoBuiltin);
7715 }
7716};
7717
7718/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7719struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7720 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7721 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7722
7723 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7724 if (!RMW.isFloatingPointOperation())
7725 return;
7726
7727 MDNode *Empty = MDNode::get(Context&: RMW.getContext(), MDs: {});
7728 RMW.setMetadata(Kind: "amdgpu.no.fine.grained.host.memory", Node: Empty);
7729 RMW.setMetadata(Kind: "amdgpu.no.remote.memory.access", Node: Empty);
7730 RMW.setMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode, Node: Empty);
7731 }
7732};
7733} // namespace
7734
7735void llvm::UpgradeFunctionAttributes(Function &F) {
7736 // If a function definition doesn't have the strictfp attribute,
7737 // convert any callsite strictfp attributes to nobuiltin.
7738 if (!F.isDeclaration() && !F.hasFnAttribute(Kind: Attribute::StrictFP)) {
7739 StrictFPUpgradeVisitor SFPV;
7740 SFPV.visit(F);
7741 }
7742
7743 // Remove all incompatibile attributes from function.
7744 F.removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
7745 Ty: F.getReturnType(), AS: F.getAttributes().getRetAttrs()));
7746 for (auto &Arg : F.args())
7747 Arg.removeAttrs(
7748 AM: AttributeFuncs::typeIncompatible(Ty: Arg.getType(), AS: Arg.getAttributes()));
7749
7750 bool AddingAttrs = false, RemovingAttrs = false;
7751 AttrBuilder AttrsToAdd(F.getContext());
7752 AttributeMask AttrsToRemove;
7753
7754 // Older versions of LLVM treated an "implicit-section-name" attribute
7755 // similarly to directly setting the section on a Function.
7756 if (Attribute A = F.getFnAttribute(Kind: "implicit-section-name");
7757 A.isValid() && A.isStringAttribute()) {
7758 F.setSection(A.getValueAsString());
7759 AttrsToRemove.addAttribute(A: "implicit-section-name");
7760 RemovingAttrs = true;
7761 }
7762
7763 if (Attribute A = F.getFnAttribute(Kind: "nooutline");
7764 A.isValid() && A.isStringAttribute()) {
7765 AttrsToRemove.addAttribute(A: "nooutline");
7766 AttrsToAdd.addAttribute(Val: Attribute::NoOutline);
7767 AddingAttrs = RemovingAttrs = true;
7768 }
7769
7770 if (Attribute A = F.getFnAttribute(Kind: "uniform-work-group-size");
7771 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7772 AttrsToRemove.addAttribute(A: "uniform-work-group-size");
7773 RemovingAttrs = true;
7774 if (A.getValueAsString() == "true") {
7775 AttrsToAdd.addAttribute(A: "uniform-work-group-size");
7776 AddingAttrs = true;
7777 }
7778 }
7779
7780 if (!F.empty()) {
7781 // For some reason this is called twice, and the first time is before any
7782 // instructions are loaded into the body.
7783
7784 if (Attribute A = F.getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics");
7785 A.isValid()) {
7786
7787 if (A.getValueAsBool()) {
7788 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7789 Visitor.visit(F);
7790 }
7791
7792 // We will leave behind dead attribute uses on external declarations, but
7793 // clang never added these to declarations anyway.
7794 AttrsToRemove.addAttribute(A: "amdgpu-unsafe-fp-atomics");
7795 RemovingAttrs = true;
7796 }
7797 }
7798
7799 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7800 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7801
7802 bool HandleDenormalMode = false;
7803
7804 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math"); Attr.isValid()) {
7805 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7806 if (ParsedMode.isValid()) {
7807 DenormalFPMath = ParsedMode;
7808 AttrsToRemove.addAttribute(A: "denormal-fp-math");
7809 AddingAttrs = RemovingAttrs = true;
7810 HandleDenormalMode = true;
7811 }
7812 }
7813
7814 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math-f32");
7815 Attr.isValid()) {
7816 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7817 if (ParsedMode.isValid()) {
7818 DenormalFPMathF32 = ParsedMode;
7819 AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
7820 AddingAttrs = RemovingAttrs = true;
7821 HandleDenormalMode = true;
7822 }
7823 }
7824
7825 if (HandleDenormalMode)
7826 AttrsToAdd.addDenormalFPEnvAttr(
7827 Mode: DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7828
7829 if (RemovingAttrs)
7830 F.removeFnAttrs(Attrs: AttrsToRemove);
7831
7832 if (AddingAttrs)
7833 F.addFnAttrs(Attrs: AttrsToAdd);
7834}
7835
7836// Check if the function attribute is not present and set it.
7837static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7838 StringRef Value) {
7839 if (!F.hasFnAttribute(Kind: FnAttrName)) {
7840 F.addFnAttr(Kind: FnAttrName, Val: Value);
7841 LLVM_DEBUG(dbgs() << "Set attribute: " << FnAttrName << "=\"" << Value
7842 << "\", function: " << F.getName() << "\n");
7843 }
7844}
7845
7846// Check if the function attribute is not present and set it if needed.
7847// If the attribute is "false" then removes it.
7848// If the attribute is "true" resets it to a valueless attribute.
7849static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7850 if (!F.hasFnAttribute(Kind: FnAttrName)) {
7851 if (Set) {
7852 F.addFnAttr(Kind: FnAttrName);
7853 LLVM_DEBUG(dbgs() << "Added attribute: " << FnAttrName
7854 << ", function: " << F.getName() << "\n");
7855 }
7856 } else {
7857 auto A = F.getFnAttribute(Kind: FnAttrName);
7858 if ("false" == A.getValueAsString()) {
7859 F.removeFnAttr(Kind: FnAttrName);
7860 LLVM_DEBUG(dbgs() << "Removed attribute: " << FnAttrName
7861 << "=\"false\", function: " << F.getName() << "\n");
7862 } else if ("true" == A.getValueAsString()) {
7863 F.removeFnAttr(Kind: FnAttrName);
7864 F.addFnAttr(Kind: FnAttrName);
7865 LLVM_DEBUG(dbgs() << "Converted attribute: " << FnAttrName
7866 << "=\"true\", function: " << F.getName() << "\n");
7867 }
7868 }
7869}
7870
7871static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior,
7872 StringRef Key, uint32_t Val) {
7873 M.setModuleFlag(Behavior, Key, Val);
7874 LLVM_DEBUG(dbgs() << "Converted module flag: " << "{" << Behavior << ", "
7875 << Key << ", " << Val << "}\n");
7876}
7877
7878void llvm::copyModuleAttrToFunctions(Module &M) {
7879 Triple T(M.getTargetTriple());
7880 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7881 return;
7882
7883 uint64_t BTEValue = 0;
7884 uint64_t BPPLRValue = 0;
7885 uint64_t GCSValue = 0;
7886 uint64_t SRAValue = 0;
7887 uint64_t SRAALLValue = 0;
7888 uint64_t SRABKeyValue = 0;
7889
7890 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7891 if (ModFlags) {
7892 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7893 MDNode *Op = ModFlags->getOperand(i: I);
7894 if (Op->getNumOperands() != 3)
7895 continue;
7896
7897 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
7898 auto *CI = mdconst::dyn_extract<ConstantInt>(MD: Op->getOperand(I: 2));
7899 if (!ID || !CI)
7900 continue;
7901
7902 StringRef IDStr = ID->getString();
7903 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7904 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7905 : IDStr == "guarded-control-stack" ? &GCSValue
7906 : IDStr == "sign-return-address" ? &SRAValue
7907 : IDStr == "sign-return-address-all" ? &SRAALLValue
7908 : IDStr == "sign-return-address-with-bkey"
7909 ? &SRABKeyValue
7910 : nullptr;
7911 if (!ValPtr)
7912 continue;
7913
7914 *ValPtr = CI->getZExtValue();
7915 if (*ValPtr == 2)
7916 return;
7917
7918 LLVM_DEBUG(dbgs() << "Found module flag: " << IDStr << "(" << *ValPtr
7919 << ")\n");
7920 }
7921 }
7922
7923 bool BTE = BTEValue == 1;
7924 bool BPPLR = BPPLRValue == 1;
7925 bool GCS = GCSValue == 1;
7926 bool SRA = SRAValue == 1;
7927
7928 StringRef SignTypeValue = "non-leaf";
7929 if (SRA && SRAALLValue == 1)
7930 SignTypeValue = "all";
7931
7932 StringRef SignKeyValue = "a_key";
7933 if (SRA && SRABKeyValue == 1)
7934 SignKeyValue = "b_key";
7935
7936 for (Function &F : M.getFunctionList()) {
7937 if (F.isDeclaration())
7938 continue;
7939
7940 if (SRA) {
7941 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address", Value: SignTypeValue);
7942 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address-key", Value: SignKeyValue);
7943 } else {
7944 if (auto A = F.getFnAttribute(Kind: "sign-return-address");
7945 A.isValid() && "none" == A.getValueAsString()) {
7946 F.removeFnAttr(Kind: "sign-return-address");
7947 F.removeFnAttr(Kind: "sign-return-address-key");
7948 }
7949 }
7950 ConvertFunctionAttr(F, Set: BTE, FnAttrName: "branch-target-enforcement");
7951 ConvertFunctionAttr(F, Set: BPPLR, FnAttrName: "branch-protection-pauth-lr");
7952 ConvertFunctionAttr(F, Set: GCS, FnAttrName: "guarded-control-stack");
7953 }
7954
7955 if (BTE)
7956 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "branch-target-enforcement", Val: 2);
7957 if (BPPLR)
7958 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "branch-protection-pauth-lr", Val: 2);
7959 if (GCS)
7960 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "guarded-control-stack", Val: 2);
7961 if (SRA) {
7962 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "sign-return-address", Val: 2);
7963 if (SRAALLValue == 1)
7964 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "sign-return-address-all", Val: 2);
7965 if (SRABKeyValue == 1) {
7966 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "sign-return-address-with-bkey",
7967 Val: 2);
7968 }
7969 }
7970}
7971
7972/// Return the replacement tags if \p T still uses a removed two-operand form.
7973static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7974 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
7975 return nullptr;
7976 auto *Tag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7977 return Tag ? findBooleanLoopTags(Name: Tag->getString()) : nullptr;
7978}
7979
7980/// Build the single-operand node that replaces a boolean operand: nonzero
7981/// selects the enable tag, zero the disable tag.
7982static Metadata *makeBooleanLoopNode(LLVMContext &C,
7983 const BooleanLoopTags &Tags,
7984 const MDOperand &Op) {
7985 bool Enable = !mdconst::extract<ConstantInt>(MD: Op)->isZero();
7986 return MDTuple::get(Context&: C,
7987 MDs: {MDString::get(Context&: C, Str: Enable ? Tags.Enable : Tags.Disable)});
7988}
7989
7990static bool isOldLoopArgument(Metadata *MD) {
7991 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
7992 if (!T)
7993 return false;
7994 if (T->getNumOperands() < 1)
7995 return false;
7996 auto *S = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7997 if (!S)
7998 return false;
7999 if (S->getString().starts_with(Prefix: "llvm.vectorizer."))
8000 return true;
8001 return getOldBooleanLoopTags(T) != nullptr;
8002}
8003
8004static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
8005 StringRef OldPrefix = "llvm.vectorizer.";
8006 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
8007
8008 if (OldTag == "llvm.vectorizer.unroll")
8009 return MDString::get(Context&: C, Str: "llvm.loop.interleave.count");
8010
8011 return MDString::get(
8012 Context&: C, Str: (Twine("llvm.loop.vectorize.") + OldTag.drop_front(N: OldPrefix.size()))
8013 .str());
8014}
8015
8016static Metadata *upgradeLoopArgument(Metadata *MD) {
8017 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
8018 if (!T)
8019 return MD;
8020 if (T->getNumOperands() < 1)
8021 return MD;
8022 auto *OldTag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
8023 if (!OldTag)
8024 return MD;
8025
8026 LLVMContext &C = T->getContext();
8027
8028 /// Rewrite a removed two-operand boolean form to the single-operand pair.
8029 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
8030 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
8031
8032 if (!OldTag->getString().starts_with(Prefix: "llvm.vectorizer."))
8033 return MD;
8034
8035 // This has an old tag. Upgrade it.
8036 MDString *NewTag = upgradeLoopTag(C, OldTag: OldTag->getString());
8037
8038 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
8039 // vectorize.enable/disable pair, not a two-operand enable node.
8040 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
8041 if (const BooleanLoopTags *Tags = findBooleanLoopTags(Name: NewTag->getString()))
8042 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
8043
8044 SmallVector<Metadata *, 8> Ops;
8045 Ops.reserve(N: T->getNumOperands());
8046 Ops.push_back(Elt: NewTag);
8047 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
8048 Ops.push_back(Elt: T->getOperand(I));
8049
8050 return MDTuple::get(Context&: C, MDs: Ops);
8051}
8052
8053MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
8054 auto *T = dyn_cast<MDTuple>(Val: &N);
8055 if (!T)
8056 return &N;
8057
8058 if (none_of(Range: T->operands(), P: isOldLoopArgument))
8059 return &N;
8060
8061 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
8062 // any MDNode carrying those tags with more than one operand, so a leftover
8063 // reference (from the distinct loop-ID) would still trigger a diagnostic.
8064 // In-place mutation is safe on distinct MDNodes.
8065 if (T->isDistinct()) {
8066 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
8067 auto *OpT = dyn_cast_or_null<MDTuple>(Val: T->getOperand(I));
8068 if (OpT && getOldBooleanLoopTags(T: OpT))
8069 T->replaceOperandWith(I, New: upgradeLoopArgument(MD: OpT));
8070 }
8071 if (none_of(Range: T->operands(), P: isOldLoopArgument))
8072 return &N;
8073 }
8074
8075 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
8076 // attachment; the original distinct loop-ID is kept as the first operand.
8077 SmallVector<Metadata *, 8> Ops;
8078 Ops.reserve(N: T->getNumOperands());
8079 for (Metadata *MD : T->operands())
8080 Ops.push_back(Elt: upgradeLoopArgument(MD));
8081
8082 return MDTuple::get(Context&: T->getContext(), MDs: Ops);
8083}
8084
8085std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
8086 Triple T(TT);
8087 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
8088 // the address space of globals to 1. This does not apply to SPIRV Logical.
8089 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
8090 !DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G")) {
8091 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
8092 }
8093
8094 if (T.isLoongArch64() || T.isRISCV64()) {
8095 // Make i32 a native type for 64-bit LoongArch and RISC-V.
8096 auto I = DL.find(Str: "-n64-");
8097 if (I != StringRef::npos)
8098 return (DL.take_front(N: I) + "-n32:64-" + DL.drop_front(N: I + 5)).str();
8099 return DL.str();
8100 }
8101
8102 // AMDGPU data layout upgrades.
8103 std::string Res = DL.str();
8104 if (T.isAMDGPU()) {
8105 // Define address spaces for constants.
8106 if (!DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G"))
8107 Res.append(s: Res.empty() ? "G1" : "-G1");
8108
8109 // AMDGCN data layout upgrades.
8110 if (T.isAMDGCN()) {
8111
8112 // Add missing non-integral declarations.
8113 // This goes before adding new address spaces to prevent incoherent string
8114 // values.
8115 if (!DL.contains(Other: "-ni") && !DL.starts_with(Prefix: "ni"))
8116 Res.append(s: "-ni:7:8:9");
8117 // Update ni:7 to ni:7:8:9.
8118 if (DL.ends_with(Suffix: "ni:7"))
8119 Res.append(s: ":8:9");
8120 if (DL.ends_with(Suffix: "ni:7:8"))
8121 Res.append(s: ":9");
8122
8123 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
8124 // resources) An empty data layout has already been upgraded to G1 by now.
8125 if (!DL.contains(Other: "-p7") && !DL.starts_with(Prefix: "p7"))
8126 Res.append(s: "-p7:160:256:256:32");
8127 if (!DL.contains(Other: "-p8") && !DL.starts_with(Prefix: "p8"))
8128 Res.append(s: "-p8:128:128:128:48");
8129 constexpr StringRef OldP8("-p8:128:128-");
8130 if (DL.contains(Other: OldP8))
8131 Res.replace(pos: Res.find(svt: OldP8), n1: OldP8.size(), s: "-p8:128:128:128:48-");
8132 if (!DL.contains(Other: "-p9") && !DL.starts_with(Prefix: "p9"))
8133 Res.append(s: "-p9:192:256:256:32");
8134
8135 // Add sizing for address space 10 through 15.
8136 // AS 10, 11 and 14 are reserved and defaulted to 32:32
8137 // AS 12 is reserved for downstream and keeps the default (64-bit) size.
8138 // AS 13 and 15 are in use and are 32:32.
8139 for (StringRef AS : {"p10", "p11", "p13", "p14", "p15"}) {
8140 if (!DL.contains(Other: ("-" + AS).str()) && !DL.starts_with(Prefix: AS))
8141 Res.append(str: ("-" + AS + ":32:32").str());
8142 }
8143 }
8144
8145 // Upgrade the ELF mangling mode.
8146 if (!DL.contains(Other: "m:e"))
8147 Res = Res.empty() ? "m:e" : "m:e-" + Res;
8148
8149 return Res;
8150 }
8151
8152 if (T.isSystemZ() && !DL.empty()) {
8153 // Make sure the stack alignment is present.
8154 if (!DL.contains(Other: "-S64"))
8155 return "E-S64" + DL.drop_front(N: 1).str();
8156 return DL.str();
8157 }
8158
8159 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
8160 // If the datalayout matches the expected format, add pointer size address
8161 // spaces to the datalayout.
8162 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
8163 if (!DL.contains(Other: AddrSpaces)) {
8164 SmallVector<StringRef, 4> Groups;
8165 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
8166 if (R.match(String: Res, Matches: &Groups))
8167 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
8168 }
8169 };
8170
8171 // AArch64 data layout upgrades.
8172 if (T.isAArch64()) {
8173 // Add "-Fn32"
8174 if (!DL.empty() && !DL.contains(Other: "-Fn32"))
8175 Res.append(s: "-Fn32");
8176 AddPtr32Ptr64AddrSpaces();
8177 return Res;
8178 }
8179
8180 if (T.isSPARC() || (T.isMIPS64() && !DL.contains(Other: "m:m")) || T.isPPC64() ||
8181 T.isWasm()) {
8182 // Mips64 with o32 ABI did not add "-i128:128".
8183 // Add "-i128:128"
8184 std::string I64 = "-i64:64";
8185 std::string I128 = "-i128:128";
8186 if (!StringRef(Res).contains(Other: I128)) {
8187 size_t Pos = Res.find(str: I64);
8188 if (Pos != size_t(-1))
8189 Res.insert(pos1: Pos + I64.size(), str: I128);
8190 }
8191 }
8192
8193 if (T.isPPC() && T.isOSAIX() && !DL.contains(Other: "f64:32:64") && !DL.empty()) {
8194 size_t Pos = Res.find(s: "-S128");
8195 if (Pos == StringRef::npos)
8196 Pos = Res.size();
8197 Res.insert(pos: Pos, s: "-f64:32:64");
8198 }
8199
8200 // ARM data layout upgrades.
8201 // Add -Fi8 if a -F has not already been specified.
8202 if (T.isARM() && !DL.empty() && !DL.contains(Other: "Fi") && !DL.contains(Other: "Fn")) {
8203 const StringRef p3232 = "p:32:32";
8204 size_t Pos = Res.find(svt: p3232);
8205 if (Pos != StringRef::npos)
8206 Res.insert(pos: Pos + p3232.size(), s: "-Fi8");
8207 }
8208
8209 if (!T.isX86())
8210 return Res;
8211
8212 AddPtr32Ptr64AddrSpaces();
8213
8214 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
8215 // for i128 operations prior to this being reflected in the data layout, and
8216 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
8217 // boundaries, so although this is a breaking change, the upgrade is expected
8218 // to fix more IR than it breaks.
8219 // Intel MCU is an exception and uses 4-byte-alignment.
8220 if (!T.isOSIAMCU()) {
8221 std::string I128 = "-i128:128";
8222 if (StringRef Ref = Res; !Ref.contains(Other: I128)) {
8223 SmallVector<StringRef, 4> Groups;
8224 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
8225 if (R.match(String: Res, Matches: &Groups))
8226 Res = (Groups[1] + I128 + Groups[3]).str();
8227 }
8228 }
8229
8230 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
8231 // Raising the alignment is safe because Clang did not produce f80 values in
8232 // the MSVC environment before this upgrade was added.
8233 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
8234 StringRef Ref = Res;
8235 auto I = Ref.find(Str: "-f80:32-");
8236 if (I != StringRef::npos)
8237 Res = (Ref.take_front(N: I) + "-f80:128-" + Ref.drop_front(N: I + 8)).str();
8238 }
8239
8240 return Res;
8241}
8242
8243void llvm::UpgradeAttributes(AttrBuilder &B) {
8244 StringRef FramePointer;
8245 Attribute A = B.getAttribute(Kind: "no-frame-pointer-elim");
8246 if (A.isValid()) {
8247 // The value can be "true" or "false".
8248 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
8249 B.removeAttribute(A: "no-frame-pointer-elim");
8250 }
8251 if (B.contains(A: "no-frame-pointer-elim-non-leaf")) {
8252 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
8253 if (FramePointer != "all")
8254 FramePointer = "non-leaf";
8255 B.removeAttribute(A: "no-frame-pointer-elim-non-leaf");
8256 }
8257 if (!FramePointer.empty())
8258 B.addAttribute(A: "frame-pointer", V: FramePointer);
8259
8260 A = B.getAttribute(Kind: "null-pointer-is-valid");
8261 if (A.isValid()) {
8262 // The value can be "true" or "false".
8263 bool NullPointerIsValid = A.getValueAsString() == "true";
8264 B.removeAttribute(A: "null-pointer-is-valid");
8265 if (NullPointerIsValid)
8266 B.addAttribute(Val: Attribute::NullPointerIsValid);
8267 }
8268
8269 A = B.getAttribute(Kind: "uniform-work-group-size");
8270 if (A.isValid()) {
8271 StringRef Val = A.getValueAsString();
8272 if (!Val.empty()) {
8273 bool IsTrue = Val == "true";
8274 B.removeAttribute(A: "uniform-work-group-size");
8275 if (IsTrue)
8276 B.addAttribute(A: "uniform-work-group-size");
8277 }
8278 }
8279}
8280
8281void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
8282 // clang.arc.attachedcall bundles are now required to have an operand.
8283 // If they don't, it's okay to drop them entirely: when there is an operand,
8284 // the "attachedcall" is meaningful and required, but without an operand,
8285 // it's just a marker NOP. Dropping it merely prevents an optimization.
8286 erase_if(C&: Bundles, P: [&](OperandBundleDef &OBD) {
8287 return OBD.getTag() == "clang.arc.attachedcall" &&
8288 OBD.inputs().empty();
8289 });
8290}
8291