1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "IROptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringSwitch.h"
21#include "llvm/BinaryFormat/Dwarf.h"
22#include "llvm/IR/AttributeMask.h"
23#include "llvm/IR/Attributes.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfo.h"
27#include "llvm/IR/DebugInfoMetadata.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstVisitor.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsAArch64.h"
37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsARM.h"
39#include "llvm/IR/IntrinsicsNVPTX.h"
40#include "llvm/IR/IntrinsicsRISCV.h"
41#include "llvm/IR/IntrinsicsWebAssembly.h"
42#include "llvm/IR/IntrinsicsX86.h"
43#include "llvm/IR/LLVMContext.h"
44#include "llvm/IR/MDBuilder.h"
45#include "llvm/IR/Metadata.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/NVVMIntrinsicUtils.h"
48#include "llvm/IR/Value.h"
49#include "llvm/IR/Verifier.h"
50#include "llvm/Support/AMDGPUAddrSpace.h"
51#include "llvm/Support/CodeGen.h"
52#include "llvm/Support/Debug.h"
53#include "llvm/Support/ErrorHandling.h"
54#include "llvm/Support/NVPTXAddrSpace.h"
55#include "llvm/Support/NVVMAttributes.h"
56#include "llvm/Support/Regex.h"
57#include "llvm/Support/TimeProfiler.h"
58#include "llvm/TargetParser/Triple.h"
59#include <cstdint>
60#include <cstring>
61#include <numeric>
62
63using namespace llvm;
64
65#define DEBUG_TYPE "auto-upgrade"
66
67static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
68
69// Report a fatal error along with the
70// Call Instruction which caused the error
71[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
72 CallBase *CI) {
73 CI->print(O&: llvm::errs());
74 llvm::errs() << "\n";
75 reportFatalUsageError(reason);
76}
77
78// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
79// changed their type from v4f32 to v2i64.
80static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
81 Function *&NewFn) {
82 // Check whether this is an old version of the function, which received
83 // v4f32 arguments.
84 Type *Arg0Type = F->getFunctionType()->getParamType(i: 0);
85 if (Arg0Type != FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4))
86 return false;
87
88 // Yes, it's old, replace it with new version.
89 rename(GV: F);
90 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
91 return true;
92}
93
94// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
95// arguments have changed their type from i32 to i8.
96static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
97 Function *&NewFn) {
98 // Check that the last argument is an i32.
99 Type *LastArgType = F->getFunctionType()->getParamType(
100 i: F->getFunctionType()->getNumParams() - 1);
101 if (!LastArgType->isIntegerTy(BitWidth: 32))
102 return false;
103
104 // Move this function aside and map down.
105 rename(GV: F);
106 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
107 return true;
108}
109
110// Upgrade the declaration of fp compare intrinsics that change return type
111// from scalar to vXi1 mask.
112static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
113 Function *&NewFn) {
114 // Check if the return type is a vector.
115 if (F->getReturnType()->isVectorTy())
116 return false;
117
118 rename(GV: F);
119 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
120 return true;
121}
122
123// Upgrade the declaration of multiply and add bytes intrinsics whose input
124// arguments' types have changed from vectors of i32 to vectors of i8
125static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
126 Function *&NewFn) {
127 // check if input argument type is a vector of i8
128 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
129 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
130 if (Arg1Type->isVectorTy() &&
131 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 8) &&
132 Arg2Type->isVectorTy() &&
133 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 8))
134 return false;
135
136 rename(GV: F);
137 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
138 return true;
139}
140
141// Upgrade the declaration of multipy and add words intrinsics whose input
142// arguments' types have changed to vectors of i32 to vectors of i16
143static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
144 Function *&NewFn) {
145 // check if input argument type is a vector of i16
146 Type *Arg1Type = F->getFunctionType()->getParamType(i: 1);
147 Type *Arg2Type = F->getFunctionType()->getParamType(i: 2);
148 if (Arg1Type->isVectorTy() &&
149 cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: 16) &&
150 Arg2Type->isVectorTy() &&
151 cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: 16))
152 return false;
153
154 rename(GV: F);
155 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
156 return true;
157}
158
159static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
160 Function *&NewFn) {
161 if (F->getReturnType()->getScalarType()->isBFloatTy())
162 return false;
163
164 rename(GV: F);
165 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
166 return true;
167}
168
169static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
170 Function *&NewFn) {
171 if (F->getFunctionType()->getParamType(i: 1)->getScalarType()->isBFloatTy())
172 return false;
173
174 rename(GV: F);
175 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
176 return true;
177}
178
179static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
180 // All of the intrinsics matches below should be marked with which llvm
181 // version started autoupgrading them. At some point in the future we would
182 // like to use this information to remove upgrade code for some older
183 // intrinsics. It is currently undecided how we will determine that future
184 // point.
185 if (Name.consume_front(Prefix: "avx."))
186 return (Name.starts_with(Prefix: "blend.p") || // Added in 3.7
187 Name == "cvt.ps2.pd.256" || // Added in 3.9
188 Name == "cvtdq2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.ps.256" || // Added in 7.0
190 Name.starts_with(Prefix: "movnt.") || // Added in 3.2
191 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
192 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
193 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 3.5
194 Name.starts_with(Prefix: "vbroadcastf128") || // Added in 4.0
195 Name.starts_with(Prefix: "vextractf128.") || // Added in 3.7
196 Name.starts_with(Prefix: "vinsertf128.") || // Added in 3.7
197 Name.starts_with(Prefix: "vperm2f128.") || // Added in 6.0
198 Name.starts_with(Prefix: "vpermil.")); // Added in 3.1
199
200 if (Name.consume_front(Prefix: "avx2."))
201 return (Name == "movntdqa" || // Added in 5.0
202 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
203 Name.starts_with(Prefix: "padds.") || // Added in 8.0
204 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
205 Name.starts_with(Prefix: "pblendd.") || // Added in 3.7
206 Name == "pblendw" || // Added in 3.7
207 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.8
208 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
209 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
210 Name.starts_with(Prefix: "pmax") || // Added in 3.9
211 Name.starts_with(Prefix: "pmin") || // Added in 3.9
212 Name.starts_with(Prefix: "pmovsx") || // Added in 3.9
213 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
214 Name.starts_with(Prefix: "pmulh.w") || // Added in 24.0
215 Name.starts_with(Prefix: "pmulhu.w") || // Added in 24.0
216 Name == "pmul.dq" || // Added in 7.0
217 Name == "pmulu.dq" || // Added in 7.0
218 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
219 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
220 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
221 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
222 Name.starts_with(Prefix: "vbroadcast") || // Added in 3.8
223 Name == "vbroadcasti128" || // Added in 3.7
224 Name == "vextracti128" || // Added in 3.7
225 Name == "vinserti128" || // Added in 3.7
226 Name == "vperm2i128"); // Added in 6.0
227
228 if (Name.consume_front(Prefix: "avx512.")) {
229 if (Name.consume_front(Prefix: "mask."))
230 // 'avx512.mask.*'
231 return (Name.starts_with(Prefix: "add.p") || // Added in 7.0. 128/256 in 4.0
232 Name.starts_with(Prefix: "and.") || // Added in 3.9
233 Name.starts_with(Prefix: "andn.") || // Added in 3.9
234 Name.starts_with(Prefix: "broadcast.s") || // Added in 3.9
235 Name.starts_with(Prefix: "broadcastf32x4.") || // Added in 6.0
236 Name.starts_with(Prefix: "broadcastf32x8.") || // Added in 6.0
237 Name.starts_with(Prefix: "broadcastf64x2.") || // Added in 6.0
238 Name.starts_with(Prefix: "broadcastf64x4.") || // Added in 6.0
239 Name.starts_with(Prefix: "broadcasti32x4.") || // Added in 6.0
240 Name.starts_with(Prefix: "broadcasti32x8.") || // Added in 6.0
241 Name.starts_with(Prefix: "broadcasti64x2.") || // Added in 6.0
242 Name.starts_with(Prefix: "broadcasti64x4.") || // Added in 6.0
243 Name.starts_with(Prefix: "cmp.b") || // Added in 5.0
244 Name.starts_with(Prefix: "cmp.d") || // Added in 5.0
245 Name.starts_with(Prefix: "cmp.q") || // Added in 5.0
246 Name.starts_with(Prefix: "cmp.w") || // Added in 5.0
247 Name.starts_with(Prefix: "compress.b") || // Added in 9.0
248 Name.starts_with(Prefix: "compress.d") || // Added in 9.0
249 Name.starts_with(Prefix: "compress.p") || // Added in 9.0
250 Name.starts_with(Prefix: "compress.q") || // Added in 9.0
251 Name.starts_with(Prefix: "compress.store.") || // Added in 7.0
252 Name.starts_with(Prefix: "compress.w") || // Added in 9.0
253 Name.starts_with(Prefix: "conflict.") || // Added in 9.0
254 Name.starts_with(Prefix: "cvtdq2pd.") || // Added in 4.0
255 Name.starts_with(Prefix: "cvtdq2ps.") || // Added in 7.0 updated 9.0
256 Name == "cvtpd2dq.256" || // Added in 7.0
257 Name == "cvtpd2ps.256" || // Added in 7.0
258 Name == "cvtps2pd.128" || // Added in 7.0
259 Name == "cvtps2pd.256" || // Added in 7.0
260 Name.starts_with(Prefix: "cvtqq2pd.") || // Added in 7.0 updated 9.0
261 Name == "cvtqq2ps.256" || // Added in 9.0
262 Name == "cvtqq2ps.512" || // Added in 9.0
263 Name == "cvttpd2dq.256" || // Added in 7.0
264 Name == "cvttps2dq.128" || // Added in 7.0
265 Name == "cvttps2dq.256" || // Added in 7.0
266 Name.starts_with(Prefix: "cvtudq2pd.") || // Added in 4.0
267 Name.starts_with(Prefix: "cvtudq2ps.") || // Added in 7.0 updated 9.0
268 Name.starts_with(Prefix: "cvtuqq2pd.") || // Added in 7.0 updated 9.0
269 Name == "cvtuqq2ps.256" || // Added in 9.0
270 Name == "cvtuqq2ps.512" || // Added in 9.0
271 Name.starts_with(Prefix: "dbpsadbw.") || // Added in 7.0
272 Name.starts_with(Prefix: "div.p") || // Added in 7.0. 128/256 in 4.0
273 Name.starts_with(Prefix: "expand.b") || // Added in 9.0
274 Name.starts_with(Prefix: "expand.d") || // Added in 9.0
275 Name.starts_with(Prefix: "expand.load.") || // Added in 7.0
276 Name.starts_with(Prefix: "expand.p") || // Added in 9.0
277 Name.starts_with(Prefix: "expand.q") || // Added in 9.0
278 Name.starts_with(Prefix: "expand.w") || // Added in 9.0
279 Name.starts_with(Prefix: "fpclass.p") || // Added in 7.0
280 Name.starts_with(Prefix: "insert") || // Added in 4.0
281 Name.starts_with(Prefix: "load.") || // Added in 3.9
282 Name.starts_with(Prefix: "loadu.") || // Added in 3.9
283 Name.starts_with(Prefix: "lzcnt.") || // Added in 5.0
284 Name.starts_with(Prefix: "max.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with(Prefix: "min.p") || // Added in 7.0. 128/256 in 5.0
286 Name.starts_with(Prefix: "movddup") || // Added in 3.9
287 Name.starts_with(Prefix: "move.s") || // Added in 4.0
288 Name.starts_with(Prefix: "movshdup") || // Added in 3.9
289 Name.starts_with(Prefix: "movsldup") || // Added in 3.9
290 Name.starts_with(Prefix: "mul.p") || // Added in 7.0. 128/256 in 4.0
291 Name.starts_with(Prefix: "or.") || // Added in 3.9
292 Name.starts_with(Prefix: "pabs.") || // Added in 6.0
293 Name.starts_with(Prefix: "packssdw.") || // Added in 5.0
294 Name.starts_with(Prefix: "packsswb.") || // Added in 5.0
295 Name.starts_with(Prefix: "packusdw.") || // Added in 5.0
296 Name.starts_with(Prefix: "packuswb.") || // Added in 5.0
297 Name.starts_with(Prefix: "padd.") || // Added in 4.0
298 Name.starts_with(Prefix: "padds.") || // Added in 8.0
299 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
300 Name.starts_with(Prefix: "palignr.") || // Added in 3.9
301 Name.starts_with(Prefix: "pand.") || // Added in 3.9
302 Name.starts_with(Prefix: "pandn.") || // Added in 3.9
303 Name.starts_with(Prefix: "pavg") || // Added in 6.0
304 Name.starts_with(Prefix: "pbroadcast") || // Added in 6.0
305 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.9
306 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.9
307 Name.starts_with(Prefix: "perm.df.") || // Added in 3.9
308 Name.starts_with(Prefix: "perm.di.") || // Added in 3.9
309 Name.starts_with(Prefix: "permvar.") || // Added in 7.0
310 Name.starts_with(Prefix: "pmaddubs.w.") || // Added in 7.0
311 Name.starts_with(Prefix: "pmaddw.d.") || // Added in 7.0
312 Name.starts_with(Prefix: "pmax") || // Added in 4.0
313 Name.starts_with(Prefix: "pmin") || // Added in 4.0
314 Name == "pmov.qd.256" || // Added in 9.0
315 Name == "pmov.qd.512" || // Added in 9.0
316 Name == "pmov.wb.256" || // Added in 9.0
317 Name == "pmov.wb.512" || // Added in 9.0
318 Name.starts_with(Prefix: "pmovsx") || // Added in 4.0
319 Name.starts_with(Prefix: "pmovzx") || // Added in 4.0
320 Name.starts_with(Prefix: "pmul.dq.") || // Added in 4.0
321 Name.starts_with(Prefix: "pmul.hr.sw.") || // Added in 7.0
322 Name.starts_with(Prefix: "pmulh.w.") || // Added in 7.0
323 Name.starts_with(Prefix: "pmulhu.w.") || // Added in 7.0
324 Name.starts_with(Prefix: "pmull.") || // Added in 4.0
325 Name.starts_with(Prefix: "pmultishift.qb.") || // Added in 8.0
326 Name.starts_with(Prefix: "pmulu.dq.") || // Added in 4.0
327 Name.starts_with(Prefix: "por.") || // Added in 3.9
328 Name.starts_with(Prefix: "prol.") || // Added in 8.0
329 Name.starts_with(Prefix: "prolv.") || // Added in 8.0
330 Name.starts_with(Prefix: "pror.") || // Added in 8.0
331 Name.starts_with(Prefix: "prorv.") || // Added in 8.0
332 Name.starts_with(Prefix: "pshuf.b.") || // Added in 4.0
333 Name.starts_with(Prefix: "pshuf.d.") || // Added in 3.9
334 Name.starts_with(Prefix: "pshufh.w.") || // Added in 3.9
335 Name.starts_with(Prefix: "pshufl.w.") || // Added in 3.9
336 Name.starts_with(Prefix: "psll.d") || // Added in 4.0
337 Name.starts_with(Prefix: "psll.q") || // Added in 4.0
338 Name.starts_with(Prefix: "psll.w") || // Added in 4.0
339 Name.starts_with(Prefix: "pslli") || // Added in 4.0
340 Name.starts_with(Prefix: "psllv") || // Added in 4.0
341 Name.starts_with(Prefix: "psra.d") || // Added in 4.0
342 Name.starts_with(Prefix: "psra.q") || // Added in 4.0
343 Name.starts_with(Prefix: "psra.w") || // Added in 4.0
344 Name.starts_with(Prefix: "psrai") || // Added in 4.0
345 Name.starts_with(Prefix: "psrav") || // Added in 4.0
346 Name.starts_with(Prefix: "psrl.d") || // Added in 4.0
347 Name.starts_with(Prefix: "psrl.q") || // Added in 4.0
348 Name.starts_with(Prefix: "psrl.w") || // Added in 4.0
349 Name.starts_with(Prefix: "psrli") || // Added in 4.0
350 Name.starts_with(Prefix: "psrlv") || // Added in 4.0
351 Name.starts_with(Prefix: "psub.") || // Added in 4.0
352 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
353 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
354 Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
355 Name.starts_with(Prefix: "punpckh") || // Added in 3.9
356 Name.starts_with(Prefix: "punpckl") || // Added in 3.9
357 Name.starts_with(Prefix: "pxor.") || // Added in 3.9
358 Name.starts_with(Prefix: "shuf.f") || // Added in 6.0
359 Name.starts_with(Prefix: "shuf.i") || // Added in 6.0
360 Name.starts_with(Prefix: "shuf.p") || // Added in 4.0
361 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
362 Name.starts_with(Prefix: "store.b.") || // Added in 3.9
363 Name.starts_with(Prefix: "store.d.") || // Added in 3.9
364 Name.starts_with(Prefix: "store.p") || // Added in 3.9
365 Name.starts_with(Prefix: "store.q.") || // Added in 3.9
366 Name.starts_with(Prefix: "store.w.") || // Added in 3.9
367 Name == "store.ss" || // Added in 7.0
368 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
369 Name.starts_with(Prefix: "sub.p") || // Added in 7.0. 128/256 in 4.0
370 Name.starts_with(Prefix: "ucmp.") || // Added in 5.0
371 Name.starts_with(Prefix: "unpckh.") || // Added in 3.9
372 Name.starts_with(Prefix: "unpckl.") || // Added in 3.9
373 Name.starts_with(Prefix: "valign.") || // Added in 4.0
374 Name == "vcvtph2ps.128" || // Added in 11.0
375 Name == "vcvtph2ps.256" || // Added in 11.0
376 Name.starts_with(Prefix: "vextract") || // Added in 4.0
377 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
378 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
379 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
380 Name.starts_with(Prefix: "vfnmsub.") || // Added in 7.0
381 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
382 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
383 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
384 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
385 Name.starts_with(Prefix: "vpermi2var.") || // Added in 7.0
386 Name.starts_with(Prefix: "vpermil.p") || // Added in 3.9
387 Name.starts_with(Prefix: "vpermilvar.") || // Added in 4.0
388 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
389 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
390 Name.starts_with(Prefix: "vpshld.") || // Added in 7.0
391 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
392 Name.starts_with(Prefix: "vpshrd.") || // Added in 7.0
393 Name.starts_with(Prefix: "vpshrdv.") || // Added in 8.0
394 Name.starts_with(Prefix: "vpshufbitqmb.") || // Added in 8.0
395 Name.starts_with(Prefix: "xor.")); // Added in 3.9
396
397 if (Name.consume_front(Prefix: "mask3."))
398 // 'avx512.mask3.*'
399 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
400 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
401 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
402 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
403 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
404
405 if (Name.consume_front(Prefix: "maskz."))
406 // 'avx512.maskz.*'
407 return (Name.starts_with(Prefix: "pternlog.") || // Added in 7.0
408 Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
409 Name.starts_with(Prefix: "vfmaddsub.") || // Added in 7.0
410 Name.starts_with(Prefix: "vpdpbusd.") || // Added in 7.0
411 Name.starts_with(Prefix: "vpdpbusds.") || // Added in 7.0
412 Name.starts_with(Prefix: "vpdpwssd.") || // Added in 7.0
413 Name.starts_with(Prefix: "vpdpwssds.") || // Added in 7.0
414 Name.starts_with(Prefix: "vpermt2var.") || // Added in 7.0
415 Name.starts_with(Prefix: "vpmadd52") || // Added in 7.0
416 Name.starts_with(Prefix: "vpshldv.") || // Added in 8.0
417 Name.starts_with(Prefix: "vpshrdv.")); // Added in 8.0
418
419 // 'avx512.*'
420 return (Name == "movntdqa" || // Added in 5.0
421 Name == "pmul.dq.512" || // Added in 7.0
422 Name == "pmulu.dq.512" || // Added in 7.0
423 Name.starts_with(Prefix: "broadcastm") || // Added in 6.0
424 Name.starts_with(Prefix: "cmp.p") || // Added in 12.0
425 Name.starts_with(Prefix: "cvtb2mask.") || // Added in 7.0
426 Name.starts_with(Prefix: "cvtd2mask.") || // Added in 7.0
427 Name.starts_with(Prefix: "cvtmask2") || // Added in 5.0
428 Name.starts_with(Prefix: "cvtq2mask.") || // Added in 7.0
429 Name == "cvtusi2sd" || // Added in 7.0
430 Name.starts_with(Prefix: "cvtw2mask.") || // Added in 7.0
431 Name == "kand.w" || // Added in 7.0
432 Name == "kandn.w" || // Added in 7.0
433 Name == "knot.w" || // Added in 7.0
434 Name == "kor.w" || // Added in 7.0
435 Name == "kortestc.w" || // Added in 7.0
436 Name == "kortestz.w" || // Added in 7.0
437 Name.starts_with(Prefix: "kunpck") || // added in 6.0
438 Name == "kxnor.w" || // Added in 7.0
439 Name == "kxor.w" || // Added in 7.0
440 Name.starts_with(Prefix: "padds.") || // Added in 8.0
441 Name.starts_with(Prefix: "pbroadcast") || // Added in 3.9
442 Name.starts_with(Prefix: "pmulh.w") || // Added in 24.0
443 Name.starts_with(Prefix: "pmulhu.w") || // Added in 24.0
444 Name.starts_with(Prefix: "prol") || // Added in 8.0
445 Name.starts_with(Prefix: "pror") || // Added in 8.0
446 Name.starts_with(Prefix: "psll.dq") || // Added in 3.9
447 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.9
448 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
449 Name.starts_with(Prefix: "ptestm") || // Added in 6.0
450 Name.starts_with(Prefix: "ptestnm") || // Added in 6.0
451 Name.starts_with(Prefix: "storent.") || // Added in 3.9
452 Name.starts_with(Prefix: "vbroadcast.s") || // Added in 7.0
453 Name.starts_with(Prefix: "vpshld.") || // Added in 8.0
454 Name.starts_with(Prefix: "vpshrd.")); // Added in 8.0
455 }
456
457 if (Name.consume_front(Prefix: "fma."))
458 return (Name.starts_with(Prefix: "vfmadd.") || // Added in 7.0
459 Name.starts_with(Prefix: "vfmsub.") || // Added in 7.0
460 Name.starts_with(Prefix: "vfmsubadd.") || // Added in 7.0
461 Name.starts_with(Prefix: "vfnmadd.") || // Added in 7.0
462 Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
463
464 if (Name.consume_front(Prefix: "fma4."))
465 return Name.starts_with(Prefix: "vfmadd.s"); // Added in 7.0
466
467 if (Name.consume_front(Prefix: "sse."))
468 return (Name == "add.ss" || // Added in 4.0
469 Name == "cvtsi2ss" || // Added in 7.0
470 Name == "cvtsi642ss" || // Added in 7.0
471 Name == "div.ss" || // Added in 4.0
472 Name == "mul.ss" || // Added in 4.0
473 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
474 Name == "sqrt.ss" || // Added in 7.0
475 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
476 Name == "sub.ss"); // Added in 4.0
477
478 if (Name.consume_front(Prefix: "sse2."))
479 return (Name == "add.sd" || // Added in 4.0
480 Name == "cvtdq2pd" || // Added in 3.9
481 Name == "cvtdq2ps" || // Added in 7.0
482 Name == "cvtps2pd" || // Added in 3.9
483 Name == "cvtsi2sd" || // Added in 7.0
484 Name == "cvtsi642sd" || // Added in 7.0
485 Name == "cvtss2sd" || // Added in 7.0
486 Name == "div.sd" || // Added in 4.0
487 Name == "mul.sd" || // Added in 4.0
488 Name.starts_with(Prefix: "padds.") || // Added in 8.0
489 Name.starts_with(Prefix: "paddus.") || // Added in 8.0
490 Name.starts_with(Prefix: "pcmpeq.") || // Added in 3.1
491 Name.starts_with(Prefix: "pcmpgt.") || // Added in 3.1
492 Name == "pmaxs.w" || // Added in 3.9
493 Name == "pmaxu.b" || // Added in 3.9
494 Name == "pmins.w" || // Added in 3.9
495 Name == "pminu.b" || // Added in 3.9
496 Name == "pmulh.w" || // Added in 24.0
497 Name == "pmulhu.w" || // Added in 24.0
498 Name == "pmulu.dq" || // Added in 7.0
499 Name.starts_with(Prefix: "pshuf") || // Added in 3.9
500 Name.starts_with(Prefix: "psll.dq") || // Added in 3.7
501 Name.starts_with(Prefix: "psrl.dq") || // Added in 3.7
502 Name.starts_with(Prefix: "psubs.") || // Added in 8.0
503 Name.starts_with(Prefix: "psubus.") || // Added in 8.0
504 Name.starts_with(Prefix: "sqrt.p") || // Added in 7.0
505 Name == "sqrt.sd" || // Added in 7.0
506 Name == "storel.dq" || // Added in 3.9
507 Name.starts_with(Prefix: "storeu.") || // Added in 3.9
508 Name == "sub.sd"); // Added in 4.0
509
510 if (Name.consume_front(Prefix: "sse41."))
511 return (Name.starts_with(Prefix: "blendp") || // Added in 3.7
512 Name == "movntdqa" || // Added in 5.0
513 Name == "pblendw" || // Added in 3.7
514 Name == "pmaxsb" || // Added in 3.9
515 Name == "pmaxsd" || // Added in 3.9
516 Name == "pmaxud" || // Added in 3.9
517 Name == "pmaxuw" || // Added in 3.9
518 Name == "pminsb" || // Added in 3.9
519 Name == "pminsd" || // Added in 3.9
520 Name == "pminud" || // Added in 3.9
521 Name == "pminuw" || // Added in 3.9
522 Name.starts_with(Prefix: "pmovsx") || // Added in 3.8
523 Name.starts_with(Prefix: "pmovzx") || // Added in 3.9
524 Name == "pmuldq"); // Added in 7.0
525
526 if (Name.consume_front(Prefix: "sse42."))
527 return Name == "crc32.64.8"; // Added in 3.4
528
529 if (Name.consume_front(Prefix: "sse4a."))
530 return Name.starts_with(Prefix: "movnt."); // Added in 3.9
531
532 if (Name.consume_front(Prefix: "ssse3."))
533 return (Name == "pabs.b.128" || // Added in 6.0
534 Name == "pabs.d.128" || // Added in 6.0
535 Name == "pabs.w.128"); // Added in 6.0
536
537 if (Name.consume_front(Prefix: "xop."))
538 return (Name == "vpcmov" || // Added in 3.8
539 Name == "vpcmov.256" || // Added in 5.0
540 Name.starts_with(Prefix: "vpcom") || // Added in 3.2, Updated in 9.0
541 Name.starts_with(Prefix: "vprot")); // Added in 8.0
542
543 if (Name.consume_front(Prefix: "bmi."))
544 return (Name.starts_with(Prefix: "pdep.") || // Added in 23.0
545 Name.starts_with(Prefix: "pext.")); // Added in 23.0
546
547 return (Name == "addcarry.u32" || // Added in 8.0
548 Name == "addcarry.u64" || // Added in 8.0
549 Name == "addcarryx.u32" || // Added in 8.0
550 Name == "addcarryx.u64" || // Added in 8.0
551 Name == "subborrow.u32" || // Added in 8.0
552 Name == "subborrow.u64" || // Added in 8.0
553 Name.starts_with(Prefix: "vcvtph2ps.")); // Added in 11.0
554}
555
556static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
557 Function *&NewFn) {
558 // Only handle intrinsics that start with "x86.".
559 if (!Name.consume_front(Prefix: "x86."))
560 return false;
561
562 if (shouldUpgradeX86Intrinsic(F, Name)) {
563 NewFn = nullptr;
564 return true;
565 }
566
567 if (Name == "rdtscp") { // Added in 8.0
568 // If this intrinsic has 0 operands, it's the new version.
569 if (F->getFunctionType()->getNumParams() == 0)
570 return false;
571
572 rename(GV: F);
573 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
574 id: Intrinsic::x86_rdtscp);
575 return true;
576 }
577
578 Intrinsic::ID ID;
579
580 // SSE4.1 ptest functions may have an old signature.
581 if (Name.consume_front(Prefix: "sse41.ptest")) { // Added in 3.2
582 ID = StringSwitch<Intrinsic::ID>(Name)
583 .Case(S: "c", Value: Intrinsic::x86_sse41_ptestc)
584 .Case(S: "z", Value: Intrinsic::x86_sse41_ptestz)
585 .Case(S: "nzc", Value: Intrinsic::x86_sse41_ptestnzc)
586 .Default(Value: Intrinsic::not_intrinsic);
587 if (ID != Intrinsic::not_intrinsic)
588 return upgradePTESTIntrinsic(F, IID: ID, NewFn);
589
590 return false;
591 }
592
593 // Several blend and other instructions with masks used the wrong number of
594 // bits.
595
596 // Added in 3.6
597 ID = StringSwitch<Intrinsic::ID>(Name)
598 .Case(S: "sse41.insertps", Value: Intrinsic::x86_sse41_insertps)
599 .Case(S: "sse41.dppd", Value: Intrinsic::x86_sse41_dppd)
600 .Case(S: "sse41.dpps", Value: Intrinsic::x86_sse41_dpps)
601 .Case(S: "sse41.mpsadbw", Value: Intrinsic::x86_sse41_mpsadbw)
602 .Case(S: "avx.dp.ps.256", Value: Intrinsic::x86_avx_dp_ps_256)
603 .Case(S: "avx2.mpsadbw", Value: Intrinsic::x86_avx2_mpsadbw)
604 .Default(Value: Intrinsic::not_intrinsic);
605 if (ID != Intrinsic::not_intrinsic)
606 return upgradeX86IntrinsicsWith8BitMask(F, IID: ID, NewFn);
607
608 if (Name.consume_front(Prefix: "avx512.")) {
609 if (Name.consume_front(Prefix: "mask.cmp.")) {
610 // Added in 7.0
611 ID = StringSwitch<Intrinsic::ID>(Name)
612 .Case(S: "pd.128", Value: Intrinsic::x86_avx512_mask_cmp_pd_128)
613 .Case(S: "pd.256", Value: Intrinsic::x86_avx512_mask_cmp_pd_256)
614 .Case(S: "pd.512", Value: Intrinsic::x86_avx512_mask_cmp_pd_512)
615 .Case(S: "ps.128", Value: Intrinsic::x86_avx512_mask_cmp_ps_128)
616 .Case(S: "ps.256", Value: Intrinsic::x86_avx512_mask_cmp_ps_256)
617 .Case(S: "ps.512", Value: Intrinsic::x86_avx512_mask_cmp_ps_512)
618 .Default(Value: Intrinsic::not_intrinsic);
619 if (ID != Intrinsic::not_intrinsic)
620 return upgradeX86MaskedFPCompare(F, IID: ID, NewFn);
621 } else if (Name.starts_with(Prefix: "vpdpbusd.") ||
622 Name.starts_with(Prefix: "vpdpbusds.")) {
623 // Added in 21.1
624 ID = StringSwitch<Intrinsic::ID>(Name)
625 .Case(S: "vpdpbusd.128", Value: Intrinsic::x86_avx512_vpdpbusd_128)
626 .Case(S: "vpdpbusd.256", Value: Intrinsic::x86_avx512_vpdpbusd_256)
627 .Case(S: "vpdpbusd.512", Value: Intrinsic::x86_avx512_vpdpbusd_512)
628 .Case(S: "vpdpbusds.128", Value: Intrinsic::x86_avx512_vpdpbusds_128)
629 .Case(S: "vpdpbusds.256", Value: Intrinsic::x86_avx512_vpdpbusds_256)
630 .Case(S: "vpdpbusds.512", Value: Intrinsic::x86_avx512_vpdpbusds_512)
631 .Default(Value: Intrinsic::not_intrinsic);
632 if (ID != Intrinsic::not_intrinsic)
633 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
634 } else if (Name.starts_with(Prefix: "vpdpwssd.") ||
635 Name.starts_with(Prefix: "vpdpwssds.")) {
636 // Added in 21.1
637 ID = StringSwitch<Intrinsic::ID>(Name)
638 .Case(S: "vpdpwssd.128", Value: Intrinsic::x86_avx512_vpdpwssd_128)
639 .Case(S: "vpdpwssd.256", Value: Intrinsic::x86_avx512_vpdpwssd_256)
640 .Case(S: "vpdpwssd.512", Value: Intrinsic::x86_avx512_vpdpwssd_512)
641 .Case(S: "vpdpwssds.128", Value: Intrinsic::x86_avx512_vpdpwssds_128)
642 .Case(S: "vpdpwssds.256", Value: Intrinsic::x86_avx512_vpdpwssds_256)
643 .Case(S: "vpdpwssds.512", Value: Intrinsic::x86_avx512_vpdpwssds_512)
644 .Default(Value: Intrinsic::not_intrinsic);
645 if (ID != Intrinsic::not_intrinsic)
646 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
647 }
648 return false; // No other 'x86.avx512.*'.
649 }
650
651 if (Name.consume_front(Prefix: "avx2.")) {
652 if (Name.consume_front(Prefix: "vpdpb")) {
653 // Added in 21.1
654 ID = StringSwitch<Intrinsic::ID>(Name)
655 .Case(S: "ssd.128", Value: Intrinsic::x86_avx2_vpdpbssd_128)
656 .Case(S: "ssd.256", Value: Intrinsic::x86_avx2_vpdpbssd_256)
657 .Case(S: "ssds.128", Value: Intrinsic::x86_avx2_vpdpbssds_128)
658 .Case(S: "ssds.256", Value: Intrinsic::x86_avx2_vpdpbssds_256)
659 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpbsud_128)
660 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpbsud_256)
661 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpbsuds_128)
662 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpbsuds_256)
663 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpbuud_128)
664 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpbuud_256)
665 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpbuuds_128)
666 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpbuuds_256)
667 .Default(Value: Intrinsic::not_intrinsic);
668 if (ID != Intrinsic::not_intrinsic)
669 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
670 } else if (Name.consume_front(Prefix: "vpdpw")) {
671 // Added in 21.1
672 ID = StringSwitch<Intrinsic::ID>(Name)
673 .Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpwsud_128)
674 .Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpwsud_256)
675 .Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpwsuds_128)
676 .Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpwsuds_256)
677 .Case(S: "usd.128", Value: Intrinsic::x86_avx2_vpdpwusd_128)
678 .Case(S: "usd.256", Value: Intrinsic::x86_avx2_vpdpwusd_256)
679 .Case(S: "usds.128", Value: Intrinsic::x86_avx2_vpdpwusds_128)
680 .Case(S: "usds.256", Value: Intrinsic::x86_avx2_vpdpwusds_256)
681 .Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpwuud_128)
682 .Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpwuud_256)
683 .Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpwuuds_128)
684 .Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpwuuds_256)
685 .Default(Value: Intrinsic::not_intrinsic);
686 if (ID != Intrinsic::not_intrinsic)
687 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
688 }
689 return false; // No other 'x86.avx2.*'
690 }
691
692 if (Name.consume_front(Prefix: "avx10.")) {
693 if (Name.consume_front(Prefix: "vpdpb")) {
694 // Added in 21.1
695 ID = StringSwitch<Intrinsic::ID>(Name)
696 .Case(S: "ssd.512", Value: Intrinsic::x86_avx10_vpdpbssd_512)
697 .Case(S: "ssds.512", Value: Intrinsic::x86_avx10_vpdpbssds_512)
698 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpbsud_512)
699 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpbsuds_512)
700 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpbuud_512)
701 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpbuuds_512)
702 .Default(Value: Intrinsic::not_intrinsic);
703 if (ID != Intrinsic::not_intrinsic)
704 return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
705 } else if (Name.consume_front(Prefix: "vpdpw")) {
706 ID = StringSwitch<Intrinsic::ID>(Name)
707 .Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpwsud_512)
708 .Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpwsuds_512)
709 .Case(S: "usd.512", Value: Intrinsic::x86_avx10_vpdpwusd_512)
710 .Case(S: "usds.512", Value: Intrinsic::x86_avx10_vpdpwusds_512)
711 .Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpwuud_512)
712 .Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpwuuds_512)
713 .Default(Value: Intrinsic::not_intrinsic);
714 if (ID != Intrinsic::not_intrinsic)
715 return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
716 }
717 return false; // No other 'x86.avx10.*'
718 }
719
720 if (Name.consume_front(Prefix: "avx512bf16.")) {
721 // Added in 9.0
722 ID = StringSwitch<Intrinsic::ID>(Name)
723 .Case(S: "cvtne2ps2bf16.128",
724 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
725 .Case(S: "cvtne2ps2bf16.256",
726 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
727 .Case(S: "cvtne2ps2bf16.512",
728 Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
729 .Case(S: "mask.cvtneps2bf16.128",
730 Value: Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
731 .Case(S: "cvtneps2bf16.256",
732 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
733 .Case(S: "cvtneps2bf16.512",
734 Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
735 .Default(Value: Intrinsic::not_intrinsic);
736 if (ID != Intrinsic::not_intrinsic)
737 return upgradeX86BF16Intrinsic(F, IID: ID, NewFn);
738
739 // Added in 9.0
740 ID = StringSwitch<Intrinsic::ID>(Name)
741 .Case(S: "dpbf16ps.128", Value: Intrinsic::x86_avx512bf16_dpbf16ps_128)
742 .Case(S: "dpbf16ps.256", Value: Intrinsic::x86_avx512bf16_dpbf16ps_256)
743 .Case(S: "dpbf16ps.512", Value: Intrinsic::x86_avx512bf16_dpbf16ps_512)
744 .Default(Value: Intrinsic::not_intrinsic);
745 if (ID != Intrinsic::not_intrinsic)
746 return upgradeX86BF16DPIntrinsic(F, IID: ID, NewFn);
747 return false; // No other 'x86.avx512bf16.*'.
748 }
749
750 if (Name.consume_front(Prefix: "xop.")) {
751 Intrinsic::ID ID = Intrinsic::not_intrinsic;
752 if (Name.starts_with(Prefix: "vpermil2")) { // Added in 3.9
753 // Upgrade any XOP PERMIL2 index operand still using a float/double
754 // vector.
755 auto Idx = F->getFunctionType()->getParamType(i: 2);
756 if (Idx->isFPOrFPVectorTy()) {
757 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
758 unsigned EltSize = Idx->getScalarSizeInBits();
759 if (EltSize == 64 && IdxSize == 128)
760 ID = Intrinsic::x86_xop_vpermil2pd;
761 else if (EltSize == 32 && IdxSize == 128)
762 ID = Intrinsic::x86_xop_vpermil2ps;
763 else if (EltSize == 64 && IdxSize == 256)
764 ID = Intrinsic::x86_xop_vpermil2pd_256;
765 else
766 ID = Intrinsic::x86_xop_vpermil2ps_256;
767 }
768 } else if (F->arg_size() == 2)
769 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
770 ID = StringSwitch<Intrinsic::ID>(Name)
771 .Case(S: "vfrcz.ss", Value: Intrinsic::x86_xop_vfrcz_ss)
772 .Case(S: "vfrcz.sd", Value: Intrinsic::x86_xop_vfrcz_sd)
773 .Default(Value: Intrinsic::not_intrinsic);
774
775 if (ID != Intrinsic::not_intrinsic) {
776 rename(GV: F);
777 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
778 return true;
779 }
780 return false; // No other 'x86.xop.*'
781 }
782
783 if (Name == "seh.recoverfp") {
784 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
785 id: Intrinsic::eh_recoverfp);
786 return true;
787 }
788
789 return false;
790}
791
792// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
793// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
794static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
795 StringRef Name,
796 Function *&NewFn) {
797 if (Name.starts_with(Prefix: "rbit")) {
798 // '(arm|aarch64).rbit'.
799 NewFn = Intrinsic::getOrInsertDeclaration(
800 M: F->getParent(), id: Intrinsic::bitreverse, OverloadTys: F->arg_begin()->getType());
801 return true;
802 }
803
804 if (Name == "thread.pointer") {
805 // '(arm|aarch64).thread.pointer'.
806 NewFn = Intrinsic::getOrInsertDeclaration(
807 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
808 return true;
809 }
810
811 bool Neon = Name.consume_front(Prefix: "neon.");
812 if (Neon) {
813 // '(arm|aarch64).neon.*'.
814 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
815 // v16i8 respectively.
816 if (Name.consume_front(Prefix: "bfdot.")) {
817 // (arm|aarch64).neon.bfdot.*'.
818 Intrinsic::ID ID =
819 StringSwitch<Intrinsic::ID>(Name)
820 .Cases(CaseStrings: {"v2f32.v8i8", "v4f32.v16i8"},
821 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
822 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
823 .Default(Value: Intrinsic::not_intrinsic);
824 if (ID != Intrinsic::not_intrinsic) {
825 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
826 assert((OperandWidth == 64 || OperandWidth == 128) &&
827 "Unexpected operand width");
828 LLVMContext &Ctx = F->getParent()->getContext();
829 std::array<Type *, 2> Tys{
830 ._M_elems: {F->getReturnType(),
831 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), NumElts: OperandWidth / 16)}};
832 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
833 return true;
834 }
835 return false; // No other '(arm|aarch64).neon.bfdot.*'.
836 }
837
838 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
839 // anymore and accept v8bf16 instead of v16i8.
840 if (Name.consume_front(Prefix: "bfm")) {
841 // (arm|aarch64).neon.bfm*'.
842 if (Name.consume_back(Suffix: ".v4f32.v16i8")) {
843 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
844 Intrinsic::ID ID =
845 StringSwitch<Intrinsic::ID>(Name)
846 .Case(S: "mla",
847 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
848 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
849 .Case(S: "lalb",
850 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
851 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
852 .Case(S: "lalt",
853 Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
854 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
855 .Default(Value: Intrinsic::not_intrinsic);
856 if (ID != Intrinsic::not_intrinsic) {
857 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
858 return true;
859 }
860 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
861 }
862 return false; // No other '(arm|aarch64).neon.bfm*.
863 }
864 // Continue on to Aarch64 Neon or Arm Neon.
865 }
866 // Continue on to Arm or Aarch64.
867
868 if (IsArm) {
869 // 'arm.*'.
870 if (Neon) {
871 // 'arm.neon.*'.
872 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
873 .StartsWith(S: "vclz.", Value: Intrinsic::ctlz)
874 .StartsWith(S: "vcnt.", Value: Intrinsic::ctpop)
875 .StartsWith(S: "vqadds.", Value: Intrinsic::sadd_sat)
876 .StartsWith(S: "vqaddu.", Value: Intrinsic::uadd_sat)
877 .StartsWith(S: "vqsubs.", Value: Intrinsic::ssub_sat)
878 .StartsWith(S: "vqsubu.", Value: Intrinsic::usub_sat)
879 .StartsWith(S: "vrinta.", Value: Intrinsic::round)
880 .StartsWith(S: "vrintn.", Value: Intrinsic::roundeven)
881 .StartsWith(S: "vrintm.", Value: Intrinsic::floor)
882 .StartsWith(S: "vrintp.", Value: Intrinsic::ceil)
883 .StartsWith(S: "vrintx.", Value: Intrinsic::rint)
884 .StartsWith(S: "vrintz.", Value: Intrinsic::trunc)
885 .Default(Value: Intrinsic::not_intrinsic);
886 if (ID != Intrinsic::not_intrinsic) {
887 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
888 OverloadTys: F->arg_begin()->getType());
889 return true;
890 }
891
892 if (Name.consume_front(Prefix: "vst")) {
893 // 'arm.neon.vst*'.
894 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
895 SmallVector<StringRef, 2> Groups;
896 if (vstRegex.match(String: Name, Matches: &Groups)) {
897 static const Intrinsic::ID StoreInts[] = {
898 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
899 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
900
901 static const Intrinsic::ID StoreLaneInts[] = {
902 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
903 Intrinsic::arm_neon_vst4lane};
904
905 auto fArgs = F->getFunctionType()->params();
906 Type *Tys[] = {fArgs[0], fArgs[1]};
907 if (Groups[1].size() == 1)
908 NewFn = Intrinsic::getOrInsertDeclaration(
909 M: F->getParent(), id: StoreInts[fArgs.size() - 3], OverloadTys: Tys);
910 else
911 NewFn = Intrinsic::getOrInsertDeclaration(
912 M: F->getParent(), id: StoreLaneInts[fArgs.size() - 5], OverloadTys: Tys);
913 return true;
914 }
915 return false; // No other 'arm.neon.vst*'.
916 }
917
918 return false; // No other 'arm.neon.*'.
919 }
920
921 if (Name.consume_front(Prefix: "mve.")) {
922 // 'arm.mve.*'.
923 if (Name == "vctp64") {
924 if (cast<FixedVectorType>(Val: F->getReturnType())->getNumElements() == 4) {
925 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
926 // the function and deal with it below in UpgradeIntrinsicCall.
927 rename(GV: F);
928 return true;
929 }
930 return false; // Not 'arm.mve.vctp64'.
931 }
932
933 if (Name.starts_with(Prefix: "vrintn.v")) {
934 NewFn = Intrinsic::getOrInsertDeclaration(
935 M: F->getParent(), id: Intrinsic::roundeven, OverloadTys: F->arg_begin()->getType());
936 return true;
937 }
938
939 // These too are changed to accept a v2i1 instead of the old v4i1.
940 if (Name.consume_back(Suffix: ".v4i1")) {
941 // 'arm.mve.*.v4i1'.
942 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i32"))
943 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
944 return Name == "mull.int" || Name == "vqdmull";
945
946 if (Name.consume_back(Suffix: ".v2i64")) {
947 // 'arm.mve.*.v2i64.v4i1'
948 bool IsGather = Name.consume_front(Prefix: "vldr.gather.");
949 if (IsGather || Name.consume_front(Prefix: "vstr.scatter.")) {
950 if (Name.consume_front(Prefix: "base.")) {
951 // Optional 'wb.' prefix.
952 Name.consume_front(Prefix: "wb.");
953 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
954 // predicated.v2i64.v2i64.v4i1'.
955 return Name == "predicated.v2i64";
956 }
957
958 if (Name.consume_front(Prefix: "offset.predicated."))
959 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
960 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
961
962 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
963 return false;
964 }
965
966 return false; // No other 'arm.mve.*.v2i64.v4i1'.
967 }
968 return false; // No other 'arm.mve.*.v4i1'.
969 }
970 return false; // No other 'arm.mve.*'.
971 }
972
973 if (Name.consume_front(Prefix: "cde.vcx")) {
974 // 'arm.cde.vcx*'.
975 if (Name.consume_back(Suffix: ".predicated.v2i64.v4i1"))
976 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
977 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
978 Name == "3q" || Name == "3qa";
979
980 return false; // No other 'arm.cde.vcx*'.
981 }
982 } else {
983 // 'aarch64.*'.
984 if (Neon) {
985 // 'aarch64.neon.*'.
986 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
987 .StartsWith(S: "frintn", Value: Intrinsic::roundeven)
988 .StartsWith(S: "rbit", Value: Intrinsic::bitreverse)
989 .Default(Value: Intrinsic::not_intrinsic);
990 if (ID != Intrinsic::not_intrinsic) {
991 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
992 OverloadTys: F->arg_begin()->getType());
993 return true;
994 }
995
996 Intrinsic::ID MinMaxID =
997 StringSwitch<Intrinsic::ID>(Name.split(Separator: '.').first)
998 .Case(S: "smax", Value: Intrinsic::smax)
999 .Case(S: "smin", Value: Intrinsic::smin)
1000 .Case(S: "umax", Value: Intrinsic::umax)
1001 .Case(S: "umin", Value: Intrinsic::umin)
1002 .Default(Value: Intrinsic::not_intrinsic);
1003 if (MinMaxID != Intrinsic::not_intrinsic) {
1004 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1005 return false; // Invalid IR.
1006 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: MinMaxID,
1007 OverloadTys: F->getReturnType());
1008 return true;
1009 }
1010
1011 if (Name.starts_with(Prefix: "addp")) {
1012 // 'aarch64.neon.addp*'.
1013 if (F->arg_size() != 2)
1014 return false; // Invalid IR.
1015 VectorType *Ty = dyn_cast<VectorType>(Val: F->getReturnType());
1016 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1017 NewFn = Intrinsic::getOrInsertDeclaration(
1018 M: F->getParent(), id: Intrinsic::aarch64_neon_faddp, OverloadTys: Ty);
1019 return true;
1020 }
1021 }
1022
1023 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1024 if (Name.starts_with(Prefix: "bfcvt")) {
1025 NewFn = nullptr;
1026 return true;
1027 }
1028
1029 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1030 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1031 NewFn = nullptr;
1032 return true;
1033 }
1034
1035 return false; // No other 'aarch64.neon.*'.
1036 }
1037 if (Name.consume_front(Prefix: "sve.")) {
1038 // 'aarch64.sve.*'.
1039 if (Name.consume_front(Prefix: "bf")) {
1040 if (Name == "mmla") {
1041 Type *Tys[] = {F->getReturnType(),
1042 std::next(x: F->arg_begin())->getType()};
1043 NewFn = Intrinsic::getOrInsertDeclaration(
1044 M: F->getParent(), id: Intrinsic::aarch64_sve_fmmla, OverloadTys: Tys);
1045 return true;
1046 }
1047 if (Name.consume_back(Suffix: ".lane")) {
1048 // 'aarch64.sve.bf*.lane'.
1049 Intrinsic::ID ID =
1050 StringSwitch<Intrinsic::ID>(Name)
1051 .Case(S: "dot", Value: Intrinsic::aarch64_sve_bfdot_lane_v2)
1052 .Case(S: "mlalb", Value: Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1053 .Case(S: "mlalt", Value: Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1054 .Default(Value: Intrinsic::not_intrinsic);
1055 if (ID != Intrinsic::not_intrinsic) {
1056 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1057 return true;
1058 }
1059 return false; // No other 'aarch64.sve.bf*.lane'.
1060 }
1061 return false; // No other 'aarch64.sve.bf*'.
1062 }
1063
1064 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1065 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1066 NewFn = nullptr;
1067 return true;
1068 }
1069
1070 if (Name.consume_front(Prefix: "convert.from.svbool")) {
1071 // 'aarch64.sve.convert.from.svbool'
1072 auto *TTy = dyn_cast<TargetExtType>(Val: F->getReturnType());
1073 if (!TTy || TTy->getName() != "aarch64.svcount")
1074 return false;
1075
1076 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1077 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1078 return true;
1079 }
1080
1081 if (Name.consume_front(Prefix: "convert.to.svbool")) {
1082 // 'aarch64.sve.convert.to.svbool'
1083 auto *TTy = dyn_cast<TargetExtType>(Val: F->arg_begin()->getType());
1084 if (!TTy || TTy->getName() != "aarch64.svcount")
1085 return false;
1086
1087 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1088 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1089 return true;
1090 }
1091
1092 if (Name.consume_front(Prefix: "addqv")) {
1093 // 'aarch64.sve.addqv'.
1094 if (!F->getReturnType()->isFPOrFPVectorTy())
1095 return false;
1096
1097 auto Args = F->getFunctionType()->params();
1098 Type *Tys[] = {F->getReturnType(), Args[1]};
1099 NewFn = Intrinsic::getOrInsertDeclaration(
1100 M: F->getParent(), id: Intrinsic::aarch64_sve_faddqv, OverloadTys: Tys);
1101 return true;
1102 }
1103
1104 if (Name.consume_front(Prefix: "ld")) {
1105 // 'aarch64.sve.ld*'.
1106 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1107 if (LdRegex.match(String: Name)) {
1108 Type *ScalarTy =
1109 cast<VectorType>(Val: F->getReturnType())->getElementType();
1110 ElementCount EC =
1111 cast<VectorType>(Val: F->arg_begin()->getType())->getElementCount();
1112 assert(F->arg_size() == 2 &&
1113 "Expected 2 arguments for ld* intrinsic.");
1114 Type *PtrTy = F->getArg(i: 1)->getType();
1115 Type *Ty = VectorType::get(ElementType: ScalarTy, EC);
1116 static const Intrinsic::ID LoadIDs[] = {
1117 Intrinsic::aarch64_sve_ld2_sret,
1118 Intrinsic::aarch64_sve_ld3_sret,
1119 Intrinsic::aarch64_sve_ld4_sret,
1120 };
1121 NewFn = Intrinsic::getOrInsertDeclaration(
1122 M: F->getParent(), id: LoadIDs[Name[0] - '2'], OverloadTys: {Ty, PtrTy});
1123 return true;
1124 }
1125 return false; // No other 'aarch64.sve.ld*'.
1126 }
1127
1128 if (Name.consume_front(Prefix: "tuple.")) {
1129 // 'aarch64.sve.tuple.*'.
1130 if (Name.starts_with(Prefix: "get")) {
1131 // 'aarch64.sve.tuple.get*'.
1132 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1133 NewFn = Intrinsic::getOrInsertDeclaration(
1134 M: F->getParent(), id: Intrinsic::vector_extract, OverloadTys: Tys);
1135 return true;
1136 }
1137
1138 if (Name.starts_with(Prefix: "set")) {
1139 // 'aarch64.sve.tuple.set*'.
1140 auto Args = F->getFunctionType()->params();
1141 Type *Tys[] = {Args[0], Args[2], Args[1]};
1142 NewFn = Intrinsic::getOrInsertDeclaration(
1143 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1144 return true;
1145 }
1146
1147 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1148 if (CreateTupleRegex.match(String: Name)) {
1149 // 'aarch64.sve.tuple.create*'.
1150 auto Args = F->getFunctionType()->params();
1151 Type *Tys[] = {F->getReturnType(), Args[1]};
1152 NewFn = Intrinsic::getOrInsertDeclaration(
1153 M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1154 return true;
1155 }
1156 return false; // No other 'aarch64.sve.tuple.*'.
1157 }
1158
1159 if (Name.starts_with(Prefix: "rev.nxv")) {
1160 // 'aarch64.sve.rev.<Ty>'
1161 NewFn = Intrinsic::getOrInsertDeclaration(
1162 M: F->getParent(), id: Intrinsic::vector_reverse, OverloadTys: F->getReturnType());
1163 return true;
1164 }
1165
1166 return false; // No other 'aarch64.sve.*'.
1167 }
1168 if (Name.consume_front(Prefix: "sme.")) {
1169 // 'aarch64.sme.*'.
1170 if (Name.consume_front(Prefix: "ftmopa.")) {
1171 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1172 // intrinsics to model their FPMR dependency.
1173 Intrinsic::ID ID =
1174 StringSwitch<Intrinsic::ID>(Name)
1175 .Case(S: "za16.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1176 .Case(S: "za32.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1177 .Default(Value: Intrinsic::not_intrinsic);
1178 if (ID != Intrinsic::not_intrinsic) {
1179 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1180 return true;
1181 }
1182 return false; // No other 'aarch64.sme.ftmopa.*'.
1183 }
1184
1185 return false; // No other 'aarch64.sme.*'.
1186 }
1187 }
1188 return false; // No other 'arm.*', 'aarch64.*'.
1189}
1190
1191// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1192// declarations requiring an auto-upgrade. The same set applies to the
1193// cluster (g2s) and CTA (g2s_cta) variants.
1194#define NVVM_TMA_G2S_MODES(M) \
1195 M(tile_1d, "tile.1d") \
1196 M(tile_2d, "tile.2d") \
1197 M(tile_3d, "tile.3d") \
1198 M(tile_4d, "tile.4d") \
1199 M(tile_5d, "tile.5d") \
1200 M(tile_gather4_2d, "tile.gather4.2d") \
1201 M(im2col_3d, "im2col.3d") \
1202 M(im2col_4d, "im2col.4d") \
1203 M(im2col_5d, "im2col.5d") \
1204 M(im2col_w_3d, "im2col.w.3d") \
1205 M(im2col_w_4d, "im2col.w.4d") \
1206 M(im2col_w_5d, "im2col.w.5d") \
1207 M(im2col_w_128_3d, "im2col.w.128.3d") \
1208 M(im2col_w_128_4d, "im2col.w.128.4d") \
1209 M(im2col_w_128_5d, "im2col.w.128.5d")
1210
1211// Two legacy tails are:
1212//
1213// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1214// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1215//
1216// The current tail appends a trailing i32 %flag_valid_pattern, so both
1217// legacy tails are recognized by an i1 at parameter N-2.
1218static Intrinsic::ID
1219shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1220 SmallVectorImpl<Type *> &OvlTys) {
1221 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s."))
1222 return Intrinsic::not_intrinsic;
1223
1224#define G2S_ID(ID_SUFFIX, NAME) \
1225 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1226 // clang-format off
1227 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1228 NVVM_TMA_G2S_MODES(G2S_ID)
1229 .Default(Value: Intrinsic::not_intrinsic);
1230#undef G2S_ID
1231 // clang-format on
1232 if (ID == Intrinsic::not_intrinsic)
1233 return ID;
1234
1235 size_t NumParams = F->getFunctionType()->getNumParams();
1236
1237 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1238 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1239 if (!F->getFunctionType()->getParamType(i: NumParams - 2)->isIntegerTy(BitWidth: 1))
1240 return Intrinsic::not_intrinsic;
1241
1242 // The multicast mask is the parameter immediately before the i64
1243 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1244 ArrayRef<Type *> Params = F->getFunctionType()->params();
1245 size_t MaskIdx =
1246 Params[NumParams - 1]->isIntegerTy(BitWidth: 1) ? NumParams - 4 : NumParams - 5;
1247 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1248 "expected the i64 cache-hint after the multicast mask");
1249 Type *MaskTy = Params[MaskIdx];
1250 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1251 OvlTys.push_back(Elt: MaskTy);
1252
1253 return ID;
1254}
1255
1256// The legacy tail is:
1257//
1258// arg1, arg2, .. i64 %ch, i1 %flag_ch
1259//
1260// The current tail appends a trailing i32 %flag_valid_pattern, so the
1261// legacy tail is recognized by an i1 at parameter N-1.
1262static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1263 StringRef Name) {
1264 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s.cta."))
1265 return Intrinsic::not_intrinsic;
1266
1267#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1268 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1269 // clang-format off
1270 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1271 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1272 .Default(Value: Intrinsic::not_intrinsic);
1273#undef G2S_CTA_ID
1274 // clang-format on
1275 if (ID == Intrinsic::not_intrinsic)
1276 return ID;
1277
1278 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1279 // with i32 %flag_valid_pattern, for which N-1 is i32.
1280 if (!F->getFunctionType()
1281 ->getParamType(i: F->getFunctionType()->getNumParams() - 1)
1282 ->isIntegerTy(BitWidth: 1))
1283 return Intrinsic::not_intrinsic;
1284
1285 return ID;
1286}
1287// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1288//
1289// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1290//
1291// The current intrinsic is overloaded on the multicast-mask type and takes a
1292// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1293// parameter N-1.
1294static Intrinsic::ID
1295shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1296 SmallVectorImpl<Type *> &OvlTys) {
1297 if (!Name.consume_front(Prefix: "cp.async.bulk.global.to.shared.cluster"))
1298 return Intrinsic::not_intrinsic;
1299
1300 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1301 // i32 %flag_valid_pattern, for which N-1 is i32.
1302 size_t NumParams = F->getFunctionType()->getNumParams();
1303 if (!F->getFunctionType()->getParamType(i: NumParams - 1)->isIntegerTy(BitWidth: 1))
1304 return Intrinsic::not_intrinsic;
1305
1306 // The multicast mask is parameter 4; legacy IR only uses i16.
1307 Type *MaskTy = F->getFunctionType()->getParamType(i: NumParams - 4);
1308 if (!MaskTy->isIntegerTy(BitWidth: 16))
1309 return Intrinsic::not_intrinsic;
1310 OvlTys.push_back(Elt: MaskTy);
1311
1312 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1313}
1314
1315// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1316//
1317// ..., i64 %ch, i1 %flag_ch
1318//
1319// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1320// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1321// recognized by an i1 at parameter N-1, whereas the current tail ends
1322// with an i32.
1323static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1324 StringRef Name) {
1325 if (!Name.consume_front(Prefix: "cp.async.bulk.global.to.shared.cta"))
1326 return Intrinsic::not_intrinsic;
1327
1328 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1329 // i32 %flag_valid_pattern, for which N-1 is i32.
1330 if (!F->getFunctionType()->getParamType(i: 5)->isIntegerTy(BitWidth: 1))
1331 return Intrinsic::not_intrinsic;
1332
1333 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1334}
1335
1336// The legacy TMA reduction intrinsics encode the reduction operator in their
1337// name, while the current ones take it as an immediate argument. Map the
1338// operator part of a legacy name to the corresponding immediate value.
1339static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1340 return StringSwitch<std::optional<unsigned>>(Name)
1341 .Case(S: "add", Value: static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1342 .Case(S: "min", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1343 .Case(S: "max", Value: static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1344 .Case(S: "inc", Value: static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1345 .Case(S: "dec", Value: static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1346 .Case(S: "and", Value: static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1347 .Case(S: "or", Value: static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1348 .Case(S: "xor", Value: static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1349 .Default(Value: std::nullopt);
1350}
1351
1352static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1353 if (!Name.consume_front(Prefix: "cp.async.bulk.tensor.reduce."))
1354 return Intrinsic::not_intrinsic;
1355
1356 auto [RedOpName, ShapeName] = Name.split(Separator: '.');
1357 if (!getNVPTXTMAReductionOp(Name: RedOpName))
1358 return Intrinsic::not_intrinsic;
1359
1360 return StringSwitch<Intrinsic::ID>(ShapeName)
1361 .Case(S: "tile.1d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1362 .Case(S: "tile.2d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1363 .Case(S: "tile.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1364 .Case(S: "tile.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1365 .Case(S: "tile.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1366 .Case(S: "im2col.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1367 .Case(S: "im2col.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1368 .Case(S: "im2col.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1369 .Default(Value: Intrinsic::not_intrinsic);
1370}
1371
1372static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1373 StringRef Name) {
1374 if (Name.consume_front(Prefix: "mapa.shared.cluster"))
1375 if (F->getReturnType()->getPointerAddressSpace() ==
1376 NVPTXAS::ADDRESS_SPACE_SHARED)
1377 return Intrinsic::nvvm_mapa_shared_cluster;
1378
1379 if (Name.consume_front(Prefix: "cp.async.bulk.")) {
1380 Intrinsic::ID ID =
1381 StringSwitch<Intrinsic::ID>(Name)
1382 .Case(S: "shared.cta.to.cluster",
1383 Value: Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1384 .Default(Value: Intrinsic::not_intrinsic);
1385
1386 if (ID != Intrinsic::not_intrinsic)
1387 if (F->getArg(i: 0)->getType()->getPointerAddressSpace() ==
1388 NVPTXAS::ADDRESS_SPACE_SHARED)
1389 return ID;
1390 }
1391
1392 return Intrinsic::not_intrinsic;
1393}
1394
1395static Intrinsic::ID
1396shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1397 if (!Name.consume_front(Prefix: "tcgen05.commit."))
1398 return Intrinsic::not_intrinsic;
1399
1400 if (Name.consume_front(Prefix: "shared."))
1401 return StringSwitch<Intrinsic::ID>(Name)
1402 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_cg1)
1403 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_cg2)
1404 .Default(Value: Intrinsic::not_intrinsic);
1405
1406 if (Name.consume_front(Prefix: "mc.shared.")) {
1407 // Only upgrade older i16 mc variants.
1408 if (!F->getArg(i: 1)->getType()->isIntegerTy(BitWidth: 16))
1409 return Intrinsic::not_intrinsic;
1410
1411 return StringSwitch<Intrinsic::ID>(Name)
1412 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1413 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1414 .Default(Value: Intrinsic::not_intrinsic);
1415 }
1416
1417 return Intrinsic::not_intrinsic;
1418}
1419
1420static Intrinsic::ID
1421shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1422 if (F->arg_size() != 2)
1423 return Intrinsic::not_intrinsic;
1424
1425 if (Name.consume_front(Prefix: "tcgen05.alloc.shared.") ||
1426 Name.consume_front(Prefix: "tcgen05.alloc."))
1427 return StringSwitch<Intrinsic::ID>(Name)
1428 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_alloc_cg1)
1429 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_alloc_cg2)
1430 .Default(Value: Intrinsic::not_intrinsic);
1431
1432 if (Name.consume_front(Prefix: "tcgen05.dealloc."))
1433 return StringSwitch<Intrinsic::ID>(Name)
1434 .Case(S: "cg1", Value: Intrinsic::nvvm_tcgen05_dealloc_cg1)
1435 .Case(S: "cg2", Value: Intrinsic::nvvm_tcgen05_dealloc_cg2)
1436 .Default(Value: Intrinsic::not_intrinsic);
1437
1438 return Intrinsic::not_intrinsic;
1439}
1440
1441static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1442 if (Name.consume_front(Prefix: "fma.rn."))
1443 return StringSwitch<Intrinsic::ID>(Name)
1444 .Case(S: "bf16", Value: Intrinsic::nvvm_fma_rn_bf16)
1445 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fma_rn_bf16x2)
1446 .Case(S: "relu.bf16", Value: Intrinsic::nvvm_fma_rn_relu_bf16)
1447 .Case(S: "relu.bf16x2", Value: Intrinsic::nvvm_fma_rn_relu_bf16x2)
1448 .Default(Value: Intrinsic::not_intrinsic);
1449
1450 if (Name.consume_front(Prefix: "fmax."))
1451 return StringSwitch<Intrinsic::ID>(Name)
1452 .Case(S: "bf16", Value: Intrinsic::nvvm_fmax_bf16)
1453 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmax_bf16x2)
1454 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmax_ftz_bf16)
1455 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_bf16x2)
1456 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16)
1457 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1458 .Case(S: "ftz.nan.xorsign.abs.bf16",
1459 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1460 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1461 Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1462 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1463 .Case(S: "ftz.xorsign.abs.bf16x2",
1464 Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1465 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmax_nan_bf16)
1466 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmax_nan_bf16x2)
1467 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1468 .Case(S: "nan.xorsign.abs.bf16x2",
1469 Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1470 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1471 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1472 .Default(Value: Intrinsic::not_intrinsic);
1473
1474 if (Name.consume_front(Prefix: "fmin."))
1475 return StringSwitch<Intrinsic::ID>(Name)
1476 .Case(S: "bf16", Value: Intrinsic::nvvm_fmin_bf16)
1477 .Case(S: "bf16x2", Value: Intrinsic::nvvm_fmin_bf16x2)
1478 .Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmin_ftz_bf16)
1479 .Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_bf16x2)
1480 .Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16)
1481 .Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1482 .Case(S: "ftz.nan.xorsign.abs.bf16",
1483 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1484 .Case(S: "ftz.nan.xorsign.abs.bf16x2",
1485 Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1486 .Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1487 .Case(S: "ftz.xorsign.abs.bf16x2",
1488 Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1489 .Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmin_nan_bf16)
1490 .Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmin_nan_bf16x2)
1491 .Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1492 .Case(S: "nan.xorsign.abs.bf16x2",
1493 Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1494 .Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1495 .Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1496 .Default(Value: Intrinsic::not_intrinsic);
1497
1498 if (Name.consume_front(Prefix: "neg."))
1499 return StringSwitch<Intrinsic::ID>(Name)
1500 .Case(S: "bf16", Value: Intrinsic::nvvm_neg_bf16)
1501 .Case(S: "bf16x2", Value: Intrinsic::nvvm_neg_bf16x2)
1502 .Default(Value: Intrinsic::not_intrinsic);
1503
1504 return Intrinsic::not_intrinsic;
1505}
1506
1507static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1508 FunctionType *NewFnTy = Intrinsic::getType(Context&: F->getContext(), id: IID);
1509 FunctionType *OldFnTy = F->getFunctionType();
1510 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1511 return OldTy->getScalarType()->isIntegerTy() &&
1512 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1513 };
1514
1515 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1516 return false;
1517
1518 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1519 return false;
1520
1521 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1522 if (!IsOldBF16StorageTy(OldFnTy->getParamType(i: I), NewFnTy->getParamType(i: I)))
1523 return false;
1524
1525 return true;
1526}
1527
1528// Overloaded fadd/fmul intrinsic IDs, indexed by [`.ftz`][`.sat`].
1529static constexpr Intrinsic::ID NVVMFAddIIDs[2][2] = {
1530 {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat},
1531 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat}};
1532static constexpr Intrinsic::ID NVVMFMulIIDs[2][2] = {
1533 {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat},
1534 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat}};
1535
1536static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1537getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID (&IIDs)[2][2]) {
1538 auto [Modifiers, Type] = Name.rsplit(Separator: '.');
1539 if (!is_contained(Set: {"f", "d", "f16", "v2f16"}, Element: Type))
1540 return std::nullopt;
1541
1542 std::optional<llvm::RoundingMode> RoundingMode =
1543 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(N: 2))
1544 .Case(S: "rn", Value: llvm::RoundingMode::NearestTiesToEven)
1545 .Case(S: "rz", Value: llvm::RoundingMode::TowardZero)
1546 .Case(S: "rm", Value: llvm::RoundingMode::TowardNegative)
1547 .Case(S: "rp", Value: llvm::RoundingMode::TowardPositive)
1548 .Default(Value: std::nullopt);
1549 if (!RoundingMode)
1550 return std::nullopt;
1551
1552 StringRef Rest = Modifiers.drop_front(N: 2);
1553 const bool IsFTZ = Rest.consume_front(Prefix: ".ftz");
1554 const bool IsSat = Rest.consume_front(Prefix: ".sat");
1555 if (!Rest.empty())
1556 return std::nullopt;
1557
1558 return std::make_pair(x: IIDs[IsFTZ][IsSat], y&: *RoundingMode);
1559}
1560
1561static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1562 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1563 return Intrinsic::not_intrinsic;
1564
1565 return Intrinsic::nvvm_mbarrier_init;
1566}
1567
1568static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1569 return Name.consume_front(Prefix: "local") || Name.consume_front(Prefix: "shared") ||
1570 Name.consume_front(Prefix: "global") || Name.consume_front(Prefix: "constant") ||
1571 Name.consume_front(Prefix: "param");
1572}
1573
1574static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1575 if (!Name.consume_front(Prefix: "vp."))
1576 return 0;
1577 return StringSwitch<unsigned>(Name)
1578 .StartsWith(S: "select", Value: Instruction::Select)
1579 .StartsWith(S: "add", Value: Instruction::Add)
1580 .StartsWith(S: "sub", Value: Instruction::Sub)
1581 .StartsWith(S: "mul", Value: Instruction::Mul)
1582 .StartsWith(S: "ashr", Value: Instruction::AShr)
1583 .StartsWith(S: "lshr", Value: Instruction::LShr)
1584 .StartsWith(S: "shl", Value: Instruction::Shl)
1585 .StartsWith(S: "or", Value: Instruction::Or)
1586 .StartsWith(S: "and", Value: Instruction::And)
1587 .StartsWith(S: "xor", Value: Instruction::Xor)
1588 .StartsWith(S: "fadd", Value: Instruction::FAdd)
1589 .StartsWith(S: "fsub", Value: Instruction::FSub)
1590 .StartsWith(S: "fmuladd", Value: 0)
1591 .StartsWith(S: "fmul", Value: Instruction::FMul)
1592 .StartsWith(S: "fdiv", Value: Instruction::FDiv)
1593 .StartsWith(S: "frem", Value: Instruction::FRem)
1594 .StartsWith(S: "fneg", Value: Instruction::FNeg)
1595 .StartsWith(S: "trunc", Value: Instruction::Trunc)
1596 .StartsWith(S: "zext", Value: Instruction::ZExt)
1597 .StartsWith(S: "sext", Value: Instruction::SExt)
1598 .StartsWith(S: "fptrunc", Value: Instruction::FPTrunc)
1599 .StartsWith(S: "fpext", Value: Instruction::FPExt)
1600 .StartsWith(S: "fptoui", Value: Instruction::FPToUI)
1601 .StartsWith(S: "fptosi", Value: Instruction::FPToSI)
1602 .StartsWith(S: "uitofp", Value: Instruction::UIToFP)
1603 .StartsWith(S: "sitofp", Value: Instruction::SIToFP)
1604 .StartsWith(S: "ptrtoint", Value: Instruction::PtrToInt)
1605 .StartsWith(S: "inttoptr", Value: Instruction::IntToPtr)
1606 .StartsWith(S: "icmp", Value: Instruction::ICmp)
1607 .StartsWith(S: "fcmp", Value: Instruction::FCmp)
1608 .Default(Value: 0);
1609}
1610
1611static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1612 if (!Name.consume_front(Prefix: "vp."))
1613 return 0;
1614 return StringSwitch<Intrinsic::ID>(Name)
1615 .StartsWith(S: "abs", Value: Intrinsic::abs)
1616 .StartsWith(S: "smax", Value: Intrinsic::smax)
1617 .StartsWith(S: "smin", Value: Intrinsic::smin)
1618 .StartsWith(S: "umax", Value: Intrinsic::umax)
1619 .StartsWith(S: "umin", Value: Intrinsic::umin)
1620 .StartsWith(S: "copysign", Value: Intrinsic::copysign)
1621 .StartsWith(S: "minnum", Value: Intrinsic::minnum)
1622 .StartsWith(S: "maxnum", Value: Intrinsic::maxnum)
1623 .StartsWith(S: "minimum", Value: Intrinsic::minimum)
1624 .StartsWith(S: "maximum", Value: Intrinsic::maximum)
1625 .StartsWith(S: "fabs", Value: Intrinsic::fabs)
1626 .StartsWith(S: "sqrt", Value: Intrinsic::sqrt)
1627 .StartsWith(S: "fma", Value: Intrinsic::fma)
1628 .StartsWith(S: "fmuladd", Value: Intrinsic::fmuladd)
1629 .StartsWith(S: "ceil", Value: Intrinsic::ceil)
1630 .StartsWith(S: "floor", Value: Intrinsic::floor)
1631 .StartsWith(S: "rint", Value: Intrinsic::rint)
1632 .StartsWith(S: "nearbyint", Value: Intrinsic::nearbyint)
1633 .StartsWith(S: "roundeven", Value: Intrinsic::roundeven)
1634 .StartsWith(S: "roundtozero", Value: Intrinsic::trunc)
1635 .StartsWith(S: "round", Value: Intrinsic::round)
1636 .StartsWith(S: "lrint", Value: Intrinsic::lrint)
1637 .StartsWith(S: "llrint", Value: Intrinsic::llrint)
1638 .StartsWith(S: "bitreverse", Value: Intrinsic::bitreverse)
1639 .StartsWith(S: "bswap", Value: Intrinsic::bswap)
1640 .StartsWith(S: "ctpop", Value: Intrinsic::ctpop)
1641 .StartsWith(S: "ctlz", Value: Intrinsic::ctlz)
1642 .StartsWith(S: "cttz.elts", Value: 0)
1643 .StartsWith(S: "cttz", Value: Intrinsic::cttz)
1644 .StartsWith(S: "sadd.sat", Value: Intrinsic::sadd_sat)
1645 .StartsWith(S: "uadd.sat", Value: Intrinsic::uadd_sat)
1646 .StartsWith(S: "ssub.sat", Value: Intrinsic::ssub_sat)
1647 .StartsWith(S: "usub.sat", Value: Intrinsic::usub_sat)
1648 .StartsWith(S: "fshl", Value: Intrinsic::fshl)
1649 .StartsWith(S: "fshr", Value: Intrinsic::fshr)
1650 .StartsWith(S: "is.fpclass", Value: Intrinsic::is_fpclass)
1651 .Default(Value: 0);
1652}
1653
1654static bool shouldUpgradeVPIntrinsic(StringRef Name) {
1655 return getFunctionalOpcodeForVP(Name) || getFunctionalIntrinsicIDForVP(Name);
1656}
1657
1658static bool convertIntrinsicValidType(StringRef Name,
1659 const FunctionType *FuncTy) {
1660 Type *HalfTy = Type::getHalfTy(C&: FuncTy->getContext());
1661 if (Name.starts_with(Prefix: "to.fp16")) {
1662 return CastInst::castIsValid(op: Instruction::FPTrunc, SrcTy: FuncTy->getParamType(i: 0),
1663 DstTy: HalfTy) &&
1664 CastInst::castIsValid(op: Instruction::BitCast, SrcTy: HalfTy,
1665 DstTy: FuncTy->getReturnType());
1666 }
1667
1668 if (Name.starts_with(Prefix: "from.fp16")) {
1669 return CastInst::castIsValid(op: Instruction::BitCast, SrcTy: FuncTy->getParamType(i: 0),
1670 DstTy: HalfTy) &&
1671 CastInst::castIsValid(op: Instruction::FPExt, SrcTy: HalfTy,
1672 DstTy: FuncTy->getReturnType());
1673 }
1674
1675 return false;
1676}
1677
1678static unsigned
1679getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID,
1680 SmallVectorImpl<Type *> &OverloadTys) {
1681 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1682 if (Defaults.empty())
1683 return 0;
1684
1685 unsigned FullArgCount = FirstDefault + Defaults.size();
1686
1687 // Only trailing default arguments can be missing.
1688 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1689 return 0;
1690
1691 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1692 if (!Intrinsic::isSignatureValid(ID: IID, FT: F->getFunctionType(), OverloadTys,
1693 NumMissingTrailingParams))
1694 return 0;
1695
1696 return FullArgCount;
1697}
1698
1699static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1700 Intrinsic::ID IID = F->getIntrinsicID();
1701 SmallVector<Type *, 4> OverloadTys;
1702
1703 unsigned FullArgCount =
1704 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1705 if (FullArgCount == 0)
1706 return false;
1707
1708 rename(GV: F);
1709 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID, OverloadTys);
1710 assert(NewFn->arg_size() == FullArgCount &&
1711 "total number of default args does not match intrinsic signature");
1712 return true;
1713}
1714
1715static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1716 bool CanUpgradeDebugIntrinsicsToRecords) {
1717 assert(F && "Illegal to upgrade a non-existent Function.");
1718
1719 StringRef Name = F->getName();
1720
1721 // Quickly eliminate it, if it's not a candidate.
1722 if (!Name.consume_front(Prefix: "llvm.") || Name.empty())
1723 return false;
1724
1725 switch (Name[0]) {
1726 default: break;
1727 case 'a': {
1728 bool IsArm = Name.consume_front(Prefix: "arm.");
1729 if (IsArm || Name.consume_front(Prefix: "aarch64.")) {
1730 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1731 return true;
1732 break;
1733 }
1734
1735 if (Name.consume_front(Prefix: "amdgcn.")) {
1736 if (Name == "alignbit") {
1737 // Target specific intrinsic became redundant
1738 NewFn = Intrinsic::getOrInsertDeclaration(
1739 M: F->getParent(), id: Intrinsic::fshr, OverloadTys: {F->getReturnType()});
1740 return true;
1741 }
1742
1743 if (Name.consume_front(Prefix: "atomic.")) {
1744 if (Name.starts_with(Prefix: "inc") || Name.starts_with(Prefix: "dec") ||
1745 Name.starts_with(Prefix: "cond.sub") || Name.starts_with(Prefix: "csub")) {
1746 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1747 // and usub_sat so there's no new declaration.
1748 NewFn = nullptr;
1749 return true;
1750 }
1751 break; // No other 'amdgcn.atomic.*'
1752 }
1753
1754 if (Name.starts_with(Prefix: "addrspacecast.nonnull")) {
1755 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1756 // so there's no new declaration.
1757 NewFn = nullptr;
1758 return true;
1759 }
1760
1761 switch (F->getIntrinsicID()) {
1762 default:
1763 break;
1764 // Legacy wmma iu intrinsics without the optional clamp operand.
1765 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1766 if (F->arg_size() == 7) {
1767 NewFn = nullptr;
1768 return true;
1769 }
1770 break;
1771 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1772 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1773 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1774 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1775 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1776 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1777 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1778 if (F->arg_size() == 8) {
1779 NewFn = nullptr;
1780 return true;
1781 }
1782 break;
1783 }
1784
1785 if (Name.consume_front(Prefix: "ds.") || Name.consume_front(Prefix: "global.atomic.") ||
1786 Name.consume_front(Prefix: "flat.atomic.")) {
1787 if (Name.starts_with(Prefix: "fadd") ||
1788 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1789 (Name.starts_with(Prefix: "fmin") && !Name.starts_with(Prefix: "fmin.num")) ||
1790 (Name.starts_with(Prefix: "fmax") && !Name.starts_with(Prefix: "fmax.num"))) {
1791 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1792 // declaration.
1793 NewFn = nullptr;
1794 return true;
1795 }
1796 }
1797
1798 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
1799 NewFn = nullptr;
1800 return true;
1801 }
1802
1803 if (Name.starts_with(Prefix: "ldexp.")) {
1804 // Target specific intrinsic became redundant
1805 NewFn = Intrinsic::getOrInsertDeclaration(
1806 M: F->getParent(), id: Intrinsic::ldexp,
1807 OverloadTys: {F->getReturnType(), F->getArg(i: 1)->getType()});
1808 return true;
1809 }
1810 break; // No other 'amdgcn.*'
1811 }
1812
1813 break;
1814 }
1815 case 'c': {
1816 if (F->arg_size() == 1) {
1817 if (Name.consume_front(Prefix: "convert.")) {
1818 if (convertIntrinsicValidType(Name, FuncTy: F->getFunctionType())) {
1819 NewFn = nullptr;
1820 return true;
1821 }
1822 }
1823
1824 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1825 .StartsWith(S: "ctlz.", Value: Intrinsic::ctlz)
1826 .StartsWith(S: "cttz.", Value: Intrinsic::cttz)
1827 .Default(Value: Intrinsic::not_intrinsic);
1828 if (ID != Intrinsic::not_intrinsic) {
1829 rename(GV: F);
1830 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1831 OverloadTys: F->arg_begin()->getType());
1832 return true;
1833 }
1834 }
1835
1836 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1837 if (Name == "coro.end" &&
1838 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(BitWidth: 1)))
1839 CoroEndID = Intrinsic::coro_end;
1840 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(BitWidth: 1))
1841 CoroEndID = Intrinsic::coro_end_async;
1842
1843 if (CoroEndID != Intrinsic::not_intrinsic) {
1844 rename(GV: F);
1845 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: CoroEndID);
1846 return true;
1847 }
1848
1849 break;
1850 }
1851 case 'd':
1852 if (Name.consume_front(Prefix: "dbg.")) {
1853 // Mark debug intrinsics for upgrade to new debug format.
1854 if (CanUpgradeDebugIntrinsicsToRecords) {
1855 if (Name == "addr" || Name == "value" || Name == "assign" ||
1856 Name == "declare" || Name == "label") {
1857 // There's no function to replace these with.
1858 NewFn = nullptr;
1859 // But we do want these to get upgraded.
1860 return true;
1861 }
1862 }
1863 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1864 // converted to DbgVariableRecords later.
1865 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1866 rename(GV: F);
1867 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1868 id: Intrinsic::dbg_value);
1869 return true;
1870 }
1871 break; // No other 'dbg.*'.
1872 }
1873 break;
1874 case 'e':
1875 if (Name.consume_front(Prefix: "experimental.vector.")) {
1876 Intrinsic::ID ID =
1877 StringSwitch<Intrinsic::ID>(Name)
1878 // Skip over extract.last.active, otherwise it will be 'upgraded'
1879 // to a regular vector extract which is a different operation.
1880 .StartsWith(S: "extract.last.active.", Value: Intrinsic::not_intrinsic)
1881 .StartsWith(S: "extract.", Value: Intrinsic::vector_extract)
1882 .StartsWith(S: "insert.", Value: Intrinsic::vector_insert)
1883 .StartsWith(S: "reverse.", Value: Intrinsic::vector_reverse)
1884 .StartsWith(S: "interleave2.", Value: Intrinsic::vector_interleave2)
1885 .StartsWith(S: "deinterleave2.", Value: Intrinsic::vector_deinterleave2)
1886 .StartsWith(S: "partial.reduce.add",
1887 Value: Intrinsic::vector_partial_reduce_add)
1888 .Default(Value: Intrinsic::not_intrinsic);
1889 if (ID != Intrinsic::not_intrinsic) {
1890 const auto *FT = F->getFunctionType();
1891 SmallVector<Type *, 2> Tys;
1892 if (ID == Intrinsic::vector_extract ||
1893 ID == Intrinsic::vector_interleave2)
1894 // Extracting overloads the return type.
1895 Tys.push_back(Elt: FT->getReturnType());
1896 if (ID != Intrinsic::vector_interleave2)
1897 Tys.push_back(Elt: FT->getParamType(i: 0));
1898 if (ID == Intrinsic::vector_insert ||
1899 ID == Intrinsic::vector_partial_reduce_add)
1900 // Inserting overloads the inserted type.
1901 Tys.push_back(Elt: FT->getParamType(i: 1));
1902 rename(GV: F);
1903 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
1904 return true;
1905 }
1906
1907 if (Name.consume_front(Prefix: "reduce.")) {
1908 SmallVector<StringRef, 2> Groups;
1909 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1910 if (R.match(String: Name, Matches: &Groups))
1911 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1912 .Case(S: "add", Value: Intrinsic::vector_reduce_add)
1913 .Case(S: "mul", Value: Intrinsic::vector_reduce_mul)
1914 .Case(S: "and", Value: Intrinsic::vector_reduce_and)
1915 .Case(S: "or", Value: Intrinsic::vector_reduce_or)
1916 .Case(S: "xor", Value: Intrinsic::vector_reduce_xor)
1917 .Case(S: "smax", Value: Intrinsic::vector_reduce_smax)
1918 .Case(S: "smin", Value: Intrinsic::vector_reduce_smin)
1919 .Case(S: "umax", Value: Intrinsic::vector_reduce_umax)
1920 .Case(S: "umin", Value: Intrinsic::vector_reduce_umin)
1921 .Case(S: "fmax", Value: Intrinsic::vector_reduce_fmax)
1922 .Case(S: "fmin", Value: Intrinsic::vector_reduce_fmin)
1923 .Default(Value: Intrinsic::not_intrinsic);
1924
1925 bool V2 = false;
1926 if (ID == Intrinsic::not_intrinsic) {
1927 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1928 Groups.clear();
1929 V2 = true;
1930 if (R2.match(String: Name, Matches: &Groups))
1931 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1932 .Case(S: "fadd", Value: Intrinsic::vector_reduce_fadd)
1933 .Case(S: "fmul", Value: Intrinsic::vector_reduce_fmul)
1934 .Default(Value: Intrinsic::not_intrinsic);
1935 }
1936 if (ID != Intrinsic::not_intrinsic) {
1937 rename(GV: F);
1938 auto Args = F->getFunctionType()->params();
1939 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1940 OverloadTys: {Args[V2 ? 1 : 0]});
1941 return true;
1942 }
1943 break; // No other 'expermental.vector.reduce.*'.
1944 }
1945
1946 if (Name.consume_front(Prefix: "splice"))
1947 return true;
1948 break; // No other 'experimental.vector.*'.
1949 }
1950 if (Name.consume_front(Prefix: "experimental.stepvector.")) {
1951 Intrinsic::ID ID = Intrinsic::stepvector;
1952 rename(GV: F);
1953 NewFn = Intrinsic::getOrInsertDeclaration(
1954 M: F->getParent(), id: ID, OverloadTys: F->getFunctionType()->getReturnType());
1955 return true;
1956 }
1957 break; // No other 'e*'.
1958 case 'f':
1959 if (Name.starts_with(Prefix: "flt.rounds")) {
1960 rename(GV: F);
1961 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1962 id: Intrinsic::get_rounding);
1963 return true;
1964 }
1965 break;
1966 case 'i':
1967 if (Name.starts_with(Prefix: "invariant.group.barrier")) {
1968 // Rename invariant.group.barrier to launder.invariant.group
1969 auto Args = F->getFunctionType()->params();
1970 Type* ObjectPtr[1] = {Args[0]};
1971 rename(GV: F);
1972 NewFn = Intrinsic::getOrInsertDeclaration(
1973 M: F->getParent(), id: Intrinsic::launder_invariant_group, OverloadTys: ObjectPtr);
1974 return true;
1975 }
1976 break;
1977 case 'l': {
1978 bool IsLifetimeStart = Name.consume_front(Prefix: "lifetime.start");
1979 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front(Prefix: "lifetime.end");
1980 if (IsLifetimeStart || IsLifetimeEnd) {
1981 if (F->arg_size() == 2) {
1982 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1983 : Intrinsic::lifetime_end;
1984 rename(GV: F);
1985 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1986 // arguments. Use the Ptr argument to create new declaration.
1987 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1988 OverloadTys: F->getArg(i: 1)->getType());
1989 return true;
1990 } else if (F->arg_size() == 1 && Name == ".i64") {
1991 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1992 // Autoupgrade prior to
1993 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1994 // intrinsic with no expected calls. To allow auto-upgrade process to
1995 // delete such invalid intrinsic declaration, set NewFn = nullptr
1996 // and return true here. If there are actual calls to this intrinsic
1997 // (which is not expected), they will be deleted in
1998 // UpgradeIntrinsicCall.
1999 NewFn = nullptr;
2000 return true;
2001 }
2002 }
2003 break;
2004 }
2005 case 'm': {
2006 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
2007 // alignment parameter to embedding the alignment as an attribute of
2008 // the pointer args.
2009 if (unsigned ID = StringSwitch<unsigned>(Name)
2010 .StartsWith(S: "memcpy.", Value: Intrinsic::memcpy)
2011 .StartsWith(S: "memmove.", Value: Intrinsic::memmove)
2012 .Default(Value: 0)) {
2013 if (F->arg_size() == 5) {
2014 rename(GV: F);
2015 // Get the types of dest, src, and len
2016 ArrayRef<Type *> ParamTypes =
2017 F->getFunctionType()->params().slice(N: 0, M: 3);
2018 NewFn =
2019 Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: ParamTypes);
2020 return true;
2021 }
2022 }
2023 if (Name.starts_with(Prefix: "memset.") && F->arg_size() == 5) {
2024 rename(GV: F);
2025 // Get the types of dest, and len
2026 const auto *FT = F->getFunctionType();
2027 Type *ParamTypes[2] = {
2028 FT->getParamType(i: 0), // Dest
2029 FT->getParamType(i: 2) // len
2030 };
2031 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2032 id: Intrinsic::memset, OverloadTys: ParamTypes);
2033 return true;
2034 }
2035
2036 unsigned MaskedID =
2037 StringSwitch<unsigned>(Name)
2038 .StartsWith(S: "masked.load", Value: Intrinsic::masked_load)
2039 .StartsWith(S: "masked.gather", Value: Intrinsic::masked_gather)
2040 .StartsWith(S: "masked.store", Value: Intrinsic::masked_store)
2041 .StartsWith(S: "masked.scatter", Value: Intrinsic::masked_scatter)
2042 .Default(Value: 0);
2043 if (MaskedID && F->arg_size() == 4) {
2044 rename(GV: F);
2045 if (MaskedID == Intrinsic::masked_load ||
2046 MaskedID == Intrinsic::masked_gather) {
2047 NewFn = Intrinsic::getOrInsertDeclaration(
2048 M: F->getParent(), id: MaskedID,
2049 OverloadTys: {F->getReturnType(), F->getArg(i: 0)->getType()});
2050 return true;
2051 }
2052 NewFn = Intrinsic::getOrInsertDeclaration(
2053 M: F->getParent(), id: MaskedID,
2054 OverloadTys: {F->getArg(i: 0)->getType(), F->getArg(i: 1)->getType()});
2055 return true;
2056 }
2057 break;
2058 }
2059 case 'n': {
2060 if (Name.consume_front(Prefix: "nvvm.")) {
2061 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2062 if (F->arg_size() == 1) {
2063 Intrinsic::ID IID =
2064 StringSwitch<Intrinsic::ID>(Name)
2065 .Cases(CaseStrings: {"brev32", "brev64"}, Value: Intrinsic::bitreverse)
2066 .Case(S: "clz.i", Value: Intrinsic::ctlz)
2067 .Case(S: "popc.i", Value: Intrinsic::ctpop)
2068 .Default(Value: Intrinsic::not_intrinsic);
2069 if (IID != Intrinsic::not_intrinsic) {
2070 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2071 OverloadTys: {F->getReturnType()});
2072 return true;
2073 }
2074 } else if (F->arg_size() == 2) {
2075 Intrinsic::ID IID =
2076 StringSwitch<Intrinsic::ID>(Name)
2077 .Cases(CaseStrings: {"max.s", "max.i", "max.ll"}, Value: Intrinsic::smax)
2078 .Cases(CaseStrings: {"min.s", "min.i", "min.ll"}, Value: Intrinsic::smin)
2079 .Cases(CaseStrings: {"max.us", "max.ui", "max.ull"}, Value: Intrinsic::umax)
2080 .Cases(CaseStrings: {"min.us", "min.ui", "min.ull"}, Value: Intrinsic::umin)
2081 .Cases(CaseStrings: {"mulhi.s", "mulhi.i", "mulhi.ll"}, Value: Intrinsic::smulh)
2082 .Cases(CaseStrings: {"mulhi.us", "mulhi.ui", "mulhi.ull"}, Value: Intrinsic::umulh)
2083 .Default(Value: Intrinsic::not_intrinsic);
2084 if (IID != Intrinsic::not_intrinsic) {
2085 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2086 OverloadTys: {F->getReturnType()});
2087 return true;
2088 }
2089 }
2090
2091 // Check for nvvm intrinsics that need a return type adjustment.
2092 {
2093 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2094 if (IID != Intrinsic::not_intrinsic &&
2095 isLegacyNVPTXBF16IntSignature(F, IID)) {
2096 NewFn = nullptr;
2097 return true;
2098 }
2099 }
2100
2101 // Upgrade Distributed Shared Memory Intrinsics
2102 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2103 if (IID != Intrinsic::not_intrinsic) {
2104 rename(GV: F);
2105 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2106 return true;
2107 }
2108
2109 // Upgrade TMA reduction intrinsics
2110 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2111 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2112 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2113 if (IID != Intrinsic::not_intrinsic) {
2114 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2115 return true;
2116 }
2117
2118 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2119 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2120 if (IID != Intrinsic::not_intrinsic) {
2121 rename(GV: F);
2122 NewFn = Intrinsic::getOrInsertDeclaration(
2123 M: F->getParent(), IID, RetTy: F->getReturnType(),
2124 ArgTys: F->getFunctionType()->params());
2125 return true;
2126 }
2127
2128 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2129 // tcgen05.alloc shared variants to anyptr intrinsics.
2130 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2131 if (IID != Intrinsic::not_intrinsic) {
2132 rename(GV: F);
2133 if (Intrinsic::isOverloaded(id: IID))
2134 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2135 OverloadTys: {F->getArg(i: 0)->getType()});
2136 else
2137 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2138 return true;
2139 }
2140
2141 // Upgrade TMA copy G2S CTA intrinsics.
2142 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2143 if (IID != Intrinsic::not_intrinsic) {
2144 rename(GV: F);
2145 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2146 return true;
2147 }
2148
2149 // Upgrade TMA copy G2S (cluster) intrinsics.
2150 SmallVector<Type *, 1> OvlTys;
2151 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2152 if (IID != Intrinsic::not_intrinsic) {
2153 rename(GV: F);
2154 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID, OverloadTys: OvlTys);
2155 return true;
2156 }
2157
2158 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2159 // (multicast-mask overloading + trailing flag_valid_pattern).
2160 SmallVector<Type *, 1> BulkG2SOvlTys;
2161 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, OvlTys&: BulkG2SOvlTys);
2162 if (IID != Intrinsic::not_intrinsic) {
2163 rename(GV: F);
2164 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2165 OverloadTys: BulkG2SOvlTys);
2166 return true;
2167 }
2168
2169 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2170 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2171 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2172 if (IID != Intrinsic::not_intrinsic) {
2173 rename(GV: F);
2174 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2175 return true;
2176 }
2177
2178 // Upgrade mbarrier.init intrinsics missing the layout operand.
2179 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2180 if (IID != Intrinsic::not_intrinsic) {
2181 rename(GV: F);
2182 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
2183 OverloadTys: F->getArg(i: 0)->getType());
2184 return true;
2185 }
2186
2187 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2188 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2189 //
2190 // TODO: We could add lohi.i2d.
2191 bool Expand = false;
2192 if (Name.consume_front(Prefix: "abs."))
2193 // nvvm.abs.{i,ii}
2194 Expand =
2195 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2196 else if (Name.consume_front(Prefix: "fabs."))
2197 // nvvm.fabs.{f,ftz.f,d}
2198 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2199 else if (Name.consume_front(Prefix: "add."))
2200 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2201 Expand = getNVVMFPArithUpgrade(Name, IIDs: NVVMFAddIIDs).has_value();
2202 else if (Name.consume_front(Prefix: "mul."))
2203 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2204 Expand = getNVVMFPArithUpgrade(Name, IIDs: NVVMFMulIIDs).has_value();
2205 else if (Name.consume_front(Prefix: "ex2.approx."))
2206 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2207 Expand =
2208 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2209 else if (Name.consume_front(Prefix: "atomic.load."))
2210 // nvvm.atomic.load.add.{f32,f64}.p
2211 // nvvm.atomic.load.{inc,dec}.32.p
2212 Expand = StringSwitch<bool>(Name)
2213 .StartsWith(S: "add.f32.p", Value: true)
2214 .StartsWith(S: "add.f64.p", Value: true)
2215 .StartsWith(S: "inc.32.p", Value: true)
2216 .StartsWith(S: "dec.32.p", Value: true)
2217 .Default(Value: false);
2218 else if (Name.consume_front(Prefix: "atomic."))
2219 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2220 // nvvm.atomic.cas.gen.i.{cta,sys}
2221 Expand = StringSwitch<bool>(Name)
2222 .StartsWith(S: "add.gen.", Value: true)
2223 .StartsWith(S: "exch.gen.", Value: true)
2224 .StartsWith(S: "max.gen.", Value: true)
2225 .StartsWith(S: "min.gen.", Value: true)
2226 .StartsWith(S: "inc.gen.", Value: true)
2227 .StartsWith(S: "dec.gen.", Value: true)
2228 .StartsWith(S: "and.gen.", Value: true)
2229 .StartsWith(S: "or.gen.", Value: true)
2230 .StartsWith(S: "xor.gen.", Value: true)
2231 .StartsWith(S: "cas.gen.", Value: true)
2232 .Default(Value: false);
2233 else if (Name.consume_front(Prefix: "bitcast."))
2234 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2235 Expand =
2236 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2237 else if (Name.consume_front(Prefix: "rotate."))
2238 // nvvm.rotate.{b32,b64,right.b64}
2239 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2240 else if (Name.consume_front(Prefix: "ptr.gen.to."))
2241 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2242 Expand = consumeNVVMPtrAddrSpace(Name);
2243 else if (Name.consume_front(Prefix: "ptr."))
2244 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2245 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(Prefix: ".to.gen");
2246 else if (Name.consume_front(Prefix: "ldg.global."))
2247 // nvvm.ldg.global.{i,p,f}
2248 Expand = (Name.starts_with(Prefix: "i.") || Name.starts_with(Prefix: "f.") ||
2249 Name.starts_with(Prefix: "p."));
2250 else
2251 Expand = StringSwitch<bool>(Name)
2252 .Case(S: "barrier0", Value: true)
2253 .Case(S: "barrier.n", Value: true)
2254 .Case(S: "barrier.sync.cnt", Value: true)
2255 .Case(S: "barrier.sync", Value: true)
2256 .Case(S: "barrier", Value: true)
2257 .Case(S: "bar.sync", Value: true)
2258 .Case(S: "barrier0.popc", Value: true)
2259 .Case(S: "barrier0.and", Value: true)
2260 .Case(S: "barrier0.or", Value: true)
2261 .Case(S: "clz.ll", Value: true)
2262 .Case(S: "popc.ll", Value: true)
2263 .Case(S: "h2f", Value: true)
2264 .Case(S: "swap.lo.hi.b64", Value: true)
2265 .Case(S: "tanh.approx.f32", Value: true)
2266 .Default(Value: false);
2267
2268 if (Expand) {
2269 NewFn = nullptr;
2270 return true;
2271 }
2272 break; // No other 'nvvm.*'.
2273 }
2274 break;
2275 }
2276 case 'o':
2277 if (Name.starts_with(Prefix: "objectsize.")) {
2278 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2279 if (F->arg_size() == 2 || F->arg_size() == 3) {
2280 rename(GV: F);
2281 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
2282 id: Intrinsic::objectsize, OverloadTys: Tys);
2283 return true;
2284 }
2285 }
2286 break;
2287
2288 case 'p':
2289 if (Name.starts_with(Prefix: "ptr.annotation.") && F->arg_size() == 4) {
2290 rename(GV: F);
2291 NewFn = Intrinsic::getOrInsertDeclaration(
2292 M: F->getParent(), id: Intrinsic::ptr_annotation,
2293 OverloadTys: {F->arg_begin()->getType(), F->getArg(i: 1)->getType()});
2294 return true;
2295 }
2296 break;
2297
2298 case 'r': {
2299 if (Name.consume_front(Prefix: "riscv.")) {
2300 Intrinsic::ID ID;
2301 ID = StringSwitch<Intrinsic::ID>(Name)
2302 .Case(S: "aes32dsi", Value: Intrinsic::riscv_aes32dsi)
2303 .Case(S: "aes32dsmi", Value: Intrinsic::riscv_aes32dsmi)
2304 .Case(S: "aes32esi", Value: Intrinsic::riscv_aes32esi)
2305 .Case(S: "aes32esmi", Value: Intrinsic::riscv_aes32esmi)
2306 .Default(Value: Intrinsic::not_intrinsic);
2307 if (ID != Intrinsic::not_intrinsic) {
2308 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32)) {
2309 rename(GV: F);
2310 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2311 return true;
2312 }
2313 break; // No other applicable upgrades.
2314 }
2315
2316 ID = StringSwitch<Intrinsic::ID>(Name)
2317 .StartsWith(S: "sm4ks", Value: Intrinsic::riscv_sm4ks)
2318 .StartsWith(S: "sm4ed", Value: Intrinsic::riscv_sm4ed)
2319 .Default(Value: Intrinsic::not_intrinsic);
2320 if (ID != Intrinsic::not_intrinsic) {
2321 if (!F->getFunctionType()->getParamType(i: 2)->isIntegerTy(BitWidth: 32) ||
2322 F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2323 rename(GV: F);
2324 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2325 return true;
2326 }
2327 break; // No other applicable upgrades.
2328 }
2329
2330 ID = StringSwitch<Intrinsic::ID>(Name)
2331 .StartsWith(S: "sha256sig0", Value: Intrinsic::riscv_sha256sig0)
2332 .StartsWith(S: "sha256sig1", Value: Intrinsic::riscv_sha256sig1)
2333 .StartsWith(S: "sha256sum0", Value: Intrinsic::riscv_sha256sum0)
2334 .StartsWith(S: "sha256sum1", Value: Intrinsic::riscv_sha256sum1)
2335 .StartsWith(S: "sm3p0", Value: Intrinsic::riscv_sm3p0)
2336 .StartsWith(S: "sm3p1", Value: Intrinsic::riscv_sm3p1)
2337 .Default(Value: Intrinsic::not_intrinsic);
2338 if (ID != Intrinsic::not_intrinsic) {
2339 if (F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: 64)) {
2340 rename(GV: F);
2341 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2342 return true;
2343 }
2344 break; // No other applicable upgrades.
2345 }
2346
2347 // Replace llvm.riscv.clmul with llvm.clmul.
2348 if (Name == "clmul.i32" || Name == "clmul.i64") {
2349 NewFn = Intrinsic::getOrInsertDeclaration(
2350 M: F->getParent(), id: Intrinsic::clmul, OverloadTys: {F->getReturnType()});
2351 return true;
2352 }
2353
2354 break; // No other 'riscv.*' intrinsics
2355 }
2356 } break;
2357
2358 case 's':
2359 if (Name == "stackprotectorcheck") {
2360 NewFn = nullptr;
2361 return true;
2362 }
2363 if (Name.starts_with(Prefix: "strip.invariant.group")) {
2364 // For clang's usage it would be safe to just drop the
2365 // strip.invariant.group, but to be conservative replace with the
2366 // stronger launder.invariant.group instead.
2367 NewFn = Intrinsic::getOrInsertDeclaration(
2368 M: F->getParent(), id: Intrinsic::launder_invariant_group,
2369 OverloadTys: F->getReturnType());
2370 return true;
2371 }
2372 break;
2373
2374 case 't':
2375 if (Name == "thread.pointer") {
2376 NewFn = Intrinsic::getOrInsertDeclaration(
2377 M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
2378 return true;
2379 }
2380 break;
2381
2382 case 'v': {
2383 if (Name == "var.annotation" && F->arg_size() == 4) {
2384 rename(GV: F);
2385 NewFn = Intrinsic::getOrInsertDeclaration(
2386 M: F->getParent(), id: Intrinsic::var_annotation,
2387 OverloadTys: {{F->arg_begin()->getType(), F->getArg(i: 1)->getType()}});
2388 return true;
2389 }
2390 if (Name.consume_front(Prefix: "vector.splice")) {
2391 if (Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"))
2392 break;
2393 return true;
2394 }
2395 if (shouldUpgradeVPIntrinsic(Name))
2396 return true;
2397 break;
2398 }
2399
2400 case 'w':
2401 if (Name.consume_front(Prefix: "wasm.")) {
2402 Intrinsic::ID ID =
2403 StringSwitch<Intrinsic::ID>(Name)
2404 .StartsWith(S: "fma.", Value: Intrinsic::wasm_relaxed_madd)
2405 .StartsWith(S: "fms.", Value: Intrinsic::wasm_relaxed_nmadd)
2406 .StartsWith(S: "laneselect.", Value: Intrinsic::wasm_relaxed_laneselect)
2407 .Default(Value: Intrinsic::not_intrinsic);
2408 if (ID != Intrinsic::not_intrinsic) {
2409 rename(GV: F);
2410 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
2411 OverloadTys: F->getReturnType());
2412 return true;
2413 }
2414
2415 if (Name.consume_front(Prefix: "dot.i8x16.i7x16.")) {
2416 ID = StringSwitch<Intrinsic::ID>(Name)
2417 .Case(S: "signed", Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2418 .Case(S: "add.signed",
2419 Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2420 .Default(Value: Intrinsic::not_intrinsic);
2421 if (ID != Intrinsic::not_intrinsic) {
2422 rename(GV: F);
2423 NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
2424 return true;
2425 }
2426 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2427 }
2428 break; // No other 'wasm.*'.
2429 }
2430 break;
2431
2432 case 'x':
2433 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2434 return true;
2435 }
2436
2437 auto *ST = dyn_cast<StructType>(Val: F->getReturnType());
2438 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2439 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2440 // Replace return type with literal non-packed struct. Only do this for
2441 // intrinsics declared to return a struct, not for intrinsics with
2442 // overloaded return type, in which case the exact struct type will be
2443 // mangled into the name.
2444 if (Intrinsic::hasStructReturnType(id: F->getIntrinsicID())) {
2445 FunctionType *FT = F->getFunctionType();
2446 auto *NewST = StructType::get(Context&: ST->getContext(), Elements: ST->elements());
2447 auto *NewFT = FunctionType::get(Result: NewST, Params: FT->params(), isVarArg: FT->isVarArg());
2448 std::string Name = F->getName().str();
2449 rename(GV: F);
2450 NewFn = Function::Create(Ty: NewFT, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
2451 N: Name, M: F->getParent());
2452
2453 // The new function may also need remangling.
2454 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F: NewFn))
2455 NewFn = *Result;
2456 return true;
2457 }
2458 }
2459
2460 // Remangle our intrinsic since we upgrade the mangling
2461 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2462 if (Result != std::nullopt) {
2463 NewFn = *Result;
2464 return true;
2465 }
2466
2467 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2468 return true;
2469
2470 // This may not belong here. This function is effectively being overloaded
2471 // to both detect an intrinsic which needs upgrading, and to provide the
2472 // upgraded form of the intrinsic. We should perhaps have two separate
2473 // functions for this.
2474
2475 return false;
2476}
2477
2478bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2479 bool CanUpgradeDebugIntrinsicsToRecords) {
2480 NewFn = nullptr;
2481 bool Upgraded =
2482 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2483
2484 // Upgrade intrinsic attributes. This does not change the function.
2485 if (NewFn)
2486 F = NewFn;
2487 if (Intrinsic::ID id = F->getIntrinsicID()) {
2488 // Only do this if the intrinsic signature is valid.
2489 SmallVector<Type *> OverloadTys;
2490 if (Intrinsic::isSignatureValid(ID: id, FT: F->getFunctionType(), OverloadTys))
2491 F->setAttributes(
2492 Intrinsic::getAttributes(C&: F->getContext(), id, FT: F->getFunctionType()));
2493 }
2494 return Upgraded;
2495}
2496
2497GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2498 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2499 GV->getName() == "llvm.global_dtors")) ||
2500 !GV->hasInitializer())
2501 return nullptr;
2502 ArrayType *ATy = dyn_cast<ArrayType>(Val: GV->getValueType());
2503 if (!ATy)
2504 return nullptr;
2505 StructType *STy = dyn_cast<StructType>(Val: ATy->getElementType());
2506 if (!STy || STy->getNumElements() != 2)
2507 return nullptr;
2508
2509 IRBuilder<> IRB(*GV->getParent());
2510 auto EltTy = StructType::get(elt1: STy->getElementType(N: 0), elts: STy->getElementType(N: 1),
2511 elts: IRB.getPtrTy());
2512 Constant *Init = GV->getInitializer();
2513 unsigned N = Init->getNumOperands();
2514 std::vector<Constant *> NewCtors(N);
2515 for (unsigned i = 0; i != N; ++i) {
2516 auto Ctor = cast<Constant>(Val: Init->getOperand(i));
2517 NewCtors[i] = ConstantStruct::get(T: EltTy, Vs: Ctor->getAggregateElement(Elt: 0u),
2518 Vs: Ctor->getAggregateElement(Elt: 1),
2519 Vs: ConstantPointerNull::get(T: IRB.getPtrTy()));
2520 }
2521 Constant *NewInit = ConstantArray::get(T: ArrayType::get(ElementType: EltTy, NumElements: N), V: NewCtors);
2522
2523 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2524 NewInit, GV->getName());
2525}
2526
2527// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2528// to byte shuffles.
2529static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2530 unsigned Shift) {
2531 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2532 unsigned NumElts = ResultTy->getNumElements() * 8;
2533
2534 // Bitcast from a 64-bit element type to a byte element type.
2535 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2536 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2537
2538 // We'll be shuffling in zeroes.
2539 Value *Res = Constant::getNullValue(Ty: VecTy);
2540
2541 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2542 // we'll just return the zero vector.
2543 if (Shift < 16) {
2544 int Idxs[64];
2545 // 256/512-bit version is split into 2/4 16-byte lanes.
2546 for (unsigned l = 0; l != NumElts; l += 16)
2547 for (unsigned i = 0; i != 16; ++i) {
2548 unsigned Idx = NumElts + i - Shift;
2549 if (Idx < NumElts)
2550 Idx -= NumElts - 16; // end of lane, switch operand.
2551 Idxs[l + i] = Idx + l;
2552 }
2553
2554 Res = Builder.CreateShuffleVector(V1: Res, V2: Op, Mask: ArrayRef(Idxs, NumElts));
2555 }
2556
2557 // Bitcast back to a 64-bit element type.
2558 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2559}
2560
2561// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2562// to byte shuffles.
2563static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2564 unsigned Shift) {
2565 auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2566 unsigned NumElts = ResultTy->getNumElements() * 8;
2567
2568 // Bitcast from a 64-bit element type to a byte element type.
2569 Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2570 Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2571
2572 // We'll be shuffling in zeroes.
2573 Value *Res = Constant::getNullValue(Ty: VecTy);
2574
2575 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2576 // we'll just return the zero vector.
2577 if (Shift < 16) {
2578 int Idxs[64];
2579 // 256/512-bit version is split into 2/4 16-byte lanes.
2580 for (unsigned l = 0; l != NumElts; l += 16)
2581 for (unsigned i = 0; i != 16; ++i) {
2582 unsigned Idx = i + Shift;
2583 if (Idx >= 16)
2584 Idx += NumElts - 16; // end of lane, switch operand.
2585 Idxs[l + i] = Idx + l;
2586 }
2587
2588 Res = Builder.CreateShuffleVector(V1: Op, V2: Res, Mask: ArrayRef(Idxs, NumElts));
2589 }
2590
2591 // Bitcast back to a 64-bit element type.
2592 return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2593}
2594
2595static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2596 unsigned NumElts) {
2597 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2598 llvm::VectorType *MaskTy = FixedVectorType::get(
2599 ElementType: Builder.getInt1Ty(), NumElts: cast<IntegerType>(Val: Mask->getType())->getBitWidth());
2600 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2601
2602 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2603 // i8 and we need to extract down to the right number of elements.
2604 if (NumElts <= 4) {
2605 int Indices[4];
2606 for (unsigned i = 0; i != NumElts; ++i)
2607 Indices[i] = i;
2608 Mask = Builder.CreateShuffleVector(V1: Mask, V2: Mask, Mask: ArrayRef(Indices, NumElts),
2609 Name: "extract");
2610 }
2611
2612 return Mask;
2613}
2614
2615static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2616 Value *Op1) {
2617 // If the mask is all ones just emit the first operation.
2618 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2619 if (C->isAllOnesValue())
2620 return Op0;
2621
2622 Mask = getX86MaskVec(Builder, Mask,
2623 NumElts: cast<FixedVectorType>(Val: Op0->getType())->getNumElements());
2624 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2625}
2626
2627static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2628 Value *Op1) {
2629 // If the mask is all ones just emit the first operation.
2630 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2631 if (C->isAllOnesValue())
2632 return Op0;
2633
2634 auto *MaskTy = FixedVectorType::get(ElementType: Builder.getInt1Ty(),
2635 NumElts: Mask->getType()->getIntegerBitWidth());
2636 Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2637 Mask = Builder.CreateExtractElement(Vec: Mask, Idx: (uint64_t)0);
2638 return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2639}
2640
2641// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2642// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2643// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2644static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2645 Value *Op1, Value *Shift,
2646 Value *Passthru, Value *Mask,
2647 bool IsVALIGN) {
2648 unsigned ShiftVal = cast<llvm::ConstantInt>(Val: Shift)->getZExtValue();
2649
2650 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2651 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2652 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2653 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2654
2655 // Mask the immediate for VALIGN.
2656 if (IsVALIGN)
2657 ShiftVal &= (NumElts - 1);
2658
2659 // If palignr is shifting the pair of vectors more than the size of two
2660 // lanes, emit zero.
2661 if (ShiftVal >= 32)
2662 return llvm::Constant::getNullValue(Ty: Op0->getType());
2663
2664 // If palignr is shifting the pair of input vectors more than one lane,
2665 // but less than two lanes, convert to shifting in zeroes.
2666 if (ShiftVal > 16) {
2667 ShiftVal -= 16;
2668 Op1 = Op0;
2669 Op0 = llvm::Constant::getNullValue(Ty: Op0->getType());
2670 }
2671
2672 int Indices[64];
2673 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2674 for (unsigned l = 0; l < NumElts; l += 16) {
2675 for (unsigned i = 0; i != 16; ++i) {
2676 unsigned Idx = ShiftVal + i;
2677 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2678 Idx += NumElts - 16; // End of lane, switch operand.
2679 Indices[l + i] = Idx + l;
2680 }
2681 }
2682
2683 Value *Align = Builder.CreateShuffleVector(
2684 V1: Op1, V2: Op0, Mask: ArrayRef(Indices, NumElts), Name: "palignr");
2685
2686 return emitX86Select(Builder, Mask, Op0: Align, Op1: Passthru);
2687}
2688
2689static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2690 bool ZeroMask, bool IndexForm) {
2691 Type *Ty = CI.getType();
2692 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2693 unsigned EltWidth = Ty->getScalarSizeInBits();
2694 bool IsFloat = Ty->isFPOrFPVectorTy();
2695 Intrinsic::ID IID;
2696 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2697 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2698 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2699 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2700 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2701 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2702 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2703 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2704 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2705 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2706 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2707 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2708 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2709 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2710 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2711 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2712 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2713 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2714 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2715 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2716 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2717 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2718 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2719 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2720 else if (VecWidth == 128 && EltWidth == 16)
2721 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2722 else if (VecWidth == 256 && EltWidth == 16)
2723 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2724 else if (VecWidth == 512 && EltWidth == 16)
2725 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2726 else if (VecWidth == 128 && EltWidth == 8)
2727 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2728 else if (VecWidth == 256 && EltWidth == 8)
2729 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2730 else if (VecWidth == 512 && EltWidth == 8)
2731 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2732 else
2733 llvm_unreachable("Unexpected intrinsic");
2734
2735 Value *Args[] = { CI.getArgOperand(i: 0) , CI.getArgOperand(i: 1),
2736 CI.getArgOperand(i: 2) };
2737
2738 // If this isn't index form we need to swap operand 0 and 1.
2739 if (!IndexForm)
2740 std::swap(a&: Args[0], b&: Args[1]);
2741
2742 Value *V = Builder.CreateIntrinsic(ID: IID, Args);
2743 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2744 : Builder.CreateBitCast(V: CI.getArgOperand(i: 1),
2745 DestTy: Ty);
2746 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: V, Op1: PassThru);
2747}
2748
2749static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2750 Intrinsic::ID IID) {
2751 Type *Ty = CI.getType();
2752 Value *Op0 = CI.getOperand(i_nocapture: 0);
2753 Value *Op1 = CI.getOperand(i_nocapture: 1);
2754 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1});
2755
2756 if (CI.arg_size() == 4) { // For masked intrinsics.
2757 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2758 Value *Mask = CI.getOperand(i_nocapture: 3);
2759 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2760 }
2761 return Res;
2762}
2763
2764static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2765 bool IsRotateRight) {
2766 Type *Ty = CI.getType();
2767 Value *Src = CI.getArgOperand(i: 0);
2768 Value *Amt = CI.getArgOperand(i: 1);
2769
2770 // Amount may be scalar immediate, in which case create a splat vector.
2771 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2772 // we only care about the lowest log2 bits anyway.
2773 if (Amt->getType() != Ty) {
2774 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2775 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2776 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2777 }
2778
2779 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2780 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Src, Src, Amt});
2781
2782 if (CI.arg_size() == 4) { // For masked intrinsics.
2783 Value *VecSrc = CI.getOperand(i_nocapture: 2);
2784 Value *Mask = CI.getOperand(i_nocapture: 3);
2785 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2786 }
2787 return Res;
2788}
2789
2790static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2791 bool IsSigned) {
2792 Type *Ty = CI.getType();
2793 Value *LHS = CI.getArgOperand(i: 0);
2794 Value *RHS = CI.getArgOperand(i: 1);
2795
2796 CmpInst::Predicate Pred;
2797 switch (Imm) {
2798 case 0x0:
2799 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2800 break;
2801 case 0x1:
2802 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2803 break;
2804 case 0x2:
2805 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2806 break;
2807 case 0x3:
2808 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2809 break;
2810 case 0x4:
2811 Pred = ICmpInst::ICMP_EQ;
2812 break;
2813 case 0x5:
2814 Pred = ICmpInst::ICMP_NE;
2815 break;
2816 case 0x6:
2817 return Constant::getNullValue(Ty); // FALSE
2818 case 0x7:
2819 return Constant::getAllOnesValue(Ty); // TRUE
2820 default:
2821 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2822 }
2823
2824 Value *Cmp = Builder.CreateICmp(P: Pred, LHS, RHS);
2825 Value *Ext = Builder.CreateSExt(V: Cmp, DestTy: Ty);
2826 return Ext;
2827}
2828
2829static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2830 bool IsShiftRight, bool ZeroMask) {
2831 Type *Ty = CI.getType();
2832 Value *Op0 = CI.getArgOperand(i: 0);
2833 Value *Op1 = CI.getArgOperand(i: 1);
2834 Value *Amt = CI.getArgOperand(i: 2);
2835
2836 if (IsShiftRight)
2837 std::swap(a&: Op0, b&: Op1);
2838
2839 // Amount may be scalar immediate, in which case create a splat vector.
2840 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2841 // we only care about the lowest log2 bits anyway.
2842 if (Amt->getType() != Ty) {
2843 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2844 Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2845 Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2846 }
2847
2848 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2849 Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1, Amt});
2850
2851 unsigned NumArgs = CI.arg_size();
2852 if (NumArgs >= 4) { // For masked intrinsics.
2853 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(i: 3) :
2854 ZeroMask ? ConstantAggregateZero::get(Ty: CI.getType()) :
2855 CI.getArgOperand(i: 0);
2856 Value *Mask = CI.getOperand(i_nocapture: NumArgs - 1);
2857 Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2858 }
2859 return Res;
2860}
2861
2862static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2863 Value *Mask, bool Aligned) {
2864 const Align Alignment =
2865 Aligned
2866 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2867 : Align(1);
2868
2869 // If the mask is all ones just emit a regular store.
2870 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2871 if (C->isAllOnesValue())
2872 return Builder.CreateAlignedStore(Val: Data, Ptr, Align: Alignment);
2873
2874 // Convert the mask from an integer type to a vector of i1.
2875 unsigned NumElts = cast<FixedVectorType>(Val: Data->getType())->getNumElements();
2876 Mask = getX86MaskVec(Builder, Mask, NumElts);
2877 return Builder.CreateMaskedStore(Val: Data, Ptr, Alignment, Mask);
2878}
2879
2880static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2881 Value *Passthru, Value *Mask, bool Aligned) {
2882 Type *ValTy = Passthru->getType();
2883 const Align Alignment =
2884 Aligned
2885 ? Align(
2886 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2887 8)
2888 : Align(1);
2889
2890 // If the mask is all ones just emit a regular store.
2891 if (const auto *C = dyn_cast<Constant>(Val: Mask))
2892 if (C->isAllOnesValue())
2893 return Builder.CreateAlignedLoad(Ty: ValTy, Ptr, Align: Alignment);
2894
2895 // Convert the mask from an integer type to a vector of i1.
2896 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
2897 Mask = getX86MaskVec(Builder, Mask, NumElts);
2898 return Builder.CreateMaskedLoad(Ty: ValTy, Ptr, Alignment, Mask, PassThru: Passthru);
2899}
2900
2901static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2902 Type *Ty = CI.getType();
2903 Value *Op0 = CI.getArgOperand(i: 0);
2904 Value *Res = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: Ty,
2905 Args: {Op0, Builder.getInt1(V: false)});
2906 if (CI.arg_size() == 3)
2907 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 2), Op0: Res, Op1: CI.getArgOperand(i: 1));
2908 return Res;
2909}
2910
2911static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2912 Type *Ty = CI.getType();
2913
2914 // Arguments have a vXi32 type so cast to vXi64.
2915 Value *LHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 0), DestTy: Ty);
2916 Value *RHS = Builder.CreateBitCast(V: CI.getArgOperand(i: 1), DestTy: Ty);
2917
2918 if (IsSigned) {
2919 // Shift left then arithmetic shift right.
2920 Constant *ShiftAmt = ConstantInt::get(Ty, V: 32);
2921 LHS = Builder.CreateShl(LHS, RHS: ShiftAmt);
2922 LHS = Builder.CreateAShr(LHS, RHS: ShiftAmt);
2923 RHS = Builder.CreateShl(LHS: RHS, RHS: ShiftAmt);
2924 RHS = Builder.CreateAShr(LHS: RHS, RHS: ShiftAmt);
2925 } else {
2926 // Clear the upper bits.
2927 Constant *Mask = ConstantInt::get(Ty, V: 0xffffffff);
2928 LHS = Builder.CreateAnd(LHS, RHS: Mask);
2929 RHS = Builder.CreateAnd(LHS: RHS, RHS: Mask);
2930 }
2931
2932 Value *Res = Builder.CreateMul(LHS, RHS);
2933
2934 if (CI.arg_size() == 4)
2935 Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Res, Op1: CI.getArgOperand(i: 2));
2936
2937 return Res;
2938}
2939
2940// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2941static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2942 Value *Mask) {
2943 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2944 if (Mask) {
2945 const auto *C = dyn_cast<Constant>(Val: Mask);
2946 if (!C || !C->isAllOnesValue())
2947 Vec = Builder.CreateAnd(LHS: Vec, RHS: getX86MaskVec(Builder, Mask, NumElts));
2948 }
2949
2950 if (NumElts < 8) {
2951 int Indices[8];
2952 for (unsigned i = 0; i != NumElts; ++i)
2953 Indices[i] = i;
2954 for (unsigned i = NumElts; i != 8; ++i)
2955 Indices[i] = NumElts + i % NumElts;
2956 Vec = Builder.CreateShuffleVector(V1: Vec,
2957 V2: Constant::getNullValue(Ty: Vec->getType()),
2958 Mask: Indices);
2959 }
2960 return Builder.CreateBitCast(V: Vec, DestTy: Builder.getIntNTy(N: std::max(a: NumElts, b: 8U)));
2961}
2962
2963static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2964 unsigned CC, bool Signed) {
2965 Value *Op0 = CI.getArgOperand(i: 0);
2966 unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2967
2968 Value *Cmp;
2969 if (CC == 3) {
2970 Cmp = Constant::getNullValue(
2971 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2972 } else if (CC == 7) {
2973 Cmp = Constant::getAllOnesValue(
2974 Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2975 } else {
2976 ICmpInst::Predicate Pred;
2977 switch (CC) {
2978 default: llvm_unreachable("Unknown condition code");
2979 case 0: Pred = ICmpInst::ICMP_EQ; break;
2980 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2981 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2982 case 4: Pred = ICmpInst::ICMP_NE; break;
2983 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2984 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2985 }
2986 Cmp = Builder.CreateICmp(P: Pred, LHS: Op0, RHS: CI.getArgOperand(i: 1));
2987 }
2988
2989 Value *Mask = CI.getArgOperand(i: CI.arg_size() - 1);
2990
2991 return applyX86MaskOn1BitsVec(Builder, Vec: Cmp, Mask);
2992}
2993
2994// Replace a masked intrinsic with an older unmasked intrinsic.
2995static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2996 Intrinsic::ID IID) {
2997 Value *Rep =
2998 Builder.CreateIntrinsic(ID: IID, Args: {CI.getArgOperand(i: 0), CI.getArgOperand(i: 1)});
2999 return emitX86Select(Builder, Mask: CI.getArgOperand(i: 3), Op0: Rep, Op1: CI.getArgOperand(i: 2));
3000}
3001
3002static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
3003 Value* A = CI.getArgOperand(i: 0);
3004 Value* B = CI.getArgOperand(i: 1);
3005 Value* Src = CI.getArgOperand(i: 2);
3006 Value* Mask = CI.getArgOperand(i: 3);
3007
3008 Value* AndNode = Builder.CreateAnd(LHS: Mask, RHS: APInt(8, 1));
3009 Value* Cmp = Builder.CreateIsNotNull(Arg: AndNode);
3010 Value* Extract1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
3011 Value* Extract2 = Builder.CreateExtractElement(Vec: Src, Idx: (uint64_t)0);
3012 Value* Select = Builder.CreateSelect(C: Cmp, True: Extract1, False: Extract2);
3013 return Builder.CreateInsertElement(Vec: A, NewElt: Select, Idx: (uint64_t)0);
3014}
3015
3016static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
3017 Value* Op = CI.getArgOperand(i: 0);
3018 Type* ReturnOp = CI.getType();
3019 unsigned NumElts = cast<FixedVectorType>(Val: CI.getType())->getNumElements();
3020 Value *Mask = getX86MaskVec(Builder, Mask: Op, NumElts);
3021 return Builder.CreateSExt(V: Mask, DestTy: ReturnOp, Name: "vpmovm2");
3022}
3023
3024// Replace intrinsic with unmasked version and a select.
3025static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
3026 CallBase &CI, Value *&Rep) {
3027 Name = Name.substr(Start: 12); // Remove avx512.mask.
3028
3029 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3030 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3031 Intrinsic::ID IID;
3032 if (Name.starts_with(Prefix: "max.p")) {
3033 if (VecWidth == 128 && EltWidth == 32)
3034 IID = Intrinsic::x86_sse_max_ps;
3035 else if (VecWidth == 128 && EltWidth == 64)
3036 IID = Intrinsic::x86_sse2_max_pd;
3037 else if (VecWidth == 256 && EltWidth == 32)
3038 IID = Intrinsic::x86_avx_max_ps_256;
3039 else if (VecWidth == 256 && EltWidth == 64)
3040 IID = Intrinsic::x86_avx_max_pd_256;
3041 else
3042 llvm_unreachable("Unexpected intrinsic");
3043 } else if (Name.starts_with(Prefix: "min.p")) {
3044 if (VecWidth == 128 && EltWidth == 32)
3045 IID = Intrinsic::x86_sse_min_ps;
3046 else if (VecWidth == 128 && EltWidth == 64)
3047 IID = Intrinsic::x86_sse2_min_pd;
3048 else if (VecWidth == 256 && EltWidth == 32)
3049 IID = Intrinsic::x86_avx_min_ps_256;
3050 else if (VecWidth == 256 && EltWidth == 64)
3051 IID = Intrinsic::x86_avx_min_pd_256;
3052 else
3053 llvm_unreachable("Unexpected intrinsic");
3054 } else if (Name.starts_with(Prefix: "pshuf.b.")) {
3055 if (VecWidth == 128)
3056 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3057 else if (VecWidth == 256)
3058 IID = Intrinsic::x86_avx2_pshuf_b;
3059 else if (VecWidth == 512)
3060 IID = Intrinsic::x86_avx512_pshuf_b_512;
3061 else
3062 llvm_unreachable("Unexpected intrinsic");
3063 } else if (Name.starts_with(Prefix: "pmul.hr.sw.")) {
3064 if (VecWidth == 128)
3065 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3066 else if (VecWidth == 256)
3067 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3068 else if (VecWidth == 512)
3069 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3070 else
3071 llvm_unreachable("Unexpected intrinsic");
3072 } else if (Name.starts_with(Prefix: "pmulh.w")) {
3073 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3074 "Unexpected intrinsic");
3075 Rep = upgradeX86BinaryIntrinsics(Builder, CI, IID: Intrinsic::smulh);
3076 return true;
3077 } else if (Name.starts_with(Prefix: "pmulhu.w")) {
3078 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3079 "Unexpected intrinsic");
3080 Rep = upgradeX86BinaryIntrinsics(Builder, CI, IID: Intrinsic::umulh);
3081 return true;
3082 } else if (Name.starts_with(Prefix: "pmaddw.d.")) {
3083 if (VecWidth == 128)
3084 IID = Intrinsic::x86_sse2_pmadd_wd;
3085 else if (VecWidth == 256)
3086 IID = Intrinsic::x86_avx2_pmadd_wd;
3087 else if (VecWidth == 512)
3088 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3089 else
3090 llvm_unreachable("Unexpected intrinsic");
3091 } else if (Name.starts_with(Prefix: "pmaddubs.w.")) {
3092 if (VecWidth == 128)
3093 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3094 else if (VecWidth == 256)
3095 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3096 else if (VecWidth == 512)
3097 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3098 else
3099 llvm_unreachable("Unexpected intrinsic");
3100 } else if (Name.starts_with(Prefix: "packsswb.")) {
3101 if (VecWidth == 128)
3102 IID = Intrinsic::x86_sse2_packsswb_128;
3103 else if (VecWidth == 256)
3104 IID = Intrinsic::x86_avx2_packsswb;
3105 else if (VecWidth == 512)
3106 IID = Intrinsic::x86_avx512_packsswb_512;
3107 else
3108 llvm_unreachable("Unexpected intrinsic");
3109 } else if (Name.starts_with(Prefix: "packssdw.")) {
3110 if (VecWidth == 128)
3111 IID = Intrinsic::x86_sse2_packssdw_128;
3112 else if (VecWidth == 256)
3113 IID = Intrinsic::x86_avx2_packssdw;
3114 else if (VecWidth == 512)
3115 IID = Intrinsic::x86_avx512_packssdw_512;
3116 else
3117 llvm_unreachable("Unexpected intrinsic");
3118 } else if (Name.starts_with(Prefix: "packuswb.")) {
3119 if (VecWidth == 128)
3120 IID = Intrinsic::x86_sse2_packuswb_128;
3121 else if (VecWidth == 256)
3122 IID = Intrinsic::x86_avx2_packuswb;
3123 else if (VecWidth == 512)
3124 IID = Intrinsic::x86_avx512_packuswb_512;
3125 else
3126 llvm_unreachable("Unexpected intrinsic");
3127 } else if (Name.starts_with(Prefix: "packusdw.")) {
3128 if (VecWidth == 128)
3129 IID = Intrinsic::x86_sse41_packusdw;
3130 else if (VecWidth == 256)
3131 IID = Intrinsic::x86_avx2_packusdw;
3132 else if (VecWidth == 512)
3133 IID = Intrinsic::x86_avx512_packusdw_512;
3134 else
3135 llvm_unreachable("Unexpected intrinsic");
3136 } else if (Name.starts_with(Prefix: "vpermilvar.")) {
3137 if (VecWidth == 128 && EltWidth == 32)
3138 IID = Intrinsic::x86_avx_vpermilvar_ps;
3139 else if (VecWidth == 128 && EltWidth == 64)
3140 IID = Intrinsic::x86_avx_vpermilvar_pd;
3141 else if (VecWidth == 256 && EltWidth == 32)
3142 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3143 else if (VecWidth == 256 && EltWidth == 64)
3144 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3145 else if (VecWidth == 512 && EltWidth == 32)
3146 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3147 else if (VecWidth == 512 && EltWidth == 64)
3148 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3149 else
3150 llvm_unreachable("Unexpected intrinsic");
3151 } else if (Name == "cvtpd2dq.256") {
3152 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3153 } else if (Name == "cvtpd2ps.256") {
3154 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3155 } else if (Name == "cvttpd2dq.256") {
3156 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3157 } else if (Name == "cvttps2dq.128") {
3158 IID = Intrinsic::x86_sse2_cvttps2dq;
3159 } else if (Name == "cvttps2dq.256") {
3160 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3161 } else if (Name.starts_with(Prefix: "permvar.")) {
3162 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3163 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3164 IID = Intrinsic::x86_avx2_permps;
3165 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3166 IID = Intrinsic::x86_avx2_permd;
3167 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3168 IID = Intrinsic::x86_avx512_permvar_df_256;
3169 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3170 IID = Intrinsic::x86_avx512_permvar_di_256;
3171 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3172 IID = Intrinsic::x86_avx512_permvar_sf_512;
3173 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3174 IID = Intrinsic::x86_avx512_permvar_si_512;
3175 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3176 IID = Intrinsic::x86_avx512_permvar_df_512;
3177 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3178 IID = Intrinsic::x86_avx512_permvar_di_512;
3179 else if (VecWidth == 128 && EltWidth == 16)
3180 IID = Intrinsic::x86_avx512_permvar_hi_128;
3181 else if (VecWidth == 256 && EltWidth == 16)
3182 IID = Intrinsic::x86_avx512_permvar_hi_256;
3183 else if (VecWidth == 512 && EltWidth == 16)
3184 IID = Intrinsic::x86_avx512_permvar_hi_512;
3185 else if (VecWidth == 128 && EltWidth == 8)
3186 IID = Intrinsic::x86_avx512_permvar_qi_128;
3187 else if (VecWidth == 256 && EltWidth == 8)
3188 IID = Intrinsic::x86_avx512_permvar_qi_256;
3189 else if (VecWidth == 512 && EltWidth == 8)
3190 IID = Intrinsic::x86_avx512_permvar_qi_512;
3191 else
3192 llvm_unreachable("Unexpected intrinsic");
3193 } else if (Name.starts_with(Prefix: "dbpsadbw.")) {
3194 if (VecWidth == 128)
3195 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3196 else if (VecWidth == 256)
3197 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3198 else if (VecWidth == 512)
3199 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3200 else
3201 llvm_unreachable("Unexpected intrinsic");
3202 } else if (Name.starts_with(Prefix: "pmultishift.qb.")) {
3203 if (VecWidth == 128)
3204 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3205 else if (VecWidth == 256)
3206 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3207 else if (VecWidth == 512)
3208 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3209 else
3210 llvm_unreachable("Unexpected intrinsic");
3211 } else if (Name.starts_with(Prefix: "conflict.")) {
3212 if (Name[9] == 'd' && VecWidth == 128)
3213 IID = Intrinsic::x86_avx512_conflict_d_128;
3214 else if (Name[9] == 'd' && VecWidth == 256)
3215 IID = Intrinsic::x86_avx512_conflict_d_256;
3216 else if (Name[9] == 'd' && VecWidth == 512)
3217 IID = Intrinsic::x86_avx512_conflict_d_512;
3218 else if (Name[9] == 'q' && VecWidth == 128)
3219 IID = Intrinsic::x86_avx512_conflict_q_128;
3220 else if (Name[9] == 'q' && VecWidth == 256)
3221 IID = Intrinsic::x86_avx512_conflict_q_256;
3222 else if (Name[9] == 'q' && VecWidth == 512)
3223 IID = Intrinsic::x86_avx512_conflict_q_512;
3224 else
3225 llvm_unreachable("Unexpected intrinsic");
3226 } else if (Name.starts_with(Prefix: "pavg.")) {
3227 if (Name[5] == 'b' && VecWidth == 128)
3228 IID = Intrinsic::x86_sse2_pavg_b;
3229 else if (Name[5] == 'b' && VecWidth == 256)
3230 IID = Intrinsic::x86_avx2_pavg_b;
3231 else if (Name[5] == 'b' && VecWidth == 512)
3232 IID = Intrinsic::x86_avx512_pavg_b_512;
3233 else if (Name[5] == 'w' && VecWidth == 128)
3234 IID = Intrinsic::x86_sse2_pavg_w;
3235 else if (Name[5] == 'w' && VecWidth == 256)
3236 IID = Intrinsic::x86_avx2_pavg_w;
3237 else if (Name[5] == 'w' && VecWidth == 512)
3238 IID = Intrinsic::x86_avx512_pavg_w_512;
3239 else
3240 llvm_unreachable("Unexpected intrinsic");
3241 } else
3242 return false;
3243
3244 SmallVector<Value *, 4> Args(CI.args());
3245 Args.pop_back();
3246 Args.pop_back();
3247 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3248 unsigned NumArgs = CI.arg_size();
3249 Rep = emitX86Select(Builder, Mask: CI.getArgOperand(i: NumArgs - 1), Op0: Rep,
3250 Op1: CI.getArgOperand(i: NumArgs - 2));
3251 return true;
3252}
3253
3254/// Upgrade comment in call to inline asm that represents an objc retain release
3255/// marker.
3256void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3257 size_t Pos;
3258 if (AsmStr->find(s: "mov\tfp") == 0 &&
3259 AsmStr->find(s: "objc_retainAutoreleaseReturnValue") != std::string::npos &&
3260 (Pos = AsmStr->find(s: "# marker")) != std::string::npos) {
3261 AsmStr->replace(pos: Pos, n1: 1, s: ";");
3262 }
3263}
3264
3265static Value *upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI,
3266 StringRef Name,
3267 const Intrinsic::ID (&IIDs)[2][2]) {
3268 auto Result = getNVVMFPArithUpgrade(Name, IIDs);
3269 assert(Result && "unsupported nvvm.add.*/nvvm.mul.* intrinsic");
3270 auto [IID, RoundingMode] = *Result;
3271 Value *A = CI->getArgOperand(i: 0);
3272 return Builder.CreateIntrinsic(
3273 RetTy: A->getType(), ID: IID,
3274 Args: {A, CI->getArgOperand(i: 1),
3275 Builder.getInt32(C: static_cast<int>(RoundingMode))});
3276}
3277
3278static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3279 Function *F, IRBuilder<> &Builder) {
3280 Value *Rep = nullptr;
3281
3282 if (Name == "abs.i" || Name == "abs.ll") {
3283 Value *Arg = CI->getArgOperand(i: 0);
3284 Rep = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: {Arg->getType()},
3285 Args: {Arg, Builder.getTrue()},
3286 /*FMFSource=*/nullptr, Name: "abs");
3287 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3288 Type *Ty = (Name == "abs.bf16")
3289 ? Builder.getBFloatTy()
3290 : FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: 2);
3291 Value *Arg = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Ty);
3292 Value *Abs = Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs, Op: Arg);
3293 Rep = Builder.CreateBitCast(V: Abs, DestTy: CI->getType());
3294 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3295 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3296 : Intrinsic::nvvm_fabs;
3297 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3298 } else if (Name.consume_front(Prefix: "add.")) {
3299 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3300 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, IIDs: NVVMFAddIIDs);
3301 } else if (Name.consume_front(Prefix: "mul.")) {
3302 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3303 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, IIDs: NVVMFMulIIDs);
3304 } else if (Name.consume_front(Prefix: "ex2.approx.")) {
3305 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3306 Intrinsic::ID IID = Name.starts_with(Prefix: "ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3307 : Intrinsic::nvvm_ex2_approx;
3308 Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: 0));
3309 } else if (Name.starts_with(Prefix: "atomic.load.add.f32.p") ||
3310 Name.starts_with(Prefix: "atomic.load.add.f64.p")) {
3311 Value *Ptr = CI->getArgOperand(i: 0);
3312 Value *Val = CI->getArgOperand(i: 1);
3313 Rep = Builder.CreateAtomicRMW(
3314 Op: AtomicRMWInst::FAdd, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3315 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3316 cast<Instruction>(Val: Rep)->setMetadata(
3317 KindID: LLVMContext::MD_atomic_ignore_denormal_mode,
3318 Node: MDNode::get(Context&: CI->getContext(), MDs: {}));
3319 // The default scope for atomic.load.* intrinsics is device
3320 // (= gpu scope in ptx), but the default LLVM atomic scope is
3321 // "system"
3322 } else if (Name.starts_with(Prefix: "atomic.load.inc.32.p") ||
3323 Name.starts_with(Prefix: "atomic.load.dec.32.p")) {
3324 Value *Ptr = CI->getArgOperand(i: 0);
3325 Value *Val = CI->getArgOperand(i: 1);
3326 auto Op = Name.starts_with(Prefix: "atomic.load.inc") ? AtomicRMWInst::UIncWrap
3327 : AtomicRMWInst::UDecWrap;
3328 Rep = Builder.CreateAtomicRMW(
3329 Op, Ptr, Val, Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic,
3330 SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
3331 // See comment above.
3332 } else if (Name.starts_with(Prefix: "atomic.") && Name.contains(Other: ".gen.")) {
3333 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3334 StringRef Op = Name.substr(Start: StringRef("atomic.").size());
3335 Value *Ptr = CI->getArgOperand(i: 0);
3336 Value *Val = CI->getArgOperand(i: 1);
3337 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3338 SSN: Op.contains(Other: ".cta.") ? "block" : "");
3339 if (Op.starts_with(Prefix: "cas.")) {
3340 Value *New = CI->getArgOperand(i: 2);
3341 Value *Pair = Builder.CreateAtomicCmpXchg(
3342 Ptr, Cmp: Val, New, Align: MaybeAlign(), SuccessOrdering: AtomicOrdering::Monotonic,
3343 FailureOrdering: AtomicOrdering::Monotonic, SSID);
3344 Rep = Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
3345 } else {
3346 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3347 // because we were actually missing those intrinsics!
3348 AtomicRMWInst::BinOp BinOp =
3349 StringSwitch<AtomicRMWInst::BinOp>(Op)
3350 .StartsWith(S: "add.gen.f", Value: AtomicRMWInst::FAdd)
3351 .StartsWith(S: "add.gen.i", Value: AtomicRMWInst::Add)
3352 .StartsWith(S: "exch.", Value: AtomicRMWInst::Xchg)
3353 .StartsWith(S: "max.", Value: AtomicRMWInst::Max)
3354 .StartsWith(S: "min.", Value: AtomicRMWInst::Min)
3355 .StartsWith(S: "inc.", Value: AtomicRMWInst::UIncWrap)
3356 .StartsWith(S: "dec.", Value: AtomicRMWInst::UDecWrap)
3357 .StartsWith(S: "and.", Value: AtomicRMWInst::And)
3358 .StartsWith(S: "or.", Value: AtomicRMWInst::Or)
3359 .StartsWith(S: "xor.", Value: AtomicRMWInst::Xor)
3360 .Default(Value: AtomicRMWInst::BAD_BINOP);
3361 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3362 "unexpected nvvm scoped atomic intrinsic");
3363 Rep = Builder.CreateAtomicRMW(Op: BinOp, Ptr, Val, Align: MaybeAlign(),
3364 Ordering: AtomicOrdering::Monotonic, SSID);
3365 if (BinOp == AtomicRMWInst::FAdd) {
3366 cast<Instruction>(Val: Rep)->setMetadata(
3367 KindID: LLVMContext::MD_atomic_ignore_denormal_mode,
3368 Node: MDNode::get(Context&: CI->getContext(), MDs: {}));
3369 }
3370 }
3371 } else if (Name == "clz.ll") {
3372 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3373 Value *Arg = CI->getArgOperand(i: 0);
3374 Value *Ctlz = Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {Arg->getType()},
3375 Args: {Arg, Builder.getFalse()},
3376 /*FMFSource=*/nullptr, Name: "ctlz");
3377 Rep = Builder.CreateTrunc(V: Ctlz, DestTy: Builder.getInt32Ty(), Name: "ctlz.trunc");
3378 } else if (Name == "popc.ll") {
3379 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3380 // i64.
3381 Value *Arg = CI->getArgOperand(i: 0);
3382 Value *Popc = Builder.CreateIntrinsic(ID: Intrinsic::ctpop, OverloadTypes: {Arg->getType()},
3383 Args: Arg, /*FMFSource=*/nullptr, Name: "ctpop");
3384 Rep = Builder.CreateTrunc(V: Popc, DestTy: Builder.getInt32Ty(), Name: "ctpop.trunc");
3385 } else if (Name == "h2f") {
3386 Value *Cast =
3387 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
3388 Rep = Builder.CreateFPExt(V: Cast, DestTy: Builder.getFloatTy());
3389 } else if (Name.consume_front(Prefix: "bitcast.") &&
3390 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3391 Name == "d2ll")) {
3392 Rep = Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3393 } else if (Name == "rotate.b32") {
3394 Value *Arg = CI->getOperand(i_nocapture: 0);
3395 Value *ShiftAmt = CI->getOperand(i_nocapture: 1);
3396 Rep = Builder.CreateIntrinsic(RetTy: Builder.getInt32Ty(), ID: Intrinsic::fshl,
3397 Args: {Arg, Arg, ShiftAmt});
3398 } else if (Name == "rotate.b64") {
3399 Type *Int64Ty = Builder.getInt64Ty();
3400 Value *Arg = CI->getOperand(i_nocapture: 0);
3401 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3402 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3403 Args: {Arg, Arg, ZExtShiftAmt});
3404 } else if (Name == "rotate.right.b64") {
3405 Type *Int64Ty = Builder.getInt64Ty();
3406 Value *Arg = CI->getOperand(i_nocapture: 0);
3407 Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: 1), DestTy: Int64Ty);
3408 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshr,
3409 Args: {Arg, Arg, ZExtShiftAmt});
3410 } else if (Name == "swap.lo.hi.b64") {
3411 Type *Int64Ty = Builder.getInt64Ty();
3412 Value *Arg = CI->getOperand(i_nocapture: 0);
3413 Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
3414 Args: {Arg, Arg, Builder.getInt64(C: 32)});
3415 } else if ((Name.consume_front(Prefix: "ptr.gen.to.") &&
3416 consumeNVVMPtrAddrSpace(Name)) ||
3417 (Name.consume_front(Prefix: "ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3418 Name.starts_with(Prefix: ".to.gen"))) {
3419 Rep = Builder.CreateAddrSpaceCast(V: CI->getArgOperand(i: 0), DestTy: CI->getType());
3420 } else if (Name.consume_front(Prefix: "ldg.global")) {
3421 Value *Ptr = CI->getArgOperand(i: 0);
3422 Align PtrAlign = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getAlignValue();
3423 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3424 Value *ASC = Builder.CreateAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 1));
3425 Instruction *LD = Builder.CreateAlignedLoad(Ty: CI->getType(), Ptr: ASC, Align: PtrAlign);
3426 MDNode *MD = MDNode::get(Context&: Builder.getContext(), MDs: {});
3427 LD->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MD);
3428 return LD;
3429 } else if (Name == "tanh.approx.f32") {
3430 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3431 FastMathFlags FMF;
3432 FMF.setApproxFunc();
3433 Rep = Builder.CreateUnaryIntrinsic(ID: Intrinsic::tanh, Op: CI->getArgOperand(i: 0),
3434 FMFSource: FMF);
3435 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3436 Value *Arg =
3437 Name.ends_with(Suffix: '0') ? Builder.getInt32(C: 0) : CI->getArgOperand(i: 0);
3438 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3439 OverloadTypes: {}, Args: {Arg});
3440 } else if (Name == "barrier") {
3441 Rep = Builder.CreateIntrinsic(
3442 ID: Intrinsic::nvvm_barrier_cta_sync_aligned_count, OverloadTypes: {},
3443 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3444 } else if (Name == "barrier.sync") {
3445 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_all, OverloadTypes: {},
3446 Args: {CI->getArgOperand(i: 0)});
3447 } else if (Name == "barrier.sync.cnt") {
3448 Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_count, OverloadTypes: {},
3449 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
3450 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3451 Name == "barrier0.or") {
3452 Value *C = CI->getArgOperand(i: 0);
3453 C = Builder.CreateICmpNE(LHS: C, RHS: Builder.getInt32(C: 0));
3454
3455 Intrinsic::ID IID =
3456 StringSwitch<Intrinsic::ID>(Name)
3457 .Case(S: "barrier0.popc",
3458 Value: Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3459 .Case(S: "barrier0.and",
3460 Value: Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3461 .Case(S: "barrier0.or",
3462 Value: Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3463 Value *Bar = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {}, Args: {Builder.getInt32(C: 0), C});
3464 Rep = Builder.CreateZExt(V: Bar, DestTy: CI->getType());
3465 } else {
3466 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3467 if (IID != Intrinsic::not_intrinsic &&
3468 isLegacyNVPTXBF16IntSignature(F, IID)) {
3469 rename(GV: F);
3470 Function *NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
3471 SmallVector<Value *, 2> Args;
3472 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3473 Value *Arg = CI->getArgOperand(i: I);
3474 Type *OldType = Arg->getType();
3475 Type *NewType = NewFn->getArg(i: I)->getType();
3476 Args.push_back(
3477 Elt: (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3478 ? Builder.CreateBitCast(V: Arg, DestTy: NewType)
3479 : Arg);
3480 }
3481 Rep = Builder.CreateCall(Callee: NewFn, Args);
3482 if (F->getReturnType()->isIntegerTy())
3483 Rep = Builder.CreateBitCast(V: Rep, DestTy: F->getReturnType());
3484 }
3485 }
3486
3487 return Rep;
3488}
3489
3490static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3491 IRBuilder<> &Builder) {
3492 LLVMContext &C = F->getContext();
3493 Value *Rep = nullptr;
3494
3495 if (Name.starts_with(Prefix: "sse4a.movnt.")) {
3496 SmallVector<Metadata *, 1> Elts;
3497 Elts.push_back(
3498 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3499 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3500
3501 Value *Arg0 = CI->getArgOperand(i: 0);
3502 Value *Arg1 = CI->getArgOperand(i: 1);
3503
3504 // Nontemporal (unaligned) store of the 0'th element of the float/double
3505 // vector.
3506 Value *Extract =
3507 Builder.CreateExtractElement(Vec: Arg1, Idx: (uint64_t)0, Name: "extractelement");
3508
3509 StoreInst *SI = Builder.CreateAlignedStore(Val: Extract, Ptr: Arg0, Align: Align(1));
3510 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3511 } else if (Name.starts_with(Prefix: "avx.movnt.") ||
3512 Name.starts_with(Prefix: "avx512.storent.")) {
3513 SmallVector<Metadata *, 1> Elts;
3514 Elts.push_back(
3515 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
3516 MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3517
3518 Value *Arg0 = CI->getArgOperand(i: 0);
3519 Value *Arg1 = CI->getArgOperand(i: 1);
3520
3521 StoreInst *SI = Builder.CreateAlignedStore(
3522 Val: Arg1, Ptr: Arg0,
3523 Align: Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3524 SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3525 } else if (Name == "sse2.storel.dq") {
3526 Value *Arg0 = CI->getArgOperand(i: 0);
3527 Value *Arg1 = CI->getArgOperand(i: 1);
3528
3529 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
3530 Value *BC0 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
3531 Value *Elt = Builder.CreateExtractElement(Vec: BC0, Idx: (uint64_t)0);
3532 Builder.CreateAlignedStore(Val: Elt, Ptr: Arg0, Align: Align(1));
3533 } else if (Name.starts_with(Prefix: "sse.storeu.") ||
3534 Name.starts_with(Prefix: "sse2.storeu.") ||
3535 Name.starts_with(Prefix: "avx.storeu.")) {
3536 Value *Arg0 = CI->getArgOperand(i: 0);
3537 Value *Arg1 = CI->getArgOperand(i: 1);
3538 Builder.CreateAlignedStore(Val: Arg1, Ptr: Arg0, Align: Align(1));
3539 } else if (Name == "avx512.mask.store.ss") {
3540 Value *Mask = Builder.CreateAnd(LHS: CI->getArgOperand(i: 2), RHS: Builder.getInt8(C: 1));
3541 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3542 Mask, Aligned: false);
3543 } else if (Name.starts_with(Prefix: "avx512.mask.store")) {
3544 // "avx512.mask.storeu." or "avx512.mask.store."
3545 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3546 upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: 0), Data: CI->getArgOperand(i: 1),
3547 Mask: CI->getArgOperand(i: 2), Aligned);
3548 } else if (Name.starts_with(Prefix: "sse2.pcmp") || Name.starts_with(Prefix: "avx2.pcmp")) {
3549 // Upgrade packed integer vector compare intrinsics to compare instructions.
3550 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3551 bool CmpEq = Name[9] == 'e';
3552 Rep = Builder.CreateICmp(P: CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3553 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
3554 Rep = Builder.CreateSExt(V: Rep, DestTy: CI->getType(), Name: "");
3555 } else if (Name.starts_with(Prefix: "avx512.broadcastm")) {
3556 Type *ExtTy = Type::getInt32Ty(C);
3557 if (CI->getOperand(i_nocapture: 0)->getType()->isIntegerTy(BitWidth: 8))
3558 ExtTy = Type::getInt64Ty(C);
3559 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3560 ExtTy->getPrimitiveSizeInBits();
3561 Rep = Builder.CreateZExt(V: CI->getArgOperand(i: 0), DestTy: ExtTy);
3562 Rep = Builder.CreateVectorSplat(NumElts, V: Rep);
3563 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3564 Value *Vec = CI->getArgOperand(i: 0);
3565 Value *Elt0 = Builder.CreateExtractElement(Vec, Idx: (uint64_t)0);
3566 Elt0 = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: Elt0->getType(), Args: Elt0);
3567 Rep = Builder.CreateInsertElement(Vec, NewElt: Elt0, Idx: (uint64_t)0);
3568 } else if (Name.starts_with(Prefix: "avx.sqrt.p") ||
3569 Name.starts_with(Prefix: "sse2.sqrt.p") ||
3570 Name.starts_with(Prefix: "sse.sqrt.p")) {
3571 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3572 Args: {CI->getArgOperand(i: 0)});
3573 } else if (Name.starts_with(Prefix: "avx512.mask.sqrt.p")) {
3574 if (CI->arg_size() == 4 &&
3575 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3576 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3577 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3578 : Intrinsic::x86_avx512_sqrt_pd_512;
3579
3580 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 3)};
3581 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3582 } else {
3583 Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3584 Args: {CI->getArgOperand(i: 0)});
3585 }
3586 Rep =
3587 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3588 } else if (Name.starts_with(Prefix: "avx512.ptestm") ||
3589 Name.starts_with(Prefix: "avx512.ptestnm")) {
3590 Value *Op0 = CI->getArgOperand(i: 0);
3591 Value *Op1 = CI->getArgOperand(i: 1);
3592 Value *Mask = CI->getArgOperand(i: 2);
3593 Rep = Builder.CreateAnd(LHS: Op0, RHS: Op1);
3594 llvm::Type *Ty = Op0->getType();
3595 Value *Zero = llvm::Constant::getNullValue(Ty);
3596 ICmpInst::Predicate Pred = Name.starts_with(Prefix: "avx512.ptestm")
3597 ? ICmpInst::ICMP_NE
3598 : ICmpInst::ICMP_EQ;
3599 Rep = Builder.CreateICmp(P: Pred, LHS: Rep, RHS: Zero);
3600 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask);
3601 } else if (Name.starts_with(Prefix: "avx512.mask.pbroadcast")) {
3602 unsigned NumElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 1)->getType())
3603 ->getNumElements();
3604 Rep = Builder.CreateVectorSplat(NumElts, V: CI->getArgOperand(i: 0));
3605 Rep =
3606 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
3607 } else if (Name.starts_with(Prefix: "avx512.kunpck")) {
3608 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3609 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts);
3610 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts);
3611 int Indices[64];
3612 for (unsigned i = 0; i != NumElts; ++i)
3613 Indices[i] = i;
3614
3615 // First extract half of each vector. This gives better codegen than
3616 // doing it in a single shuffle.
3617 LHS = Builder.CreateShuffleVector(V1: LHS, V2: LHS, Mask: ArrayRef(Indices, NumElts / 2));
3618 RHS = Builder.CreateShuffleVector(V1: RHS, V2: RHS, Mask: ArrayRef(Indices, NumElts / 2));
3619 // Concat the vectors.
3620 // NOTE: Operands have to be swapped to match intrinsic definition.
3621 Rep = Builder.CreateShuffleVector(V1: RHS, V2: LHS, Mask: ArrayRef(Indices, NumElts));
3622 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3623 } else if (Name == "avx512.kand.w") {
3624 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3625 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3626 Rep = Builder.CreateAnd(LHS, RHS);
3627 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3628 } else if (Name == "avx512.kandn.w") {
3629 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3630 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3631 LHS = Builder.CreateNot(V: LHS);
3632 Rep = Builder.CreateAnd(LHS, RHS);
3633 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3634 } else if (Name == "avx512.kor.w") {
3635 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3636 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3637 Rep = Builder.CreateOr(LHS, RHS);
3638 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3639 } else if (Name == "avx512.kxor.w") {
3640 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3641 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3642 Rep = Builder.CreateXor(LHS, RHS);
3643 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3644 } else if (Name == "avx512.kxnor.w") {
3645 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3646 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3647 LHS = Builder.CreateNot(V: LHS);
3648 Rep = Builder.CreateXor(LHS, RHS);
3649 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3650 } else if (Name == "avx512.knot.w") {
3651 Rep = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3652 Rep = Builder.CreateNot(V: Rep);
3653 Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3654 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3655 Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 0), NumElts: 16);
3656 Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 1), NumElts: 16);
3657 Rep = Builder.CreateOr(LHS, RHS);
3658 Rep = Builder.CreateBitCast(V: Rep, DestTy: Builder.getInt16Ty());
3659 Value *C;
3660 if (Name[14] == 'c')
3661 C = ConstantInt::getAllOnesValue(Ty: Builder.getInt16Ty());
3662 else
3663 C = ConstantInt::getNullValue(Ty: Builder.getInt16Ty());
3664 Rep = Builder.CreateICmpEQ(LHS: Rep, RHS: C);
3665 Rep = Builder.CreateZExt(V: Rep, DestTy: Builder.getInt32Ty());
3666 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3667 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3668 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3669 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3670 Type *I32Ty = Type::getInt32Ty(C);
3671 Value *Elt0 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 0),
3672 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3673 Value *Elt1 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1),
3674 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3675 Value *EltOp;
3676 if (Name.contains(Other: ".add."))
3677 EltOp = Builder.CreateFAdd(L: Elt0, R: Elt1);
3678 else if (Name.contains(Other: ".sub."))
3679 EltOp = Builder.CreateFSub(L: Elt0, R: Elt1);
3680 else if (Name.contains(Other: ".mul."))
3681 EltOp = Builder.CreateFMul(L: Elt0, R: Elt1);
3682 else
3683 EltOp = Builder.CreateFDiv(L: Elt0, R: Elt1);
3684 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: EltOp,
3685 Idx: ConstantInt::get(Ty: I32Ty, V: 0));
3686 } else if (Name.starts_with(Prefix: "avx512.mask.pcmp")) {
3687 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3688 bool CmpEq = Name[16] == 'e';
3689 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: CmpEq ? 0 : 6, Signed: true);
3690 } else if (Name.starts_with(Prefix: "avx512.mask.vpshufbitqmb.")) {
3691 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3692 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3693 Intrinsic::ID IID;
3694 switch (VecWidth) {
3695 default:
3696 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3697 break;
3698 case 128:
3699 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3700 break;
3701 case 256:
3702 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3703 break;
3704 case 512:
3705 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3706 break;
3707 }
3708
3709 Rep =
3710 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3711 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3712 } else if (Name.starts_with(Prefix: "avx512.mask.fpclass.p")) {
3713 Type *OpTy = CI->getArgOperand(i: 0)->getType();
3714 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3715 unsigned EltWidth = OpTy->getScalarSizeInBits();
3716 Intrinsic::ID IID;
3717 if (VecWidth == 128 && EltWidth == 32)
3718 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3719 else if (VecWidth == 256 && EltWidth == 32)
3720 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3721 else if (VecWidth == 512 && EltWidth == 32)
3722 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3723 else if (VecWidth == 128 && EltWidth == 64)
3724 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3725 else if (VecWidth == 256 && EltWidth == 64)
3726 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3727 else if (VecWidth == 512 && EltWidth == 64)
3728 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3729 else
3730 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3731
3732 Rep =
3733 Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: 0), CI->getArgOperand(i: 1)});
3734 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: 2));
3735 } else if (Name.starts_with(Prefix: "avx512.cmp.p")) {
3736 SmallVector<Value *, 4> Args(CI->args());
3737 Type *OpTy = Args[0]->getType();
3738 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3739 unsigned EltWidth = OpTy->getScalarSizeInBits();
3740 Intrinsic::ID IID;
3741 if (VecWidth == 128 && EltWidth == 32)
3742 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3743 else if (VecWidth == 256 && EltWidth == 32)
3744 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3745 else if (VecWidth == 512 && EltWidth == 32)
3746 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3747 else if (VecWidth == 128 && EltWidth == 64)
3748 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3749 else if (VecWidth == 256 && EltWidth == 64)
3750 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3751 else if (VecWidth == 512 && EltWidth == 64)
3752 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3753 else
3754 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3755
3756 Value *Mask = Constant::getAllOnesValue(Ty: CI->getType());
3757 if (VecWidth == 512)
3758 std::swap(a&: Mask, b&: Args.back());
3759 Args.push_back(Elt: Mask);
3760
3761 Rep = Builder.CreateIntrinsic(ID: IID, Args);
3762 } else if (Name.starts_with(Prefix: "avx512.mask.cmp.")) {
3763 // Integer compare intrinsics.
3764 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3765 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: true);
3766 } else if (Name.starts_with(Prefix: "avx512.mask.ucmp.")) {
3767 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3768 Rep = upgradeMaskedCompare(Builder, CI&: *CI, CC: Imm, Signed: false);
3769 } else if (Name.starts_with(Prefix: "avx512.cvtb2mask.") ||
3770 Name.starts_with(Prefix: "avx512.cvtw2mask.") ||
3771 Name.starts_with(Prefix: "avx512.cvtd2mask.") ||
3772 Name.starts_with(Prefix: "avx512.cvtq2mask.")) {
3773 Value *Op = CI->getArgOperand(i: 0);
3774 Value *Zero = llvm::Constant::getNullValue(Ty: Op->getType());
3775 Rep = Builder.CreateICmp(P: ICmpInst::ICMP_SLT, LHS: Op, RHS: Zero);
3776 Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: nullptr);
3777 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3778 Name == "ssse3.pabs.d.128" || Name.starts_with(Prefix: "avx2.pabs") ||
3779 Name.starts_with(Prefix: "avx512.mask.pabs")) {
3780 Rep = upgradeAbs(Builder, CI&: *CI);
3781 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3782 Name == "sse41.pmaxsd" || Name.starts_with(Prefix: "avx2.pmaxs") ||
3783 Name.starts_with(Prefix: "avx512.mask.pmaxs")) {
3784 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smax);
3785 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3786 Name == "sse41.pmaxud" || Name.starts_with(Prefix: "avx2.pmaxu") ||
3787 Name.starts_with(Prefix: "avx512.mask.pmaxu")) {
3788 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umax);
3789 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3790 Name == "sse41.pminsd" || Name.starts_with(Prefix: "avx2.pmins") ||
3791 Name.starts_with(Prefix: "avx512.mask.pmins")) {
3792 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smin);
3793 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3794 Name == "sse41.pminud" || Name.starts_with(Prefix: "avx2.pminu") ||
3795 Name.starts_with(Prefix: "avx512.mask.pminu")) {
3796 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umin);
3797 } else if (Name == "sse2.pmulh.w" || Name.starts_with(Prefix: "avx2.pmulh.w") ||
3798 Name.starts_with(Prefix: "avx512.pmulh.w")) {
3799 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smulh);
3800 } else if (Name == "sse2.pmulhu.w" || Name.starts_with(Prefix: "avx2.pmulhu.w") ||
3801 Name.starts_with(Prefix: "avx512.pmulhu.w")) {
3802 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umulh);
3803 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3804 Name == "avx512.pmulu.dq.512" ||
3805 Name.starts_with(Prefix: "avx512.mask.pmulu.dq.")) {
3806 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: false);
3807 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3808 Name == "avx512.pmul.dq.512" ||
3809 Name.starts_with(Prefix: "avx512.mask.pmul.dq.")) {
3810 Rep = upgradePMULDQ(Builder, CI&: *CI, /*Signed*/ IsSigned: true);
3811 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3812 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3813 Rep =
3814 Builder.CreateSIToFP(V: CI->getArgOperand(i: 1),
3815 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3816 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3817 } else if (Name == "avx512.cvtusi2sd") {
3818 Rep =
3819 Builder.CreateUIToFP(V: CI->getArgOperand(i: 1),
3820 DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3821 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3822 } else if (Name == "sse2.cvtss2sd") {
3823 Rep = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 1), Idx: (uint64_t)0);
3824 Rep = Builder.CreateFPExt(
3825 V: Rep, DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3826 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
3827 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3828 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3829 Name.starts_with(Prefix: "avx512.mask.cvtdq2pd.") ||
3830 Name.starts_with(Prefix: "avx512.mask.cvtudq2pd.") ||
3831 Name.starts_with(Prefix: "avx512.mask.cvtdq2ps.") ||
3832 Name.starts_with(Prefix: "avx512.mask.cvtudq2ps.") ||
3833 Name.starts_with(Prefix: "avx512.mask.cvtqq2pd.") ||
3834 Name.starts_with(Prefix: "avx512.mask.cvtuqq2pd.") ||
3835 Name == "avx512.mask.cvtqq2ps.256" ||
3836 Name == "avx512.mask.cvtqq2ps.512" ||
3837 Name == "avx512.mask.cvtuqq2ps.256" ||
3838 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3839 Name == "avx.cvt.ps2.pd.256" ||
3840 Name == "avx512.mask.cvtps2pd.128" ||
3841 Name == "avx512.mask.cvtps2pd.256") {
3842 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3843 Rep = CI->getArgOperand(i: 0);
3844 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3845
3846 unsigned NumDstElts = DstTy->getNumElements();
3847 if (NumDstElts < SrcTy->getNumElements()) {
3848 assert(NumDstElts == 2 && "Unexpected vector size");
3849 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1});
3850 }
3851
3852 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3853 bool IsUnsigned = Name.contains(Other: "cvtu");
3854 if (IsPS2PD)
3855 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtps2pd");
3856 else if (CI->arg_size() == 4 &&
3857 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 3)) ||
3858 cast<ConstantInt>(Val: CI->getArgOperand(i: 3))->getZExtValue() != 4)) {
3859 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3860 : Intrinsic::x86_avx512_sitofp_round;
3861 Rep = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {DstTy, SrcTy},
3862 Args: {Rep, CI->getArgOperand(i: 3)});
3863 } else {
3864 Rep = IsUnsigned ? Builder.CreateUIToFP(V: Rep, DestTy: DstTy, Name: "cvt")
3865 : Builder.CreateSIToFP(V: Rep, DestTy: DstTy, Name: "cvt");
3866 }
3867
3868 if (CI->arg_size() >= 3)
3869 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3870 Op1: CI->getArgOperand(i: 1));
3871 } else if (Name.starts_with(Prefix: "avx512.mask.vcvtph2ps.") ||
3872 Name.starts_with(Prefix: "vcvtph2ps.")) {
3873 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3874 Rep = CI->getArgOperand(i: 0);
3875 auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3876 unsigned NumDstElts = DstTy->getNumElements();
3877 if (NumDstElts != SrcTy->getNumElements()) {
3878 assert(NumDstElts == 4 && "Unexpected vector size");
3879 Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{0, 1, 2, 3});
3880 }
3881 Rep = Builder.CreateBitCast(
3882 V: Rep, DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C), NumElts: NumDstElts));
3883 Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtph2ps");
3884 if (CI->arg_size() >= 3)
3885 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
3886 Op1: CI->getArgOperand(i: 1));
3887 } else if (Name.starts_with(Prefix: "avx512.mask.load")) {
3888 // "avx512.mask.loadu." or "avx512.mask.load."
3889 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3890 Rep = upgradeMaskedLoad(Builder, Ptr: CI->getArgOperand(i: 0), Passthru: CI->getArgOperand(i: 1),
3891 Mask: CI->getArgOperand(i: 2), Aligned);
3892 } else if (Name.starts_with(Prefix: "avx512.mask.expand.load.")) {
3893 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3894 auto *PtrTy = CI->getOperand(i_nocapture: 0)->getType();
3895 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3896 NumElts: ResultTy->getNumElements());
3897 Rep = Builder.CreateIntrinsic(
3898 ID: Intrinsic::masked_expandload, OverloadTypes: {ResultTy, PtrTy},
3899 Args: {CI->getOperand(i_nocapture: 0), MaskVec, CI->getOperand(i_nocapture: 1)});
3900 } else if (Name.starts_with(Prefix: "avx512.mask.compress.store.")) {
3901 auto *ResultTy = cast<VectorType>(Val: CI->getArgOperand(i: 1)->getType());
3902 auto *PtrTy = CI->getArgOperand(i: 0)->getType();
3903 Value *MaskVec =
3904 getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3905 NumElts: cast<FixedVectorType>(Val: ResultTy)->getNumElements());
3906 Rep = Builder.CreateIntrinsic(
3907 ID: Intrinsic::masked_compressstore, OverloadTypes: {ResultTy, PtrTy},
3908 Args: {CI->getArgOperand(i: 1), CI->getArgOperand(i: 0), MaskVec});
3909 } else if (Name.starts_with(Prefix: "avx512.mask.compress.") ||
3910 Name.starts_with(Prefix: "avx512.mask.expand.")) {
3911 auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3912
3913 Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: 2),
3914 NumElts: ResultTy->getNumElements());
3915
3916 bool IsCompress = Name[12] == 'c';
3917 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3918 : Intrinsic::x86_avx512_mask_expand;
3919 Rep = Builder.CreateIntrinsic(
3920 ID: IID, OverloadTypes: ResultTy, Args: {CI->getOperand(i_nocapture: 0), CI->getOperand(i_nocapture: 1), MaskVec});
3921 } else if (Name.starts_with(Prefix: "xop.vpcom")) {
3922 bool IsSigned;
3923 if (Name.ends_with(Suffix: "ub") || Name.ends_with(Suffix: "uw") || Name.ends_with(Suffix: "ud") ||
3924 Name.ends_with(Suffix: "uq"))
3925 IsSigned = false;
3926 else if (Name.ends_with(Suffix: "b") || Name.ends_with(Suffix: "w") ||
3927 Name.ends_with(Suffix: "d") || Name.ends_with(Suffix: "q"))
3928 IsSigned = true;
3929 else
3930 reportFatalUsageErrorWithCI(reason: "Intrinsic has unknown suffix", CI);
3931
3932 unsigned Imm;
3933 if (CI->arg_size() == 3) {
3934 Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
3935 } else {
3936 Name = Name.substr(Start: 9); // strip off "xop.vpcom"
3937 if (Name.starts_with(Prefix: "lt"))
3938 Imm = 0;
3939 else if (Name.starts_with(Prefix: "le"))
3940 Imm = 1;
3941 else if (Name.starts_with(Prefix: "gt"))
3942 Imm = 2;
3943 else if (Name.starts_with(Prefix: "ge"))
3944 Imm = 3;
3945 else if (Name.starts_with(Prefix: "eq"))
3946 Imm = 4;
3947 else if (Name.starts_with(Prefix: "ne"))
3948 Imm = 5;
3949 else if (Name.starts_with(Prefix: "false"))
3950 Imm = 6;
3951 else if (Name.starts_with(Prefix: "true"))
3952 Imm = 7;
3953 else
3954 llvm_unreachable("Unknown condition");
3955 }
3956
3957 Rep = upgradeX86vpcom(Builder, CI&: *CI, Imm, IsSigned);
3958 } else if (Name.starts_with(Prefix: "xop.vpcmov")) {
3959 Value *Sel = CI->getArgOperand(i: 2);
3960 Value *NotSel = Builder.CreateNot(V: Sel);
3961 Value *Sel0 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 0), RHS: Sel);
3962 Value *Sel1 = Builder.CreateAnd(LHS: CI->getArgOperand(i: 1), RHS: NotSel);
3963 Rep = Builder.CreateOr(LHS: Sel0, RHS: Sel1);
3964 } else if (Name.starts_with(Prefix: "xop.vprot") || Name.starts_with(Prefix: "avx512.prol") ||
3965 Name.starts_with(Prefix: "avx512.mask.prol")) {
3966 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: false);
3967 } else if (Name.starts_with(Prefix: "avx512.pror") ||
3968 Name.starts_with(Prefix: "avx512.mask.pror")) {
3969 Rep = upgradeX86Rotate(Builder, CI&: *CI, IsRotateRight: true);
3970 } else if (Name.starts_with(Prefix: "avx512.vpshld.") ||
3971 Name.starts_with(Prefix: "avx512.mask.vpshld") ||
3972 Name.starts_with(Prefix: "avx512.maskz.vpshld")) {
3973 bool ZeroMask = Name[11] == 'z';
3974 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: false, ZeroMask);
3975 } else if (Name.starts_with(Prefix: "avx512.vpshrd.") ||
3976 Name.starts_with(Prefix: "avx512.mask.vpshrd") ||
3977 Name.starts_with(Prefix: "avx512.maskz.vpshrd")) {
3978 bool ZeroMask = Name[11] == 'z';
3979 Rep = upgradeX86ConcatShift(Builder, CI&: *CI, IsShiftRight: true, ZeroMask);
3980 } else if (Name == "sse42.crc32.64.8") {
3981 Value *Trunc0 =
3982 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Type::getInt32Ty(C));
3983 Rep = Builder.CreateIntrinsic(ID: Intrinsic::x86_sse42_crc32_32_8,
3984 Args: {Trunc0, CI->getArgOperand(i: 1)});
3985 Rep = Builder.CreateZExt(V: Rep, DestTy: CI->getType(), Name: "");
3986 } else if (Name.starts_with(Prefix: "avx.vbroadcast.s") ||
3987 Name.starts_with(Prefix: "avx512.vbroadcast.s")) {
3988 // Replace broadcasts with a series of insertelements.
3989 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3990 Type *EltTy = VecTy->getElementType();
3991 unsigned EltNum = VecTy->getNumElements();
3992 Value *Load = Builder.CreateLoad(Ty: EltTy, Ptr: CI->getArgOperand(i: 0));
3993 Type *I32Ty = Type::getInt32Ty(C);
3994 Rep = PoisonValue::get(T: VecTy);
3995 for (unsigned I = 0; I < EltNum; ++I)
3996 Rep = Builder.CreateInsertElement(Vec: Rep, NewElt: Load, Idx: ConstantInt::get(Ty: I32Ty, V: I));
3997 } else if (Name.starts_with(Prefix: "sse41.pmovsx") ||
3998 Name.starts_with(Prefix: "sse41.pmovzx") ||
3999 Name.starts_with(Prefix: "avx2.pmovsx") ||
4000 Name.starts_with(Prefix: "avx2.pmovzx") ||
4001 Name.starts_with(Prefix: "avx512.mask.pmovsx") ||
4002 Name.starts_with(Prefix: "avx512.mask.pmovzx")) {
4003 auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
4004 unsigned NumDstElts = DstTy->getNumElements();
4005
4006 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
4007 SmallVector<int, 8> ShuffleMask(NumDstElts);
4008 for (unsigned i = 0; i != NumDstElts; ++i)
4009 ShuffleMask[i] = i;
4010
4011 Value *SV = Builder.CreateShuffleVector(V: CI->getArgOperand(i: 0), Mask: ShuffleMask);
4012
4013 bool DoSext = Name.contains(Other: "pmovsx");
4014 Rep =
4015 DoSext ? Builder.CreateSExt(V: SV, DestTy: DstTy) : Builder.CreateZExt(V: SV, DestTy: DstTy);
4016 // If there are 3 arguments, it's a masked intrinsic so we need a select.
4017 if (CI->arg_size() == 3)
4018 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
4019 Op1: CI->getArgOperand(i: 1));
4020 } else if (Name == "avx512.mask.pmov.qd.256" ||
4021 Name == "avx512.mask.pmov.qd.512" ||
4022 Name == "avx512.mask.pmov.wb.256" ||
4023 Name == "avx512.mask.pmov.wb.512") {
4024 Type *Ty = CI->getArgOperand(i: 1)->getType();
4025 Rep = Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Ty);
4026 Rep =
4027 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4028 } else if (Name.starts_with(Prefix: "avx.vbroadcastf128") ||
4029 Name == "avx2.vbroadcasti128") {
4030 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4031 Type *EltTy = cast<VectorType>(Val: CI->getType())->getElementType();
4032 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4033 auto *VT = FixedVectorType::get(ElementType: EltTy, NumElts: NumSrcElts);
4034 Value *Load = Builder.CreateAlignedLoad(Ty: VT, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
4035 if (NumSrcElts == 2)
4036 Rep = Builder.CreateShuffleVector(V: Load, Mask: ArrayRef<int>{0, 1, 0, 1});
4037 else
4038 Rep = Builder.CreateShuffleVector(V: Load,
4039 Mask: ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4040 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.i") ||
4041 Name.starts_with(Prefix: "avx512.mask.shuf.f")) {
4042 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4043 Type *VT = CI->getType();
4044 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4045 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4046 unsigned ControlBitsMask = NumLanes - 1;
4047 unsigned NumControlBits = NumLanes / 2;
4048 SmallVector<int, 8> ShuffleMask(0);
4049
4050 for (unsigned l = 0; l != NumLanes; ++l) {
4051 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4052 // We actually need the other source.
4053 if (l >= NumLanes / 2)
4054 LaneMask += NumLanes;
4055 for (unsigned i = 0; i != NumElementsInLane; ++i)
4056 ShuffleMask.push_back(Elt: LaneMask * NumElementsInLane + i);
4057 }
4058 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
4059 V2: CI->getArgOperand(i: 1), Mask: ShuffleMask);
4060 Rep =
4061 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
4062 } else if (Name.starts_with(Prefix: "avx512.mask.broadcastf") ||
4063 Name.starts_with(Prefix: "avx512.mask.broadcasti")) {
4064 unsigned NumSrcElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: 0)->getType())
4065 ->getNumElements();
4066 unsigned NumDstElts =
4067 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4068
4069 SmallVector<int, 8> ShuffleMask(NumDstElts);
4070 for (unsigned i = 0; i != NumDstElts; ++i)
4071 ShuffleMask[i] = i % NumSrcElts;
4072
4073 Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: 0),
4074 V2: CI->getArgOperand(i: 0), Mask: ShuffleMask);
4075 Rep =
4076 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4077 } else if (Name.starts_with(Prefix: "avx2.pbroadcast") ||
4078 Name.starts_with(Prefix: "avx2.vbroadcast") ||
4079 Name.starts_with(Prefix: "avx512.pbroadcast") ||
4080 Name.starts_with(Prefix: "avx512.mask.broadcast.s")) {
4081 // Replace vp?broadcasts with a vector shuffle.
4082 Value *Op = CI->getArgOperand(i: 0);
4083 ElementCount EC = cast<VectorType>(Val: CI->getType())->getElementCount();
4084 Type *MaskTy = VectorType::get(ElementType: Type::getInt32Ty(C), EC);
4085 SmallVector<int, 8> M;
4086 ShuffleVectorInst::getShuffleMask(Mask: Constant::getNullValue(Ty: MaskTy), Result&: M);
4087 Rep = Builder.CreateShuffleVector(V: Op, Mask: M);
4088
4089 if (CI->arg_size() == 3)
4090 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep,
4091 Op1: CI->getArgOperand(i: 1));
4092 } else if (Name.starts_with(Prefix: "sse2.padds.") ||
4093 Name.starts_with(Prefix: "avx2.padds.") ||
4094 Name.starts_with(Prefix: "avx512.padds.") ||
4095 Name.starts_with(Prefix: "avx512.mask.padds.")) {
4096 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::sadd_sat);
4097 } else if (Name.starts_with(Prefix: "sse2.psubs.") ||
4098 Name.starts_with(Prefix: "avx2.psubs.") ||
4099 Name.starts_with(Prefix: "avx512.psubs.") ||
4100 Name.starts_with(Prefix: "avx512.mask.psubs.")) {
4101 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::ssub_sat);
4102 } else if (Name.starts_with(Prefix: "sse2.paddus.") ||
4103 Name.starts_with(Prefix: "avx2.paddus.") ||
4104 Name.starts_with(Prefix: "avx512.mask.paddus.")) {
4105 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::uadd_sat);
4106 } else if (Name.starts_with(Prefix: "sse2.psubus.") ||
4107 Name.starts_with(Prefix: "avx2.psubus.") ||
4108 Name.starts_with(Prefix: "avx512.mask.psubus.")) {
4109 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::usub_sat);
4110 } else if (Name.starts_with(Prefix: "avx512.mask.palignr.")) {
4111 Rep = upgradeX86ALIGNIntrinsics(Builder, Op0: CI->getArgOperand(i: 0),
4112 Op1: CI->getArgOperand(i: 1), Shift: CI->getArgOperand(i: 2),
4113 Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4),
4114 IsVALIGN: false);
4115 } else if (Name.starts_with(Prefix: "avx512.mask.valign.")) {
4116 Rep = upgradeX86ALIGNIntrinsics(
4117 Builder, Op0: CI->getArgOperand(i: 0), Op1: CI->getArgOperand(i: 1),
4118 Shift: CI->getArgOperand(i: 2), Passthru: CI->getArgOperand(i: 3), Mask: CI->getArgOperand(i: 4), IsVALIGN: true);
4119 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4120 // 128/256-bit shift left specified in bits.
4121 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4122 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
4123 Shift: Shift / 8); // Shift is in bits.
4124 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4125 // 128/256-bit shift right specified in bits.
4126 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4127 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0),
4128 Shift: Shift / 8); // Shift is in bits.
4129 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4130 Name == "avx512.psll.dq.512") {
4131 // 128/256/512-bit shift left specified in bytes.
4132 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4133 Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
4134 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4135 Name == "avx512.psrl.dq.512") {
4136 // 128/256/512-bit shift right specified in bytes.
4137 unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4138 Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: 0), Shift);
4139 } else if (Name == "sse41.pblendw" || Name.starts_with(Prefix: "sse41.blendp") ||
4140 Name.starts_with(Prefix: "avx.blend.p") || Name == "avx2.pblendw" ||
4141 Name.starts_with(Prefix: "avx2.pblendd.")) {
4142 Value *Op0 = CI->getArgOperand(i: 0);
4143 Value *Op1 = CI->getArgOperand(i: 1);
4144 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4145 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4146 unsigned NumElts = VecTy->getNumElements();
4147
4148 SmallVector<int, 16> Idxs(NumElts);
4149 for (unsigned i = 0; i != NumElts; ++i)
4150 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4151
4152 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4153 } else if (Name.starts_with(Prefix: "avx.vinsertf128.") ||
4154 Name == "avx2.vinserti128" ||
4155 Name.starts_with(Prefix: "avx512.mask.insert")) {
4156 Value *Op0 = CI->getArgOperand(i: 0);
4157 Value *Op1 = CI->getArgOperand(i: 1);
4158 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4159 unsigned DstNumElts =
4160 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4161 unsigned SrcNumElts =
4162 cast<FixedVectorType>(Val: Op1->getType())->getNumElements();
4163 unsigned Scale = DstNumElts / SrcNumElts;
4164
4165 // Mask off the high bits of the immediate value; hardware ignores those.
4166 Imm = Imm % Scale;
4167
4168 // Extend the second operand into a vector the size of the destination.
4169 SmallVector<int, 8> Idxs(DstNumElts);
4170 for (unsigned i = 0; i != SrcNumElts; ++i)
4171 Idxs[i] = i;
4172 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4173 Idxs[i] = SrcNumElts;
4174 Rep = Builder.CreateShuffleVector(V: Op1, Mask: Idxs);
4175
4176 // Insert the second operand into the first operand.
4177
4178 // Note that there is no guarantee that instruction lowering will actually
4179 // produce a vinsertf128 instruction for the created shuffles. In
4180 // particular, the 0 immediate case involves no lane changes, so it can
4181 // be handled as a blend.
4182
4183 // Example of shuffle mask for 32-bit elements:
4184 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4185 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4186
4187 // First fill with identify mask.
4188 for (unsigned i = 0; i != DstNumElts; ++i)
4189 Idxs[i] = i;
4190 // Then replace the elements where we need to insert.
4191 for (unsigned i = 0; i != SrcNumElts; ++i)
4192 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4193 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Rep, Mask: Idxs);
4194
4195 // If the intrinsic has a mask operand, handle that.
4196 if (CI->arg_size() == 5)
4197 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep,
4198 Op1: CI->getArgOperand(i: 3));
4199 } else if (Name.starts_with(Prefix: "avx.vextractf128.") ||
4200 Name == "avx2.vextracti128" ||
4201 Name.starts_with(Prefix: "avx512.mask.vextract")) {
4202 Value *Op0 = CI->getArgOperand(i: 0);
4203 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4204 unsigned DstNumElts =
4205 cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4206 unsigned SrcNumElts =
4207 cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
4208 unsigned Scale = SrcNumElts / DstNumElts;
4209
4210 // Mask off the high bits of the immediate value; hardware ignores those.
4211 Imm = Imm % Scale;
4212
4213 // Get indexes for the subvector of the input vector.
4214 SmallVector<int, 8> Idxs(DstNumElts);
4215 for (unsigned i = 0; i != DstNumElts; ++i) {
4216 Idxs[i] = i + (Imm * DstNumElts);
4217 }
4218 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4219
4220 // If the intrinsic has a mask operand, handle that.
4221 if (CI->arg_size() == 4)
4222 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4223 Op1: CI->getArgOperand(i: 2));
4224 } else if (Name.starts_with(Prefix: "avx512.mask.perm.df.") ||
4225 Name.starts_with(Prefix: "avx512.mask.perm.di.")) {
4226 Value *Op0 = CI->getArgOperand(i: 0);
4227 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4228 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4229 unsigned NumElts = VecTy->getNumElements();
4230
4231 SmallVector<int, 8> Idxs(NumElts);
4232 for (unsigned i = 0; i != NumElts; ++i)
4233 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4234
4235 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4236
4237 if (CI->arg_size() == 4)
4238 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4239 Op1: CI->getArgOperand(i: 2));
4240 } else if (Name.starts_with(Prefix: "avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4241 // The immediate permute control byte looks like this:
4242 // [1:0] - select 128 bits from sources for low half of destination
4243 // [2] - ignore
4244 // [3] - zero low half of destination
4245 // [5:4] - select 128 bits from sources for high half of destination
4246 // [6] - ignore
4247 // [7] - zero high half of destination
4248
4249 uint8_t Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4250
4251 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4252 unsigned HalfSize = NumElts / 2;
4253 SmallVector<int, 8> ShuffleMask(NumElts);
4254
4255 // Determine which operand(s) are actually in use for this instruction.
4256 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4257 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(i: 1) : CI->getArgOperand(i: 0);
4258
4259 // If needed, replace operands based on zero mask.
4260 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(Ty: CI->getType()) : V0;
4261 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(Ty: CI->getType()) : V1;
4262
4263 // Permute low half of result.
4264 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4265 for (unsigned i = 0; i < HalfSize; ++i)
4266 ShuffleMask[i] = StartIndex + i;
4267
4268 // Permute high half of result.
4269 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4270 for (unsigned i = 0; i < HalfSize; ++i)
4271 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4272
4273 Rep = Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
4274
4275 } else if (Name.starts_with(Prefix: "avx.vpermil.") || Name == "sse2.pshuf.d" ||
4276 Name.starts_with(Prefix: "avx512.mask.vpermil.p") ||
4277 Name.starts_with(Prefix: "avx512.mask.pshuf.d.")) {
4278 Value *Op0 = CI->getArgOperand(i: 0);
4279 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4280 auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
4281 unsigned NumElts = VecTy->getNumElements();
4282 // Calculate the size of each index in the immediate.
4283 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4284 unsigned IdxMask = ((1 << IdxSize) - 1);
4285
4286 SmallVector<int, 8> Idxs(NumElts);
4287 // Lookup the bits for this element, wrapping around the immediate every
4288 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4289 // to offset by the first index of each group.
4290 for (unsigned i = 0; i != NumElts; ++i)
4291 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4292
4293 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4294
4295 if (CI->arg_size() == 4)
4296 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4297 Op1: CI->getArgOperand(i: 2));
4298 } else if (Name == "sse2.pshufl.w" ||
4299 Name.starts_with(Prefix: "avx512.mask.pshufl.w.")) {
4300 Value *Op0 = CI->getArgOperand(i: 0);
4301 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4302 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4303
4304 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4305 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4306
4307 SmallVector<int, 16> Idxs(NumElts);
4308 for (unsigned l = 0; l != NumElts; l += 8) {
4309 for (unsigned i = 0; i != 4; ++i)
4310 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4311 for (unsigned i = 4; i != 8; ++i)
4312 Idxs[i + l] = i + l;
4313 }
4314
4315 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4316
4317 if (CI->arg_size() == 4)
4318 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4319 Op1: CI->getArgOperand(i: 2));
4320 } else if (Name == "sse2.pshufh.w" ||
4321 Name.starts_with(Prefix: "avx512.mask.pshufh.w.")) {
4322 Value *Op0 = CI->getArgOperand(i: 0);
4323 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
4324 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4325
4326 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4327 reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
4328
4329 SmallVector<int, 16> Idxs(NumElts);
4330 for (unsigned l = 0; l != NumElts; l += 8) {
4331 for (unsigned i = 0; i != 4; ++i)
4332 Idxs[i + l] = i + l;
4333 for (unsigned i = 0; i != 4; ++i)
4334 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4335 }
4336
4337 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4338
4339 if (CI->arg_size() == 4)
4340 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep,
4341 Op1: CI->getArgOperand(i: 2));
4342 } else if (Name.starts_with(Prefix: "avx512.mask.shuf.p")) {
4343 Value *Op0 = CI->getArgOperand(i: 0);
4344 Value *Op1 = CI->getArgOperand(i: 1);
4345 unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue();
4346 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4347
4348 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4349 unsigned HalfLaneElts = NumLaneElts / 2;
4350
4351 SmallVector<int, 16> Idxs(NumElts);
4352 for (unsigned i = 0; i != NumElts; ++i) {
4353 // Base index is the starting element of the lane.
4354 Idxs[i] = i - (i % NumLaneElts);
4355 // If we are half way through the lane switch to the other source.
4356 if ((i % NumLaneElts) >= HalfLaneElts)
4357 Idxs[i] += NumElts;
4358 // Now select the specific element. By adding HalfLaneElts bits from
4359 // the immediate. Wrapping around the immediate every 8-bits.
4360 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4361 }
4362
4363 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4364
4365 Rep =
4366 emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: CI->getArgOperand(i: 3));
4367 } else if (Name.starts_with(Prefix: "avx512.mask.movddup") ||
4368 Name.starts_with(Prefix: "avx512.mask.movshdup") ||
4369 Name.starts_with(Prefix: "avx512.mask.movsldup")) {
4370 Value *Op0 = CI->getArgOperand(i: 0);
4371 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4372 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4373
4374 unsigned Offset = 0;
4375 if (Name.starts_with(Prefix: "avx512.mask.movshdup."))
4376 Offset = 1;
4377
4378 SmallVector<int, 16> Idxs(NumElts);
4379 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4380 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4381 Idxs[i + l + 0] = i + l + Offset;
4382 Idxs[i + l + 1] = i + l + Offset;
4383 }
4384
4385 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
4386
4387 Rep =
4388 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4389 } else if (Name.starts_with(Prefix: "avx512.mask.punpckl") ||
4390 Name.starts_with(Prefix: "avx512.mask.unpckl.")) {
4391 Value *Op0 = CI->getArgOperand(i: 0);
4392 Value *Op1 = CI->getArgOperand(i: 1);
4393 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4394 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4395
4396 SmallVector<int, 64> Idxs(NumElts);
4397 for (int l = 0; l != NumElts; l += NumLaneElts)
4398 for (int i = 0; i != NumLaneElts; ++i)
4399 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4400
4401 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4402
4403 Rep =
4404 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4405 } else if (Name.starts_with(Prefix: "avx512.mask.punpckh") ||
4406 Name.starts_with(Prefix: "avx512.mask.unpckh.")) {
4407 Value *Op0 = CI->getArgOperand(i: 0);
4408 Value *Op1 = CI->getArgOperand(i: 1);
4409 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4410 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4411
4412 SmallVector<int, 64> Idxs(NumElts);
4413 for (int l = 0; l != NumElts; l += NumLaneElts)
4414 for (int i = 0; i != NumLaneElts; ++i)
4415 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4416
4417 Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
4418
4419 Rep =
4420 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4421 } else if (Name.starts_with(Prefix: "avx512.mask.and.") ||
4422 Name.starts_with(Prefix: "avx512.mask.pand.")) {
4423 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4424 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4425 Rep = Builder.CreateAnd(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4426 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4427 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4428 Rep =
4429 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4430 } else if (Name.starts_with(Prefix: "avx512.mask.andn.") ||
4431 Name.starts_with(Prefix: "avx512.mask.pandn.")) {
4432 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4433 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4434 Rep = Builder.CreateNot(V: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy));
4435 Rep = Builder.CreateAnd(LHS: Rep,
4436 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4437 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4438 Rep =
4439 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4440 } else if (Name.starts_with(Prefix: "avx512.mask.or.") ||
4441 Name.starts_with(Prefix: "avx512.mask.por.")) {
4442 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4443 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4444 Rep = Builder.CreateOr(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4445 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4446 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4447 Rep =
4448 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4449 } else if (Name.starts_with(Prefix: "avx512.mask.xor.") ||
4450 Name.starts_with(Prefix: "avx512.mask.pxor.")) {
4451 VectorType *FTy = cast<VectorType>(Val: CI->getType());
4452 VectorType *ITy = VectorType::getInteger(VTy: FTy);
4453 Rep = Builder.CreateXor(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: ITy),
4454 RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: 1), DestTy: ITy));
4455 Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
4456 Rep =
4457 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4458 } else if (Name.starts_with(Prefix: "avx512.mask.padd.")) {
4459 Rep = Builder.CreateAdd(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4460 Rep =
4461 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4462 } else if (Name.starts_with(Prefix: "avx512.mask.psub.")) {
4463 Rep = Builder.CreateSub(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4464 Rep =
4465 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4466 } else if (Name.starts_with(Prefix: "avx512.mask.pmull.")) {
4467 Rep = Builder.CreateMul(LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
4468 Rep =
4469 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4470 } else if (Name.starts_with(Prefix: "avx512.mask.add.p")) {
4471 if (Name.ends_with(Suffix: ".512")) {
4472 Intrinsic::ID IID;
4473 if (Name[17] == 's')
4474 IID = Intrinsic::x86_avx512_add_ps_512;
4475 else
4476 IID = Intrinsic::x86_avx512_add_pd_512;
4477
4478 Rep = Builder.CreateIntrinsic(
4479 ID: IID,
4480 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4481 } else {
4482 Rep = Builder.CreateFAdd(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4483 }
4484 Rep =
4485 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4486 } else if (Name.starts_with(Prefix: "avx512.mask.div.p")) {
4487 if (Name.ends_with(Suffix: ".512")) {
4488 Intrinsic::ID IID;
4489 if (Name[17] == 's')
4490 IID = Intrinsic::x86_avx512_div_ps_512;
4491 else
4492 IID = Intrinsic::x86_avx512_div_pd_512;
4493
4494 Rep = Builder.CreateIntrinsic(
4495 ID: IID,
4496 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4497 } else {
4498 Rep = Builder.CreateFDiv(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4499 }
4500 Rep =
4501 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4502 } else if (Name.starts_with(Prefix: "avx512.mask.mul.p")) {
4503 if (Name.ends_with(Suffix: ".512")) {
4504 Intrinsic::ID IID;
4505 if (Name[17] == 's')
4506 IID = Intrinsic::x86_avx512_mul_ps_512;
4507 else
4508 IID = Intrinsic::x86_avx512_mul_pd_512;
4509
4510 Rep = Builder.CreateIntrinsic(
4511 ID: IID,
4512 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4513 } else {
4514 Rep = Builder.CreateFMul(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4515 }
4516 Rep =
4517 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4518 } else if (Name.starts_with(Prefix: "avx512.mask.sub.p")) {
4519 if (Name.ends_with(Suffix: ".512")) {
4520 Intrinsic::ID IID;
4521 if (Name[17] == 's')
4522 IID = Intrinsic::x86_avx512_sub_ps_512;
4523 else
4524 IID = Intrinsic::x86_avx512_sub_pd_512;
4525
4526 Rep = Builder.CreateIntrinsic(
4527 ID: IID,
4528 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4529 } else {
4530 Rep = Builder.CreateFSub(L: CI->getArgOperand(i: 0), R: CI->getArgOperand(i: 1));
4531 }
4532 Rep =
4533 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4534 } else if ((Name.starts_with(Prefix: "avx512.mask.max.p") ||
4535 Name.starts_with(Prefix: "avx512.mask.min.p")) &&
4536 Name.drop_front(N: 18) == ".512") {
4537 bool IsDouble = Name[17] == 'd';
4538 bool IsMin = Name[13] == 'i';
4539 static const Intrinsic::ID MinMaxTbl[2][2] = {
4540 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4541 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4542 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4543
4544 Rep = Builder.CreateIntrinsic(
4545 ID: IID,
4546 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 4)});
4547 Rep =
4548 emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: CI->getArgOperand(i: 2));
4549 } else if (Name.starts_with(Prefix: "avx512.mask.lzcnt.")) {
4550 Rep =
4551 Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: CI->getType(),
4552 Args: {CI->getArgOperand(i: 0), Builder.getInt1(V: false)});
4553 Rep =
4554 emitX86Select(Builder, Mask: CI->getArgOperand(i: 2), Op0: Rep, Op1: CI->getArgOperand(i: 1));
4555 } else if (Name.starts_with(Prefix: "avx512.mask.psll")) {
4556 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4557 bool IsVariable = Name[16] == 'v';
4558 char Size = Name[16] == '.' ? Name[17]
4559 : Name[17] == '.' ? Name[18]
4560 : Name[18] == '.' ? Name[19]
4561 : Name[20];
4562
4563 Intrinsic::ID IID;
4564 if (IsVariable && Name[17] != '.') {
4565 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4566 IID = Intrinsic::x86_avx2_psllv_q;
4567 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4568 IID = Intrinsic::x86_avx2_psllv_q_256;
4569 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4570 IID = Intrinsic::x86_avx2_psllv_d;
4571 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4572 IID = Intrinsic::x86_avx2_psllv_d_256;
4573 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4574 IID = Intrinsic::x86_avx512_psllv_w_128;
4575 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4576 IID = Intrinsic::x86_avx512_psllv_w_256;
4577 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4578 IID = Intrinsic::x86_avx512_psllv_w_512;
4579 else
4580 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4581 } else if (Name.ends_with(Suffix: ".128")) {
4582 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4583 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4584 : Intrinsic::x86_sse2_psll_d;
4585 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4586 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4587 : Intrinsic::x86_sse2_psll_q;
4588 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4589 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4590 : Intrinsic::x86_sse2_psll_w;
4591 else
4592 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4593 } else if (Name.ends_with(Suffix: ".256")) {
4594 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4595 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4596 : Intrinsic::x86_avx2_psll_d;
4597 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4598 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4599 : Intrinsic::x86_avx2_psll_q;
4600 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4601 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4602 : Intrinsic::x86_avx2_psll_w;
4603 else
4604 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4605 } else {
4606 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4607 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4608 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4609 : Intrinsic::x86_avx512_psll_d_512;
4610 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4611 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4612 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4613 : Intrinsic::x86_avx512_psll_q_512;
4614 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4615 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4616 : Intrinsic::x86_avx512_psll_w_512;
4617 else
4618 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4619 }
4620
4621 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4622 } else if (Name.starts_with(Prefix: "avx512.mask.psrl")) {
4623 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4624 bool IsVariable = Name[16] == 'v';
4625 char Size = Name[16] == '.' ? Name[17]
4626 : Name[17] == '.' ? Name[18]
4627 : Name[18] == '.' ? Name[19]
4628 : Name[20];
4629
4630 Intrinsic::ID IID;
4631 if (IsVariable && Name[17] != '.') {
4632 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4633 IID = Intrinsic::x86_avx2_psrlv_q;
4634 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4635 IID = Intrinsic::x86_avx2_psrlv_q_256;
4636 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4637 IID = Intrinsic::x86_avx2_psrlv_d;
4638 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4639 IID = Intrinsic::x86_avx2_psrlv_d_256;
4640 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4641 IID = Intrinsic::x86_avx512_psrlv_w_128;
4642 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4643 IID = Intrinsic::x86_avx512_psrlv_w_256;
4644 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4645 IID = Intrinsic::x86_avx512_psrlv_w_512;
4646 else
4647 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4648 } else if (Name.ends_with(Suffix: ".128")) {
4649 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4650 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4651 : Intrinsic::x86_sse2_psrl_d;
4652 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4653 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4654 : Intrinsic::x86_sse2_psrl_q;
4655 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4656 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4657 : Intrinsic::x86_sse2_psrl_w;
4658 else
4659 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4660 } else if (Name.ends_with(Suffix: ".256")) {
4661 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4662 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4663 : Intrinsic::x86_avx2_psrl_d;
4664 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4665 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4666 : Intrinsic::x86_avx2_psrl_q;
4667 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4668 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4669 : Intrinsic::x86_avx2_psrl_w;
4670 else
4671 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4672 } else {
4673 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4674 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4675 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4676 : Intrinsic::x86_avx512_psrl_d_512;
4677 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4678 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4679 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4680 : Intrinsic::x86_avx512_psrl_q_512;
4681 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4682 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4683 : Intrinsic::x86_avx512_psrl_w_512;
4684 else
4685 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4686 }
4687
4688 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4689 } else if (Name.starts_with(Prefix: "avx512.mask.psra")) {
4690 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4691 bool IsVariable = Name[16] == 'v';
4692 char Size = Name[16] == '.' ? Name[17]
4693 : Name[17] == '.' ? Name[18]
4694 : Name[18] == '.' ? Name[19]
4695 : Name[20];
4696
4697 Intrinsic::ID IID;
4698 if (IsVariable && Name[17] != '.') {
4699 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4700 IID = Intrinsic::x86_avx2_psrav_d;
4701 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4702 IID = Intrinsic::x86_avx2_psrav_d_256;
4703 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4704 IID = Intrinsic::x86_avx512_psrav_w_128;
4705 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4706 IID = Intrinsic::x86_avx512_psrav_w_256;
4707 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4708 IID = Intrinsic::x86_avx512_psrav_w_512;
4709 else
4710 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4711 } else if (Name.ends_with(Suffix: ".128")) {
4712 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4713 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4714 : Intrinsic::x86_sse2_psra_d;
4715 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4716 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4717 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4718 : Intrinsic::x86_avx512_psra_q_128;
4719 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4720 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4721 : Intrinsic::x86_sse2_psra_w;
4722 else
4723 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4724 } else if (Name.ends_with(Suffix: ".256")) {
4725 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4726 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4727 : Intrinsic::x86_avx2_psra_d;
4728 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4729 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4730 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4731 : Intrinsic::x86_avx512_psra_q_256;
4732 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4733 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4734 : Intrinsic::x86_avx2_psra_w;
4735 else
4736 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4737 } else {
4738 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4739 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4740 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4741 : Intrinsic::x86_avx512_psra_d_512;
4742 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4743 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4744 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4745 : Intrinsic::x86_avx512_psra_q_512;
4746 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4747 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4748 : Intrinsic::x86_avx512_psra_w_512;
4749 else
4750 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4751 }
4752
4753 Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4754 } else if (Name.starts_with(Prefix: "avx512.mask.move.s")) {
4755 Rep = upgradeMaskedMove(Builder, CI&: *CI);
4756 } else if (Name.starts_with(Prefix: "avx512.cvtmask2")) {
4757 Rep = upgradeMaskToInt(Builder, CI&: *CI);
4758 } else if (Name.ends_with(Suffix: ".movntdqa")) {
4759 MDNode *Node = MDNode::get(
4760 Context&: C, MDs: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)));
4761
4762 LoadInst *LI = Builder.CreateAlignedLoad(
4763 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0),
4764 Align: Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4765 LI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
4766 Rep = LI;
4767 } else if (Name.starts_with(Prefix: "fma.vfmadd.") ||
4768 Name.starts_with(Prefix: "fma.vfmsub.") ||
4769 Name.starts_with(Prefix: "fma.vfnmadd.") ||
4770 Name.starts_with(Prefix: "fma.vfnmsub.")) {
4771 bool NegMul = Name[6] == 'n';
4772 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4773 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4774
4775 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4776 CI->getArgOperand(i: 2)};
4777
4778 if (IsScalar) {
4779 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4780 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4781 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4782 }
4783
4784 if (NegMul && !IsScalar)
4785 Ops[0] = Builder.CreateFNeg(V: Ops[0]);
4786 if (NegMul && IsScalar)
4787 Ops[1] = Builder.CreateFNeg(V: Ops[1]);
4788 if (NegAcc)
4789 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4790
4791 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4792
4793 if (IsScalar)
4794 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: 0), NewElt: Rep, Idx: (uint64_t)0);
4795 } else if (Name.starts_with(Prefix: "fma4.vfmadd.s")) {
4796 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4797 CI->getArgOperand(i: 2)};
4798
4799 Ops[0] = Builder.CreateExtractElement(Vec: Ops[0], Idx: (uint64_t)0);
4800 Ops[1] = Builder.CreateExtractElement(Vec: Ops[1], Idx: (uint64_t)0);
4801 Ops[2] = Builder.CreateExtractElement(Vec: Ops[2], Idx: (uint64_t)0);
4802
4803 Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[0]->getType(), Args: Ops);
4804
4805 Rep = Builder.CreateInsertElement(Vec: Constant::getNullValue(Ty: CI->getType()),
4806 NewElt: Rep, Idx: (uint64_t)0);
4807 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.s") ||
4808 Name.starts_with(Prefix: "avx512.maskz.vfmadd.s") ||
4809 Name.starts_with(Prefix: "avx512.mask3.vfmadd.s") ||
4810 Name.starts_with(Prefix: "avx512.mask3.vfmsub.s") ||
4811 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.s")) {
4812 bool IsMask3 = Name[11] == '3';
4813 bool IsMaskZ = Name[11] == 'z';
4814 // Drop the "avx512.mask." to make it easier.
4815 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4816 bool NegMul = Name[2] == 'n';
4817 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4818
4819 Value *A = CI->getArgOperand(i: 0);
4820 Value *B = CI->getArgOperand(i: 1);
4821 Value *C = CI->getArgOperand(i: 2);
4822
4823 if (NegMul && (IsMask3 || IsMaskZ))
4824 A = Builder.CreateFNeg(V: A);
4825 if (NegMul && !(IsMask3 || IsMaskZ))
4826 B = Builder.CreateFNeg(V: B);
4827 if (NegAcc)
4828 C = Builder.CreateFNeg(V: C);
4829
4830 A = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)0);
4831 B = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)0);
4832 C = Builder.CreateExtractElement(Vec: C, Idx: (uint64_t)0);
4833
4834 if (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4835 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4) {
4836 Value *Ops[] = {A, B, C, CI->getArgOperand(i: 4)};
4837
4838 Intrinsic::ID IID;
4839 if (Name.back() == 'd')
4840 IID = Intrinsic::x86_avx512_vfmadd_f64;
4841 else
4842 IID = Intrinsic::x86_avx512_vfmadd_f32;
4843 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4844 } else {
4845 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4846 }
4847
4848 Value *PassThru = IsMaskZ ? Constant::getNullValue(Ty: Rep->getType())
4849 : IsMask3 ? C
4850 : A;
4851
4852 // For Mask3 with NegAcc, we need to create a new extractelement that
4853 // avoids the negation above.
4854 if (NegAcc && IsMask3)
4855 PassThru =
4856 Builder.CreateExtractElement(Vec: CI->getArgOperand(i: 2), Idx: (uint64_t)0);
4857
4858 Rep = emitX86ScalarSelect(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4859 Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: IsMask3 ? 2 : 0), NewElt: Rep,
4860 Idx: (uint64_t)0);
4861 } else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.p") ||
4862 Name.starts_with(Prefix: "avx512.mask.vfnmadd.p") ||
4863 Name.starts_with(Prefix: "avx512.mask.vfnmsub.p") ||
4864 Name.starts_with(Prefix: "avx512.mask3.vfmadd.p") ||
4865 Name.starts_with(Prefix: "avx512.mask3.vfmsub.p") ||
4866 Name.starts_with(Prefix: "avx512.mask3.vfnmsub.p") ||
4867 Name.starts_with(Prefix: "avx512.maskz.vfmadd.p")) {
4868 bool IsMask3 = Name[11] == '3';
4869 bool IsMaskZ = Name[11] == 'z';
4870 // Drop the "avx512.mask." to make it easier.
4871 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4872 bool NegMul = Name[2] == 'n';
4873 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4874
4875 Value *A = CI->getArgOperand(i: 0);
4876 Value *B = CI->getArgOperand(i: 1);
4877 Value *C = CI->getArgOperand(i: 2);
4878
4879 if (NegMul && (IsMask3 || IsMaskZ))
4880 A = Builder.CreateFNeg(V: A);
4881 if (NegMul && !(IsMask3 || IsMaskZ))
4882 B = Builder.CreateFNeg(V: B);
4883 if (NegAcc)
4884 C = Builder.CreateFNeg(V: C);
4885
4886 if (CI->arg_size() == 5 &&
4887 (!isa<ConstantInt>(Val: CI->getArgOperand(i: 4)) ||
4888 cast<ConstantInt>(Val: CI->getArgOperand(i: 4))->getZExtValue() != 4)) {
4889 Intrinsic::ID IID;
4890 // Check the character before ".512" in string.
4891 if (Name[Name.size() - 5] == 's')
4892 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4893 else
4894 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4895
4896 Rep = Builder.CreateIntrinsic(ID: IID, Args: {A, B, C, CI->getArgOperand(i: 4)});
4897 } else {
4898 Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4899 }
4900
4901 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4902 : IsMask3 ? CI->getArgOperand(i: 2)
4903 : CI->getArgOperand(i: 0);
4904
4905 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4906 } else if (Name.starts_with(Prefix: "fma.vfmsubadd.p")) {
4907 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4908 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4909 Intrinsic::ID IID;
4910 if (VecWidth == 128 && EltWidth == 32)
4911 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4912 else if (VecWidth == 256 && EltWidth == 32)
4913 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4914 else if (VecWidth == 128 && EltWidth == 64)
4915 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4916 else if (VecWidth == 256 && EltWidth == 64)
4917 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4918 else
4919 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4920
4921 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4922 CI->getArgOperand(i: 2)};
4923 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4924 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4925 } else if (Name.starts_with(Prefix: "avx512.mask.vfmaddsub.p") ||
4926 Name.starts_with(Prefix: "avx512.mask3.vfmaddsub.p") ||
4927 Name.starts_with(Prefix: "avx512.maskz.vfmaddsub.p") ||
4928 Name.starts_with(Prefix: "avx512.mask3.vfmsubadd.p")) {
4929 bool IsMask3 = Name[11] == '3';
4930 bool IsMaskZ = Name[11] == 'z';
4931 // Drop the "avx512.mask." to make it easier.
4932 Name = Name.drop_front(N: IsMask3 || IsMaskZ ? 13 : 12);
4933 bool IsSubAdd = Name[3] == 's';
4934 if (CI->arg_size() == 5) {
4935 Intrinsic::ID IID;
4936 // Check the character before ".512" in string.
4937 if (Name[Name.size() - 5] == 's')
4938 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4939 else
4940 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4941
4942 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4943 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
4944 if (IsSubAdd)
4945 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4946
4947 Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4948 } else {
4949 int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4950
4951 Value *Ops[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4952 CI->getArgOperand(i: 2)};
4953
4954 Function *FMA = Intrinsic::getOrInsertDeclaration(
4955 M: CI->getModule(), id: Intrinsic::fma, OverloadTys: Ops[0]->getType());
4956 Value *Odd = Builder.CreateCall(Callee: FMA, Args: Ops);
4957 Ops[2] = Builder.CreateFNeg(V: Ops[2]);
4958 Value *Even = Builder.CreateCall(Callee: FMA, Args: Ops);
4959
4960 if (IsSubAdd)
4961 std::swap(a&: Even, b&: Odd);
4962
4963 SmallVector<int, 32> Idxs(NumElts);
4964 for (int i = 0; i != NumElts; ++i)
4965 Idxs[i] = i + (i % 2) * NumElts;
4966
4967 Rep = Builder.CreateShuffleVector(V1: Even, V2: Odd, Mask: Idxs);
4968 }
4969
4970 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4971 : IsMask3 ? CI->getArgOperand(i: 2)
4972 : CI->getArgOperand(i: 0);
4973
4974 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
4975 } else if (Name.starts_with(Prefix: "avx512.mask.pternlog.") ||
4976 Name.starts_with(Prefix: "avx512.maskz.pternlog.")) {
4977 bool ZeroMask = Name[11] == 'z';
4978 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4979 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4980 Intrinsic::ID IID;
4981 if (VecWidth == 128 && EltWidth == 32)
4982 IID = Intrinsic::x86_avx512_pternlog_d_128;
4983 else if (VecWidth == 256 && EltWidth == 32)
4984 IID = Intrinsic::x86_avx512_pternlog_d_256;
4985 else if (VecWidth == 512 && EltWidth == 32)
4986 IID = Intrinsic::x86_avx512_pternlog_d_512;
4987 else if (VecWidth == 128 && EltWidth == 64)
4988 IID = Intrinsic::x86_avx512_pternlog_q_128;
4989 else if (VecWidth == 256 && EltWidth == 64)
4990 IID = Intrinsic::x86_avx512_pternlog_q_256;
4991 else if (VecWidth == 512 && EltWidth == 64)
4992 IID = Intrinsic::x86_avx512_pternlog_q_512;
4993 else
4994 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4995
4996 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
4997 CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)};
4998 Rep = Builder.CreateIntrinsic(ID: IID, Args);
4999 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5000 : CI->getArgOperand(i: 0);
5001 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 4), Op0: Rep, Op1: PassThru);
5002 } else if (Name.starts_with(Prefix: "avx512.mask.vpmadd52") ||
5003 Name.starts_with(Prefix: "avx512.maskz.vpmadd52")) {
5004 bool ZeroMask = Name[11] == 'z';
5005 bool High = Name[20] == 'h' || Name[21] == 'h';
5006 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5007 Intrinsic::ID IID;
5008 if (VecWidth == 128 && !High)
5009 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5010 else if (VecWidth == 256 && !High)
5011 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5012 else if (VecWidth == 512 && !High)
5013 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5014 else if (VecWidth == 128 && High)
5015 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5016 else if (VecWidth == 256 && High)
5017 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5018 else if (VecWidth == 512 && High)
5019 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5020 else
5021 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5022
5023 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5024 CI->getArgOperand(i: 2)};
5025 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5026 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5027 : CI->getArgOperand(i: 0);
5028 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5029 } else if (Name.starts_with(Prefix: "avx512.mask.vpermi2var.") ||
5030 Name.starts_with(Prefix: "avx512.mask.vpermt2var.") ||
5031 Name.starts_with(Prefix: "avx512.maskz.vpermt2var.")) {
5032 bool ZeroMask = Name[11] == 'z';
5033 bool IndexForm = Name[17] == 'i';
5034 Rep = upgradeX86VPERMT2Intrinsics(Builder, CI&: *CI, ZeroMask, IndexForm);
5035 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpbusd.") ||
5036 Name.starts_with(Prefix: "avx512.maskz.vpdpbusd.") ||
5037 Name.starts_with(Prefix: "avx512.mask.vpdpbusds.") ||
5038 Name.starts_with(Prefix: "avx512.maskz.vpdpbusds.")) {
5039 bool ZeroMask = Name[11] == 'z';
5040 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5041 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5042 Intrinsic::ID IID;
5043 if (VecWidth == 128 && !IsSaturating)
5044 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5045 else if (VecWidth == 256 && !IsSaturating)
5046 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5047 else if (VecWidth == 512 && !IsSaturating)
5048 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5049 else if (VecWidth == 128 && IsSaturating)
5050 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5051 else if (VecWidth == 256 && IsSaturating)
5052 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5053 else if (VecWidth == 512 && IsSaturating)
5054 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5055 else
5056 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5057
5058 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5059 CI->getArgOperand(i: 2)};
5060
5061 // Input arguments types were incorrectly set to vectors of i32 before but
5062 // they should be vectors of i8. Insert bit cast when encountering the old
5063 // types
5064 if (Args[1]->getType()->isVectorTy() &&
5065 cast<VectorType>(Val: Args[1]->getType())
5066 ->getElementType()
5067 ->isIntegerTy(BitWidth: 32) &&
5068 Args[2]->getType()->isVectorTy() &&
5069 cast<VectorType>(Val: Args[2]->getType())
5070 ->getElementType()
5071 ->isIntegerTy(BitWidth: 32)) {
5072 Type *NewArgType = nullptr;
5073 if (VecWidth == 128)
5074 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 16, Scalable: false);
5075 else if (VecWidth == 256)
5076 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 32, Scalable: false);
5077 else if (VecWidth == 512)
5078 NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: 64, Scalable: false);
5079 else
5080 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
5081 CI);
5082
5083 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
5084 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
5085 }
5086
5087 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5088 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5089 : CI->getArgOperand(i: 0);
5090 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5091 } else if (Name.starts_with(Prefix: "avx512.mask.vpdpwssd.") ||
5092 Name.starts_with(Prefix: "avx512.maskz.vpdpwssd.") ||
5093 Name.starts_with(Prefix: "avx512.mask.vpdpwssds.") ||
5094 Name.starts_with(Prefix: "avx512.maskz.vpdpwssds.")) {
5095 bool ZeroMask = Name[11] == 'z';
5096 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5097 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5098 Intrinsic::ID IID;
5099 if (VecWidth == 128 && !IsSaturating)
5100 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5101 else if (VecWidth == 256 && !IsSaturating)
5102 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5103 else if (VecWidth == 512 && !IsSaturating)
5104 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5105 else if (VecWidth == 128 && IsSaturating)
5106 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5107 else if (VecWidth == 256 && IsSaturating)
5108 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5109 else if (VecWidth == 512 && IsSaturating)
5110 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5111 else
5112 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5113
5114 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5115 CI->getArgOperand(i: 2)};
5116
5117 // Input arguments types were incorrectly set to vectors of i32 before but
5118 // they should be vectors of i16. Insert bit cast when encountering the old
5119 // types
5120 if (Args[1]->getType()->isVectorTy() &&
5121 cast<VectorType>(Val: Args[1]->getType())
5122 ->getElementType()
5123 ->isIntegerTy(BitWidth: 32) &&
5124 Args[2]->getType()->isVectorTy() &&
5125 cast<VectorType>(Val: Args[2]->getType())
5126 ->getElementType()
5127 ->isIntegerTy(BitWidth: 32)) {
5128 Type *NewArgType = nullptr;
5129 if (VecWidth == 128)
5130 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 8, Scalable: false);
5131 else if (VecWidth == 256)
5132 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 16, Scalable: false);
5133 else if (VecWidth == 512)
5134 NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: 32, Scalable: false);
5135 else
5136 reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
5137 CI);
5138
5139 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
5140 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
5141 }
5142
5143 Rep = Builder.CreateIntrinsic(ID: IID, Args);
5144 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
5145 : CI->getArgOperand(i: 0);
5146 Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: 3), Op0: Rep, Op1: PassThru);
5147 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5148 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5149 Name == "subborrow.u32" || Name == "subborrow.u64") {
5150 Intrinsic::ID IID;
5151 if (Name[0] == 'a' && Name.back() == '2')
5152 IID = Intrinsic::x86_addcarry_32;
5153 else if (Name[0] == 'a' && Name.back() == '4')
5154 IID = Intrinsic::x86_addcarry_64;
5155 else if (Name[0] == 's' && Name.back() == '2')
5156 IID = Intrinsic::x86_subborrow_32;
5157 else if (Name[0] == 's' && Name.back() == '4')
5158 IID = Intrinsic::x86_subborrow_64;
5159 else
5160 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5161
5162 // Make a call with 3 operands.
5163 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5164 CI->getArgOperand(i: 2)};
5165 Value *NewCall = Builder.CreateIntrinsic(ID: IID, Args);
5166
5167 // Extract the second result and store it.
5168 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
5169 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 3), Align: Align(1));
5170 // Replace the original call result with the first result of the new call.
5171 Value *CF = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
5172
5173 CI->replaceAllUsesWith(V: CF);
5174 Rep = nullptr;
5175 } else if (Name.starts_with(Prefix: "avx512.mask.") &&
5176 upgradeAVX512MaskToSelect(Name, Builder, CI&: *CI, Rep)) {
5177 // Rep will be updated by the call in the condition.
5178 } else if (Name.starts_with(Prefix: "bmi.pdep.")) {
5179 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pdep);
5180 } else if (Name.starts_with(Prefix: "bmi.pext.")) {
5181 Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pext);
5182 } else
5183 reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
5184
5185 return Rep;
5186}
5187
5188static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5189 Function *F, IRBuilder<> &Builder) {
5190 if (Name.starts_with(Prefix: "neon.bfcvt")) {
5191 if (Name.starts_with(Prefix: "neon.bfcvtn2")) {
5192 SmallVector<int, 32> LoMask(4);
5193 std::iota(first: LoMask.begin(), last: LoMask.end(), value: 0);
5194 SmallVector<int, 32> ConcatMask(8);
5195 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
5196 Value *Inactive = Builder.CreateShuffleVector(V: CI->getOperand(i_nocapture: 0), Mask: LoMask);
5197 Value *Trunc =
5198 Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 1), DestTy: Inactive->getType());
5199 return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
5200 } else if (Name.starts_with(Prefix: "neon.bfcvtn")) {
5201 SmallVector<int, 32> ConcatMask(8);
5202 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
5203 Type *V4BF16 =
5204 FixedVectorType::get(ElementType: Type::getBFloatTy(C&: F->getContext()), NumElts: 4);
5205 Value *Trunc = Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0), DestTy: V4BF16);
5206 dbgs() << "Trunc: " << *Trunc << "\n";
5207 return Builder.CreateShuffleVector(
5208 V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
5209 } else {
5210 return Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: 0),
5211 DestTy: Type::getBFloatTy(C&: F->getContext()));
5212 }
5213 } else if (Name.starts_with(Prefix: "sve.fcvt")) {
5214 Intrinsic::ID NewID =
5215 StringSwitch<Intrinsic::ID>(Name)
5216 .Case(S: "sve.fcvt.bf16f32", Value: Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5217 .Case(S: "sve.fcvtnt.bf16f32",
5218 Value: Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5219 .Default(Value: Intrinsic::not_intrinsic);
5220 if (NewID == Intrinsic::not_intrinsic)
5221 llvm_unreachable("Unhandled Intrinsic!");
5222
5223 SmallVector<Value *, 3> Args(CI->args());
5224
5225 // The original intrinsics incorrectly used a predicate based on the
5226 // smallest element type rather than the largest.
5227 Type *BadPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 8);
5228 Type *GoodPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: 4);
5229
5230 if (Args[1]->getType() != BadPredTy)
5231 llvm_unreachable("Unexpected predicate type!");
5232
5233 Args[1] = Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
5234 OverloadTypes: BadPredTy, Args: Args[1]);
5235 Args[1] = Builder.CreateIntrinsic(
5236 ID: Intrinsic::aarch64_sve_convert_from_svbool, OverloadTypes: GoodPredTy, Args: Args[1]);
5237
5238 return Builder.CreateIntrinsic(ID: NewID, Args, /*FMFSource=*/nullptr,
5239 Name: CI->getName());
5240 }
5241
5242 if (Name == "neon.vcvtfp2hf")
5243 return Builder.CreateBitCast(
5244 V: Builder.CreateFPTrunc(
5245 V: CI->getOperand(i_nocapture: 0),
5246 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
5247 DestTy: FixedVectorType::get(ElementType: Type::getInt16Ty(C&: F->getContext()), NumElts: 4));
5248 if (Name == "neon.vcvthf2fp")
5249 return Builder.CreateFPExt(
5250 V: Builder.CreateBitCast(
5251 V: CI->getOperand(i_nocapture: 0),
5252 DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: 4)),
5253 DestTy: FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: 4));
5254
5255 llvm_unreachable("Unhandled Intrinsic!");
5256}
5257
5258static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5259 IRBuilder<> &Builder) {
5260 if (Name == "mve.vctp64.old") {
5261 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5262 // correct type.
5263 Value *VCTP = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vctp64, OverloadTypes: {},
5264 Args: CI->getArgOperand(i: 0),
5265 /*FMFSource=*/nullptr, Name: CI->getName());
5266 Value *C1 = Builder.CreateIntrinsic(
5267 ID: Intrinsic::arm_mve_pred_v2i,
5268 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 2, Scalable: false)}, Args: VCTP);
5269 return Builder.CreateIntrinsic(
5270 ID: Intrinsic::arm_mve_pred_i2v,
5271 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: C1);
5272 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5273 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5274 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5275 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5276 Name ==
5277 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5278 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5279 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5280 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5281 Name ==
5282 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5283 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5284 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5285 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5286 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5287 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5288 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5289 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5290 std::vector<Type *> Tys;
5291 unsigned ID = CI->getIntrinsicID();
5292 Type *V2I1Ty = FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts: 2);
5293 switch (ID) {
5294 case Intrinsic::arm_mve_mull_int_predicated:
5295 case Intrinsic::arm_mve_vqdmull_predicated:
5296 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5297 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(), V2I1Ty};
5298 break;
5299 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5300 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5301 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5302 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5303 V2I1Ty};
5304 break;
5305 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5306 Tys = {CI->getType(), CI->getOperand(i_nocapture: 0)->getType(),
5307 CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5308 break;
5309 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5310 Tys = {CI->getOperand(i_nocapture: 0)->getType(), CI->getOperand(i_nocapture: 1)->getType(),
5311 CI->getOperand(i_nocapture: 2)->getType(), V2I1Ty};
5312 break;
5313 case Intrinsic::arm_cde_vcx1q_predicated:
5314 case Intrinsic::arm_cde_vcx1qa_predicated:
5315 case Intrinsic::arm_cde_vcx2q_predicated:
5316 case Intrinsic::arm_cde_vcx2qa_predicated:
5317 case Intrinsic::arm_cde_vcx3q_predicated:
5318 case Intrinsic::arm_cde_vcx3qa_predicated:
5319 Tys = {CI->getOperand(i_nocapture: 1)->getType(), V2I1Ty};
5320 break;
5321 default:
5322 llvm_unreachable("Unhandled Intrinsic!");
5323 }
5324
5325 std::vector<Value *> Ops;
5326 for (Value *Op : CI->args()) {
5327 Type *Ty = Op->getType();
5328 if (Ty->getScalarSizeInBits() == 1) {
5329 Value *C1 = Builder.CreateIntrinsic(
5330 ID: Intrinsic::arm_mve_pred_v2i,
5331 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: 4, Scalable: false)}, Args: Op);
5332 Op = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_pred_i2v, OverloadTypes: {V2I1Ty}, Args: C1);
5333 }
5334 Ops.push_back(x: Op);
5335 }
5336
5337 return Builder.CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops, /*FMFSource=*/nullptr,
5338 Name: CI->getName());
5339 }
5340 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5341}
5342
5343// These are expected to have the arguments:
5344// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5345//
5346// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5347//
5348static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5349 Function *F, IRBuilder<> &Builder) {
5350 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5351 // for compatibility.
5352 auto UpgradeLegacyWMMAIUIntrinsicCall =
5353 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5354 ArrayRef<Type *> OverloadTys) -> Value * {
5355 // Prepare arguments, append clamp=0 for compatibility
5356 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5357 Args.push_back(Elt: Builder.getFalse());
5358
5359 // Insert the declaration for the right overload types
5360 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5361 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys);
5362
5363 // Copy operand bundles if any
5364 SmallVector<OperandBundleDef, 1> Bundles;
5365 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5366
5367 // Create the new call and copy calling properties
5368 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5369 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5370 NewCall->setCallingConv(CI->getCallingConv());
5371 NewCall->setAttributes(CI->getAttributes());
5372 NewCall->copyMetadata(SrcInst: *CI);
5373 return NewCall;
5374 };
5375
5376 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5377 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5378 "intrinsic should have 7 arguments");
5379 Type *T1 = CI->getArgOperand(i: 4)->getType();
5380 Type *T2 = CI->getArgOperand(i: 1)->getType();
5381 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5382 }
5383 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5384 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5385 "intrinsic should have 8 arguments");
5386 Type *T1 = CI->getArgOperand(i: 4)->getType();
5387 Type *T2 = CI->getArgOperand(i: 1)->getType();
5388 Type *T3 = CI->getArgOperand(i: 3)->getType();
5389 Type *T4 = CI->getArgOperand(i: 5)->getType();
5390 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5391 }
5392
5393 switch (F->getIntrinsicID()) {
5394 default:
5395 break;
5396 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5397 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5398 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5399 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5400 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5401 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5402 // Drop src0 and src1 modifiers.
5403 const Value *Op0 = CI->getArgOperand(i: 0);
5404 const Value *Op2 = CI->getArgOperand(i: 2);
5405 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5406 const ConstantInt *ModA = dyn_cast<ConstantInt>(Val: Op0);
5407 const ConstantInt *ModB = dyn_cast<ConstantInt>(Val: Op2);
5408 if (!ModA->isZero() || !ModB->isZero())
5409 reportFatalUsageError(reason: Name + " matrix A and B modifiers shall be zero");
5410
5411 SmallVector<Value *, 8> Args{CI->getArgOperand(i: 1), CI->getArgOperand(i: 3)};
5412 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5413 Args.push_back(Elt: CI->getArgOperand(i: I));
5414
5415 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5416 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5417 Overloads.push_back(Elt: Args[3]->getType());
5418 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5419 M: F->getParent(), id: F->getIntrinsicID(), OverloadTys: Overloads);
5420
5421 SmallVector<OperandBundleDef, 1> Bundles;
5422 CI->getOperandBundlesAsDefs(Defs&: Bundles);
5423
5424 auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
5425 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5426 NewCall->setCallingConv(CI->getCallingConv());
5427 NewCall->setAttributes(CI->getAttributes());
5428 NewCall->copyMetadata(SrcInst: *CI);
5429 NewCall->takeName(V: CI);
5430 return NewCall;
5431 }
5432 }
5433
5434 if (Name.starts_with(Prefix: "fcmp.") || Name.starts_with(Prefix: "icmp.")) {
5435 Value *LHS = CI->getArgOperand(i: 0);
5436 Value *RHS = CI->getArgOperand(i: 1);
5437 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5438 cast<ConstantInt>(Val: CI->getArgOperand(i: 2))->getZExtValue());
5439 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5440 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5441 RetTy: CI->getType(), ID: Intrinsic::amdgcn_ballot, Args: Cmp);
5442 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5443 NewCall->setCallingConv(CI->getCallingConv());
5444 NewCall->copyMetadata(SrcInst: *CI);
5445 NewCall->takeName(V: CI);
5446 return NewCall;
5447 }
5448
5449 if (Name.starts_with(Prefix: "addrspacecast.nonnull")) {
5450 if (CI->getNumOperands() < 2) // Malformed bitcode.
5451 return nullptr;
5452 Value *ASC = Builder.CreateAddrSpaceCast(
5453 V: CI->getArgOperand(i: 0), DestTy: CI->getType(), Name: "", /*IsNonNull=*/true);
5454 ASC->takeName(V: CI);
5455 return ASC;
5456 }
5457
5458 AtomicRMWInst::BinOp RMWOp =
5459 StringSwitch<AtomicRMWInst::BinOp>(Name)
5460 .StartsWith(S: "ds.fadd", Value: AtomicRMWInst::FAdd)
5461 .StartsWith(S: "ds.fmin", Value: AtomicRMWInst::FMin)
5462 .StartsWith(S: "ds.fmax", Value: AtomicRMWInst::FMax)
5463 .StartsWith(S: "atomic.inc.", Value: AtomicRMWInst::UIncWrap)
5464 .StartsWith(S: "atomic.dec.", Value: AtomicRMWInst::UDecWrap)
5465 .StartsWith(S: "global.atomic.fadd", Value: AtomicRMWInst::FAdd)
5466 .StartsWith(S: "flat.atomic.fadd", Value: AtomicRMWInst::FAdd)
5467 .StartsWith(S: "global.atomic.fmin", Value: AtomicRMWInst::FMin)
5468 .StartsWith(S: "flat.atomic.fmin", Value: AtomicRMWInst::FMin)
5469 .StartsWith(S: "global.atomic.fmax", Value: AtomicRMWInst::FMax)
5470 .StartsWith(S: "flat.atomic.fmax", Value: AtomicRMWInst::FMax)
5471 .StartsWith(S: "atomic.cond.sub", Value: AtomicRMWInst::USubCond)
5472 .StartsWith(S: "atomic.csub", Value: AtomicRMWInst::USubSat);
5473
5474 unsigned NumOperands = CI->getNumOperands();
5475 if (NumOperands < 3) // Malformed bitcode.
5476 return nullptr;
5477
5478 Value *Ptr = CI->getArgOperand(i: 0);
5479 PointerType *PtrTy = dyn_cast<PointerType>(Val: Ptr->getType());
5480 if (!PtrTy) // Malformed.
5481 return nullptr;
5482
5483 Value *Val = CI->getArgOperand(i: 1);
5484 if (Val->getType() != CI->getType()) // Malformed.
5485 return nullptr;
5486
5487 ConstantInt *OrderArg = nullptr;
5488 bool IsVolatile = false;
5489
5490 // These should have 5 arguments (plus the callee). A separate version of the
5491 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5492 if (NumOperands > 3)
5493 OrderArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5494
5495 // Ignore scope argument at 3
5496
5497 if (NumOperands > 5) {
5498 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 4));
5499 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5500 }
5501
5502 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5503 if (OrderArg && isValidAtomicOrdering(I: OrderArg->getZExtValue()))
5504 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5505 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5506 Order = AtomicOrdering::SequentiallyConsistent;
5507
5508 LLVMContext &Ctx = F->getContext();
5509
5510 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5511 Type *RetTy = CI->getType();
5512 if (VectorType *VT = dyn_cast<VectorType>(Val: RetTy)) {
5513 if (VT->getElementType()->isIntegerTy(BitWidth: 16)) {
5514 VectorType *AsBF16 =
5515 VectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), EC: VT->getElementCount());
5516 Val = Builder.CreateBitCast(V: Val, DestTy: AsBF16);
5517 }
5518 }
5519
5520 // The scope argument never really worked correctly. Use agent as the most
5521 // conservative option which should still always produce the instruction.
5522 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID(SSN: "agent");
5523 AtomicRMWInst *RMW =
5524 Builder.CreateAtomicRMW(Op: RMWOp, Ptr, Val, Align: std::nullopt, Ordering: Order, SSID);
5525
5526 unsigned AddrSpace = PtrTy->getAddressSpace();
5527 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5528 MDNode *EmptyMD = MDNode::get(Context&: F->getContext(), MDs: {});
5529 RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: EmptyMD);
5530 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5531 RMW->setMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode, Node: EmptyMD);
5532 }
5533
5534 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5535 MDBuilder MDB(F->getContext());
5536 MDNode *RangeNotPrivate =
5537 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5538 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5539 RMW->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: RangeNotPrivate);
5540 }
5541
5542 if (IsVolatile)
5543 RMW->setVolatile(true);
5544
5545 return Builder.CreateBitCast(V: RMW, DestTy: RetTy);
5546}
5547
5548/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5549/// plain MDNode, as it's the verifier's job to check these are the correct
5550/// types later.
5551static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5552 if (Op < CI->arg_size()) {
5553 if (MetadataAsValue *MAV =
5554 dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op))) {
5555 Metadata *MD = MAV->getMetadata();
5556 return dyn_cast_if_present<MDNode>(Val: MD);
5557 }
5558 }
5559 return nullptr;
5560}
5561
5562/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5563static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5564 if (Op < CI->arg_size())
5565 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op)))
5566 return MAV->getMetadata();
5567 return nullptr;
5568}
5569
5570/// Convert debug intrinsic calls to non-instruction debug records.
5571/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5572/// \p CI - The debug intrinsic call.
5573static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5574 DbgRecord *DR = nullptr;
5575 if (Name == "label") {
5576 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(Label: unwrapMAVOp(CI, Op: 0));
5577 } else if (Name == "assign") {
5578 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5579 Type: DbgVariableRecord::LocationType::Assign, Val: unwrapMAVMetadataOp(CI, Op: 0),
5580 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: unwrapMAVOp(CI, Op: 3),
5581 Address: unwrapMAVMetadataOp(CI, Op: 4),
5582 /*The address is a Value ref, it will be stored as a Metadata */
5583 AddressExpression: unwrapMAVOp(CI, Op: 5));
5584 } else if (Name == "declare") {
5585 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5586 Type: DbgVariableRecord::LocationType::Declare, Val: unwrapMAVMetadataOp(CI, Op: 0),
5587 Variable: unwrapMAVOp(CI, Op: 1), Expression: unwrapMAVOp(CI, Op: 2), AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5588 } else if (Name == "addr") {
5589 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5590 MDNode *ExprNode = unwrapMAVOp(CI, Op: 2);
5591 // Don't try to add something to the expression if it's not an expression.
5592 // Instead, allow the verifier to fail later.
5593 if (DIExpression *Expr = dyn_cast<DIExpression>(Val: ExprNode)) {
5594 ExprNode = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
5595 }
5596 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5597 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5598 Variable: unwrapMAVOp(CI, Op: 1), Expression: ExprNode, AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5599 } else if (Name == "value") {
5600 // An old version of dbg.value had an extra offset argument.
5601 unsigned VarOp = 1;
5602 unsigned ExprOp = 2;
5603 if (CI->arg_size() == 4) {
5604 auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1));
5605 // Nonzero offset dbg.values get dropped without a replacement.
5606 if (!Offset || !Offset->isNullValue())
5607 return;
5608 VarOp = 2;
5609 ExprOp = 3;
5610 }
5611 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5612 Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: 0),
5613 Variable: unwrapMAVOp(CI, Op: VarOp), Expression: unwrapMAVOp(CI, Op: ExprOp), AssignID: nullptr, Address: nullptr,
5614 AddressExpression: nullptr);
5615 }
5616 DR->setDebugLoc(CI->getDebugLoc());
5617 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5618 CI->getParent()->insertDbgRecordBefore(DR, Here: CI->getIterator());
5619}
5620
5621static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5622 auto *Offset = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: 2));
5623 if (!Offset)
5624 reportFatalUsageError(reason: "Invalid llvm.vector.splice offset argument");
5625 int64_t OffsetVal = Offset->getSExtValue();
5626 return Builder.CreateIntrinsic(ID: OffsetVal >= 0
5627 ? Intrinsic::vector_splice_left
5628 : Intrinsic::vector_splice_right,
5629 OverloadTypes: CI->getType(),
5630 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
5631 Builder.getInt32(C: std::abs(i: OffsetVal))});
5632}
5633
5634static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5635 Function *F, IRBuilder<> &Builder) {
5636 if (Name.starts_with(Prefix: "to.fp16")) {
5637 Value *Cast =
5638 Builder.CreateFPTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5639 return Builder.CreateBitCast(V: Cast, DestTy: CI->getType());
5640 }
5641
5642 if (Name.starts_with(Prefix: "from.fp16")) {
5643 Value *Cast =
5644 Builder.CreateBitCast(V: CI->getArgOperand(i: 0), DestTy: Builder.getHalfTy());
5645 return Builder.CreateFPExt(V: Cast, DestTy: CI->getType());
5646 }
5647
5648 return nullptr;
5649}
5650
5651static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5652 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5653 if (!MD || !isa<MDString>(Val: MD))
5654 return ICmpInst::BAD_ICMP_PREDICATE;
5655 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5656 .Case(S: "eq", Value: ICmpInst::ICMP_EQ)
5657 .Case(S: "ne", Value: ICmpInst::ICMP_NE)
5658 .Case(S: "ugt", Value: ICmpInst::ICMP_UGT)
5659 .Case(S: "uge", Value: ICmpInst::ICMP_UGE)
5660 .Case(S: "ult", Value: ICmpInst::ICMP_ULT)
5661 .Case(S: "ule", Value: ICmpInst::ICMP_ULE)
5662 .Case(S: "sgt", Value: ICmpInst::ICMP_SGT)
5663 .Case(S: "sge", Value: ICmpInst::ICMP_SGE)
5664 .Case(S: "slt", Value: ICmpInst::ICMP_SLT)
5665 .Case(S: "sle", Value: ICmpInst::ICMP_SLE)
5666 .Default(Value: ICmpInst::BAD_ICMP_PREDICATE);
5667}
5668
5669static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5670 Metadata *MD = cast<MetadataAsValue>(Val: Op)->getMetadata();
5671 if (!MD || !isa<MDString>(Val: MD))
5672 return FCmpInst::BAD_FCMP_PREDICATE;
5673 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(Val: MD)->getString())
5674 .Case(S: "oeq", Value: FCmpInst::FCMP_OEQ)
5675 .Case(S: "ogt", Value: FCmpInst::FCMP_OGT)
5676 .Case(S: "oge", Value: FCmpInst::FCMP_OGE)
5677 .Case(S: "olt", Value: FCmpInst::FCMP_OLT)
5678 .Case(S: "ole", Value: FCmpInst::FCMP_OLE)
5679 .Case(S: "one", Value: FCmpInst::FCMP_ONE)
5680 .Case(S: "ord", Value: FCmpInst::FCMP_ORD)
5681 .Case(S: "uno", Value: FCmpInst::FCMP_UNO)
5682 .Case(S: "ueq", Value: FCmpInst::FCMP_UEQ)
5683 .Case(S: "ugt", Value: FCmpInst::FCMP_UGT)
5684 .Case(S: "uge", Value: FCmpInst::FCMP_UGE)
5685 .Case(S: "ult", Value: FCmpInst::FCMP_ULT)
5686 .Case(S: "ule", Value: FCmpInst::FCMP_ULE)
5687 .Case(S: "une", Value: FCmpInst::FCMP_UNE)
5688 .Default(Value: FCmpInst::BAD_FCMP_PREDICATE);
5689}
5690
5691static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5692 IRBuilder<> &Builder) {
5693 Value *Rep;
5694 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5695 if (Opcode && Instruction::isUnaryOp(Opcode))
5696 Rep =
5697 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)Opcode, V: CI->getArgOperand(i: 0));
5698 else if (Opcode && Instruction::isBinaryOp(Opcode))
5699 Rep = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opcode,
5700 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5701 else if (Opcode && Instruction::isCast(Opcode))
5702 Rep = Builder.CreateCast(Op: (Instruction::CastOps)Opcode, V: CI->getArgOperand(i: 0),
5703 DestTy: CI->getType());
5704 else if (Opcode == Instruction::ICmp)
5705 Rep = Builder.CreateICmp(P: getVPIntPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5706 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5707 else if (Opcode == Instruction::FCmp)
5708 Rep = Builder.CreateFCmp(P: getVPFPPredicateFromMD(Op: CI->getArgOperand(i: 2)),
5709 LHS: CI->getArgOperand(i: 0), RHS: CI->getArgOperand(i: 1));
5710 else if (Opcode == Instruction::Select)
5711 Rep = Builder.CreateSelect(C: CI->getArgOperand(i: 0), True: CI->getArgOperand(i: 1),
5712 False: CI->getArgOperand(i: 2));
5713 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5714 SmallVector<Value *, 2> Args(drop_end(RangeOrContainer: CI->args(), N: 2));
5715 Rep = Builder.CreateIntrinsic(RetTy: CI->getType(), ID: IntrinsicID, Args, FMFSource: {});
5716 } else
5717 llvm_unreachable("Unexpected vp intrinsic");
5718 Rep->takeName(V: CI);
5719 return Rep;
5720}
5721
5722static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5723 IRBuilder<> &Builder) {
5724 Intrinsic::ID IID = NewFn->getIntrinsicID();
5725
5726 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5727 if (Defaults.empty())
5728 return false;
5729
5730 unsigned OldArgCount = CI->arg_size();
5731 unsigned NewArgCount = NewFn->arg_size();
5732
5733 if (OldArgCount < FirstDefault)
5734 return false;
5735
5736 // More arguments than the new intrinsic accepts, cannot upgrade.
5737 if (OldArgCount > NewArgCount)
5738 return false;
5739
5740 // The call already passes the defaulted arguments explicitly; only the
5741 // callee is still the old, shorter declaration, so retarget it.
5742 if (OldArgCount == NewArgCount) {
5743 if (CI->getFunctionType() != NewFn->getFunctionType())
5744 return false;
5745 CI->setCalledFunction(NewFn);
5746 return true;
5747 }
5748
5749 // OldArgCount < NewArgCount: Fill in each missing trailing default
5750 // argument from the table.
5751 SmallVector<Value *, 8> NewArgs(CI->args());
5752
5753 FunctionType *NewFT = NewFn->getFunctionType();
5754 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5755 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5756 "missing argument outside the default range");
5757 Type *ParamTy = NewFT->getParamType(i: Idx);
5758
5759 // Only integer types are supported (i1, i8, i16, i32, i64).
5760 if (!ParamTy->isIntegerTy())
5761 return false;
5762 NewArgs.push_back(Elt: ConstantInt::get(Ty: ParamTy, V: Defaults[Idx - FirstDefault]));
5763 }
5764
5765 // Preserve operand bundles by creating the call with them.
5766 SmallVector<OperandBundleDef, 1> OpBundles;
5767 CI->getOperandBundlesAsDefs(Defs&: OpBundles);
5768 CallInst *NewCall = Builder.CreateCall(Callee: NewFn, Args: NewArgs, OpBundles);
5769
5770 NewCall->takeName(V: CI);
5771 NewCall->setCallingConv(CI->getCallingConv());
5772 NewCall->copyMetadata(SrcInst: *CI);
5773 if (auto *OldCI = dyn_cast<CallInst>(Val: CI))
5774 NewCall->setTailCallKind(OldCI->getTailCallKind());
5775
5776 CI->replaceAllUsesWith(V: NewCall);
5777 CI->eraseFromParent();
5778 return true;
5779}
5780
5781/// Upgrade a call to an old intrinsic. All argument and return casting must be
5782/// provided to seamlessly integrate with existing context.
5783void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5784 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5785 // checks the callee's function type matches. It's likely we need to handle
5786 // type changes here.
5787 Function *F = dyn_cast<Function>(Val: CI->getCalledOperand());
5788 if (!F)
5789 return;
5790
5791 LLVMContext &C = CI->getContext();
5792 IRBuilder<> Builder(CI->getIterator());
5793 if (isa<FPMathOperator>(Val: CI))
5794 Builder.setFastMathFlags(CI->getFastMathFlags());
5795
5796 if (!NewFn) {
5797 // Get the Function's name.
5798 StringRef Name = F->getName();
5799 if (!Name.consume_front(Prefix: "llvm."))
5800 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5801
5802 bool IsX86 = Name.consume_front(Prefix: "x86.");
5803 bool IsNVVM = Name.consume_front(Prefix: "nvvm.");
5804 bool IsAArch64 = Name.consume_front(Prefix: "aarch64.");
5805 bool IsARM = Name.consume_front(Prefix: "arm.");
5806 bool IsAMDGCN = Name.consume_front(Prefix: "amdgcn.");
5807 bool IsDbg = Name.consume_front(Prefix: "dbg.");
5808 bool IsOldSplice =
5809 (Name.consume_front(Prefix: "experimental.vector.splice") ||
5810 Name.consume_front(Prefix: "vector.splice")) &&
5811 !(Name.starts_with(Prefix: ".left") || Name.starts_with(Prefix: ".right"));
5812 Value *Rep = nullptr;
5813
5814 if (!IsX86 && Name == "stackprotectorcheck") {
5815 Rep = nullptr;
5816 } else if (IsNVVM) {
5817 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5818 } else if (IsX86) {
5819 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5820 } else if (IsAArch64) {
5821 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5822 } else if (IsARM) {
5823 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5824 } else if (IsAMDGCN) {
5825 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5826 } else if (IsDbg) {
5827 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5828 } else if (IsOldSplice) {
5829 Rep = upgradeVectorSplice(CI, Builder);
5830 } else if (Name.consume_front(Prefix: "convert.")) {
5831 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5832 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5833 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5834 Rep = nullptr;
5835 } else if (shouldUpgradeVPIntrinsic(Name)) {
5836 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5837 } else {
5838 llvm_unreachable("Unknown function for CallBase upgrade.");
5839 }
5840
5841 if (Rep)
5842 CI->replaceAllUsesWith(V: Rep);
5843 CI->eraseFromParent();
5844 return;
5845 }
5846
5847 const auto &DefaultCase = [&]() -> void {
5848 if (F == NewFn)
5849 return;
5850
5851 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5852 // Handle generic mangling change.
5853 assert(
5854 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5855 "Unknown function for CallBase upgrade and isn't just a name change");
5856 CI->setCalledFunction(NewFn);
5857 return;
5858 }
5859
5860 // This must be an upgrade from a named to a literal struct.
5861 if (auto *OldST = dyn_cast<StructType>(Val: CI->getType())) {
5862 assert(OldST != NewFn->getReturnType() &&
5863 "Return type must have changed");
5864 assert(OldST->getNumElements() ==
5865 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5866 "Must have same number of elements");
5867
5868 SmallVector<Value *> Args(CI->args());
5869 CallInst *NewCI = Builder.CreateCall(Callee: NewFn, Args);
5870 NewCI->setAttributes(CI->getAttributes());
5871 Value *Res = PoisonValue::get(T: OldST);
5872 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5873 Value *Elem = Builder.CreateExtractValue(Agg: NewCI, Idxs: Idx);
5874 Res = Builder.CreateInsertValue(Agg: Res, Val: Elem, Idxs: Idx);
5875 }
5876 CI->replaceAllUsesWith(V: Res);
5877 CI->eraseFromParent();
5878 return;
5879 }
5880
5881 // We're probably about to produce something invalid. Let the verifier catch
5882 // it instead of dying here.
5883 CI->setCalledOperand(
5884 ConstantExpr::getPointerCast(C: NewFn, Ty: CI->getCalledOperand()->getType()));
5885 return;
5886 };
5887 CallInst *NewCall = nullptr;
5888 switch (NewFn->getIntrinsicID()) {
5889 default: {
5890 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5891 return;
5892 DefaultCase();
5893 return;
5894 }
5895 case Intrinsic::arm_neon_vst1:
5896 case Intrinsic::arm_neon_vst2:
5897 case Intrinsic::arm_neon_vst3:
5898 case Intrinsic::arm_neon_vst4:
5899 case Intrinsic::arm_neon_vst2lane:
5900 case Intrinsic::arm_neon_vst3lane:
5901 case Intrinsic::arm_neon_vst4lane: {
5902 SmallVector<Value *, 4> Args(CI->args());
5903 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5904 break;
5905 }
5906 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5907 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5908 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5909 LLVMContext &Ctx = F->getParent()->getContext();
5910 SmallVector<Value *, 4> Args(CI->args());
5911 Args[3] = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx),
5912 V: cast<ConstantInt>(Val: Args[3])->getZExtValue());
5913 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5914 break;
5915 }
5916 case Intrinsic::aarch64_sve_ld3_sret:
5917 case Intrinsic::aarch64_sve_ld4_sret:
5918 case Intrinsic::aarch64_sve_ld2_sret: {
5919 // Is this a trivial remangle of the name to support ptr address spaces?
5920 if (isa<StructType>(Val: F->getReturnType())) {
5921 DefaultCase();
5922 return;
5923 }
5924
5925 StringRef Name = F->getName();
5926 Name = Name.substr(Start: 5);
5927 unsigned N = StringSwitch<unsigned>(Name)
5928 .StartsWith(S: "aarch64.sve.ld2", Value: 2)
5929 .StartsWith(S: "aarch64.sve.ld3", Value: 3)
5930 .StartsWith(S: "aarch64.sve.ld4", Value: 4)
5931 .Default(Value: 0);
5932 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5933 unsigned MinElts = RetTy->getMinNumElements() / N;
5934 SmallVector<Value *, 2> Args(CI->args());
5935 Value *NewLdCall = Builder.CreateCall(Callee: NewFn, Args);
5936 Value *Ret = llvm::PoisonValue::get(T: RetTy);
5937 for (unsigned I = 0; I < N; I++) {
5938 Value *SRet = Builder.CreateExtractValue(Agg: NewLdCall, Idxs: I);
5939 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: SRet, Idx: I * MinElts);
5940 }
5941 NewCall = dyn_cast<CallInst>(Val: Ret);
5942 break;
5943 }
5944
5945 case Intrinsic::coro_end_async:
5946 case Intrinsic::coro_end: {
5947 SmallVector<Value *, 3> Args(CI->args());
5948 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5949 Args.push_back(Elt: ConstantTokenNone::get(Context&: CI->getContext()));
5950 NewCall = Builder.CreateCall(Callee: NewFn, Args);
5951
5952 if (!CI->getType()->isVoidTy()) {
5953 if (!CI->use_empty()) {
5954 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5955 M: CI->getModule(), id: Intrinsic::coro_is_in_ramp);
5956 Value *InRamp = Builder.CreateCall(Callee: IsInRamp);
5957 CI->replaceAllUsesWith(V: Builder.CreateNot(V: InRamp));
5958 }
5959 CI->eraseFromParent();
5960 return;
5961 }
5962
5963 break;
5964 }
5965
5966 case Intrinsic::vector_extract: {
5967 StringRef Name = F->getName();
5968 Name = Name.substr(Start: 5); // Strip llvm
5969 if (!Name.starts_with(Prefix: "aarch64.sve.tuple.get")) {
5970 DefaultCase();
5971 return;
5972 }
5973 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5974 unsigned MinElts = RetTy->getMinNumElements();
5975 uint64_t I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
5976 Value *NewIdx = ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * MinElts);
5977 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), NewIdx});
5978 break;
5979 }
5980
5981 case Intrinsic::vector_insert: {
5982 StringRef Name = F->getName();
5983 Name = Name.substr(Start: 5);
5984 if (!Name.starts_with(Prefix: "aarch64.sve.tuple")) {
5985 DefaultCase();
5986 return;
5987 }
5988 if (Name.starts_with(Prefix: "aarch64.sve.tuple.set")) {
5989 uint64_t I = cast<ConstantInt>(Val: CI->getArgOperand(i: 1))->getZExtValue();
5990 auto *Ty = cast<ScalableVectorType>(Val: CI->getArgOperand(i: 2)->getType());
5991 Value *NewIdx =
5992 ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * Ty->getMinNumElements());
5993 NewCall = Builder.CreateCall(
5994 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), NewIdx});
5995 break;
5996 }
5997 if (Name.starts_with(Prefix: "aarch64.sve.tuple.create")) {
5998 unsigned N = StringSwitch<unsigned>(Name)
5999 .StartsWith(S: "aarch64.sve.tuple.create2", Value: 2)
6000 .StartsWith(S: "aarch64.sve.tuple.create3", Value: 3)
6001 .StartsWith(S: "aarch64.sve.tuple.create4", Value: 4)
6002 .Default(Value: 0);
6003 assert(N > 1 && "Create is expected to be between 2-4");
6004 auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
6005 Value *Ret = llvm::PoisonValue::get(T: RetTy);
6006 unsigned MinElts = RetTy->getMinNumElements() / N;
6007 for (unsigned I = 0; I < N; I++) {
6008 Value *V = CI->getArgOperand(i: I);
6009 Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: V, Idx: I * MinElts);
6010 }
6011 NewCall = dyn_cast<CallInst>(Val: Ret);
6012 }
6013 break;
6014 }
6015
6016 case Intrinsic::arm_neon_bfdot:
6017 case Intrinsic::arm_neon_bfmmla:
6018 case Intrinsic::arm_neon_bfmlalb:
6019 case Intrinsic::arm_neon_bfmlalt:
6020 case Intrinsic::aarch64_neon_bfdot:
6021 case Intrinsic::aarch64_neon_bfmmla:
6022 case Intrinsic::aarch64_neon_bfmlalb:
6023 case Intrinsic::aarch64_neon_bfmlalt: {
6024 SmallVector<Value *, 3> Args;
6025 assert(CI->arg_size() == 3 &&
6026 "Mismatch between function args and call args");
6027 size_t OperandWidth =
6028 CI->getArgOperand(i: 1)->getType()->getPrimitiveSizeInBits();
6029 assert((OperandWidth == 64 || OperandWidth == 128) &&
6030 "Unexpected operand width");
6031 Type *NewTy = FixedVectorType::get(ElementType: Type::getBFloatTy(C), NumElts: OperandWidth / 16);
6032 auto Iter = CI->args().begin();
6033 Args.push_back(Elt: *Iter++);
6034 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
6035 Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
6036 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6037 break;
6038 }
6039
6040 case Intrinsic::bitreverse:
6041 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
6042 break;
6043
6044 case Intrinsic::ctlz:
6045 case Intrinsic::cttz: {
6046 if (CI->arg_size() != 1) {
6047 DefaultCase();
6048 return;
6049 }
6050
6051 NewCall =
6052 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), Builder.getFalse()});
6053 break;
6054 }
6055
6056 case Intrinsic::objectsize: {
6057 Value *NullIsUnknownSize =
6058 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(i: 2);
6059 Value *Dynamic =
6060 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(i: 3);
6061 NewCall = Builder.CreateCall(
6062 Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), NullIsUnknownSize, Dynamic});
6063 break;
6064 }
6065
6066 case Intrinsic::ctpop:
6067 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0)});
6068 break;
6069 case Intrinsic::dbg_value: {
6070 StringRef Name = F->getName();
6071 Name = Name.substr(Start: 5); // Strip llvm.
6072 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6073 if (Name.starts_with(Prefix: "dbg.addr")) {
6074 DIExpression *Expr = cast<DIExpression>(
6075 Val: cast<MetadataAsValue>(Val: CI->getArgOperand(i: 2))->getMetadata());
6076 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
6077 NewCall =
6078 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6079 MetadataAsValue::get(Context&: C, MD: Expr)});
6080 break;
6081 }
6082
6083 // Upgrade from the old version that had an extra offset argument.
6084 assert(CI->arg_size() == 4);
6085 // Drop nonzero offsets instead of attempting to upgrade them.
6086 if (auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: 1)))
6087 if (Offset->isNullValue()) {
6088 NewCall = Builder.CreateCall(
6089 Callee: NewFn,
6090 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 2), CI->getArgOperand(i: 3)});
6091 break;
6092 }
6093 CI->eraseFromParent();
6094 return;
6095 }
6096
6097 case Intrinsic::ptr_annotation:
6098 // Upgrade from versions that lacked the annotation attribute argument.
6099 if (CI->arg_size() != 4) {
6100 DefaultCase();
6101 return;
6102 }
6103
6104 // Create a new call with an added null annotation attribute argument.
6105 NewCall = Builder.CreateCall(
6106 Callee: NewFn,
6107 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
6108 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
6109 NewCall->takeName(V: CI);
6110 CI->replaceAllUsesWith(V: NewCall);
6111 CI->eraseFromParent();
6112 return;
6113
6114 case Intrinsic::var_annotation:
6115 // Upgrade from versions that lacked the annotation attribute argument.
6116 if (CI->arg_size() != 4) {
6117 DefaultCase();
6118 return;
6119 }
6120 // Create a new call with an added null annotation attribute argument.
6121 NewCall = Builder.CreateCall(
6122 Callee: NewFn,
6123 Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1), CI->getArgOperand(i: 2),
6124 CI->getArgOperand(i: 3), ConstantPointerNull::get(T: Builder.getPtrTy())});
6125 NewCall->takeName(V: CI);
6126 CI->replaceAllUsesWith(V: NewCall);
6127 CI->eraseFromParent();
6128 return;
6129
6130 case Intrinsic::riscv_aes32dsi:
6131 case Intrinsic::riscv_aes32dsmi:
6132 case Intrinsic::riscv_aes32esi:
6133 case Intrinsic::riscv_aes32esmi:
6134 case Intrinsic::riscv_sm4ks:
6135 case Intrinsic::riscv_sm4ed: {
6136 // The last argument to these intrinsics used to be i8 and changed to i32.
6137 // The type overload for sm4ks and sm4ed was removed.
6138 Value *Arg2 = CI->getArgOperand(i: 2);
6139 if (Arg2->getType()->isIntegerTy(BitWidth: 32) && !CI->getType()->isIntegerTy(BitWidth: 64))
6140 return;
6141
6142 Value *Arg0 = CI->getArgOperand(i: 0);
6143 Value *Arg1 = CI->getArgOperand(i: 1);
6144 if (CI->getType()->isIntegerTy(BitWidth: 64)) {
6145 Arg0 = Builder.CreateTrunc(V: Arg0, DestTy: Builder.getInt32Ty());
6146 Arg1 = Builder.CreateTrunc(V: Arg1, DestTy: Builder.getInt32Ty());
6147 }
6148
6149 Arg2 = ConstantInt::get(Ty: Type::getInt32Ty(C),
6150 V: cast<ConstantInt>(Val: Arg2)->getZExtValue());
6151
6152 NewCall = Builder.CreateCall(Callee: NewFn, Args: {Arg0, Arg1, Arg2});
6153 Value *Res = NewCall;
6154 if (Res->getType() != CI->getType())
6155 Res = Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
6156 NewCall->takeName(V: CI);
6157 CI->replaceAllUsesWith(V: Res);
6158 CI->eraseFromParent();
6159 return;
6160 }
6161 case Intrinsic::nvvm_mapa_shared_cluster: {
6162 // Create a new call with the correct address space.
6163 NewCall =
6164 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1)});
6165 Value *Res = NewCall;
6166 Res = Builder.CreateAddrSpaceCast(
6167 V: Res, DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED));
6168 NewCall->takeName(V: CI);
6169 CI->replaceAllUsesWith(V: Res);
6170 CI->eraseFromParent();
6171 return;
6172 }
6173 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6174 SmallVector<Value *, 4> Args(CI->args());
6175 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6176 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6177 Args[0] = Builder.CreateAddrSpaceCast(
6178 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6179
6180 // Append the missing trailing flag_valid_pattern (0 = disabled).
6181 Args.push_back(Elt: Builder.getInt32(C: 0));
6182
6183 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6184 NewCall->takeName(V: CI);
6185 CI->replaceAllUsesWith(V: NewCall);
6186 CI->eraseFromParent();
6187 return;
6188 }
6189 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6190 // (dst, mbar, src, size, ch, flag_ch)
6191 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6192 // i32 0 /* flag_valid_pattern=disabled */)
6193 SmallVector<Value *, 10> Args;
6194 for (unsigned I = 0; I < 4; ++I)
6195 Args.push_back(Elt: CI->getArgOperand(i: I));
6196 Args.push_back(Elt: Builder.getInt32(C: 0)); // ignore_bytes_left
6197 Args.push_back(Elt: Builder.getInt32(C: 0)); // ignore_bytes_right
6198 Args.push_back(Elt: CI->getArgOperand(i: 4)); // cache_hint
6199 Args.push_back(Elt: CI->getArgOperand(i: 5)); // flag_ch
6200 Args.push_back(Elt: Builder.getInt1(V: false)); // flag_oob
6201 Args.push_back(Elt: Builder.getInt32(C: 0)); // flag_valid_pattern
6202
6203 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6204 NewCall->takeName(V: CI);
6205 CI->replaceAllUsesWith(V: NewCall);
6206 CI->eraseFromParent();
6207 return;
6208 }
6209 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6210 // Create a new call with the correct address space.
6211 SmallVector<Value *, 4> Args(CI->args());
6212 Args[0] = Builder.CreateAddrSpaceCast(
6213 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6214
6215 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6216 NewCall->takeName(V: CI);
6217 CI->replaceAllUsesWith(V: NewCall);
6218 CI->eraseFromParent();
6219 return;
6220 }
6221 // clang-format off
6222#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6223 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6224 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6225#undef G2S_CLUSTER_CASE
6226 {
6227 SmallVector<Value *, 16> Args(CI->args());
6228 unsigned AS = CI->getArgOperand(i: 0)->getType()->getPointerAddressSpace();
6229 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6230 Args[0] = Builder.CreateAddrSpaceCast(
6231 V: Args[0], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6232
6233 // Append the missing trailing arguments with default values (cta_group,
6234 // flag_valid_pattern).
6235 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6236 Args.push_back(Elt: Builder.getInt32(C: 0));
6237
6238 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6239 NewCall->takeName(V: CI);
6240 CI->replaceAllUsesWith(V: NewCall);
6241 CI->eraseFromParent();
6242 return;
6243 }
6244
6245#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6246 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6247 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6248#undef G2S_CTA_CASE
6249 {
6250 SmallVector<Value *, 16> Args(CI->args());
6251 // Append the missing trailing flag_valid_pattern argument with default
6252 // value 0.
6253 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6254 "expected only the trailing flag_valid_pattern to be missing");
6255 Args.push_back(Elt: Builder.getInt32(C: 0));
6256
6257 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6258 NewCall->takeName(V: CI);
6259 CI->replaceAllUsesWith(V: NewCall);
6260 CI->eraseFromParent();
6261 return;
6262 }
6263#undef NVVM_TMA_G2S_MODES
6264 // clang-format on
6265
6266 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6267 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6268 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6269 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6270 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6271 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6272 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6273 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6274 StringRef Name = F->getName();
6275 Name.consume_front(Prefix: "llvm.nvvm.cp.async.bulk.tensor.reduce.");
6276 auto RedOp = getNVPTXTMAReductionOp(Name: Name.split(Separator: '.').first);
6277
6278 SmallVector<Value *, 16> Args(CI->args());
6279 Args.insert(I: Args.end() - 1, Elt: Builder.getInt32(C: *RedOp));
6280 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6281 break;
6282 }
6283 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6284 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6285 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6286 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6287 NewCall =
6288 Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6289 Builder.getFalse()});
6290 break;
6291 case Intrinsic::nvvm_mbarrier_init: {
6292 SmallVector<Value *, 3> Args(CI->args());
6293 // The .shared variant folded into the overloaded form without gaining an
6294 // operand, so only the pre-layout two-argument form needs one appended.
6295 if (Args.size() == 2)
6296 Args.push_back(Elt: Builder.getInt32(C: 0)); // layout = default(0)
6297 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6298 break;
6299 }
6300 case Intrinsic::riscv_sha256sig0:
6301 case Intrinsic::riscv_sha256sig1:
6302 case Intrinsic::riscv_sha256sum0:
6303 case Intrinsic::riscv_sha256sum1:
6304 case Intrinsic::riscv_sm3p0:
6305 case Intrinsic::riscv_sm3p1: {
6306 // The last argument to these intrinsics used to be i8 and changed to i32.
6307 // The type overload for sm4ks and sm4ed was removed.
6308 if (!CI->getType()->isIntegerTy(BitWidth: 64))
6309 return;
6310
6311 Value *Arg =
6312 Builder.CreateTrunc(V: CI->getArgOperand(i: 0), DestTy: Builder.getInt32Ty());
6313
6314 NewCall = Builder.CreateCall(Callee: NewFn, Args: Arg);
6315 Value *Res =
6316 Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /*isSigned*/ true);
6317 NewCall->takeName(V: CI);
6318 CI->replaceAllUsesWith(V: Res);
6319 CI->eraseFromParent();
6320 return;
6321 }
6322
6323 case Intrinsic::x86_xop_vfrcz_ss:
6324 case Intrinsic::x86_xop_vfrcz_sd:
6325 NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: 1)});
6326 break;
6327
6328 case Intrinsic::x86_xop_vpermil2pd:
6329 case Intrinsic::x86_xop_vpermil2ps:
6330 case Intrinsic::x86_xop_vpermil2pd_256:
6331 case Intrinsic::x86_xop_vpermil2ps_256: {
6332 SmallVector<Value *, 4> Args(CI->args());
6333 VectorType *FltIdxTy = cast<VectorType>(Val: Args[2]->getType());
6334 VectorType *IntIdxTy = VectorType::getInteger(VTy: FltIdxTy);
6335 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: IntIdxTy);
6336 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6337 break;
6338 }
6339
6340 case Intrinsic::x86_sse41_ptestc:
6341 case Intrinsic::x86_sse41_ptestz:
6342 case Intrinsic::x86_sse41_ptestnzc: {
6343 // The arguments for these intrinsics used to be v4f32, and changed
6344 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6345 // So, the only thing required is a bitcast for both arguments.
6346 // First, check the arguments have the old type.
6347 Value *Arg0 = CI->getArgOperand(i: 0);
6348 if (Arg0->getType() != FixedVectorType::get(ElementType: Type::getFloatTy(C), NumElts: 4))
6349 return;
6350
6351 // Old intrinsic, add bitcasts
6352 Value *Arg1 = CI->getArgOperand(i: 1);
6353
6354 auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: 2);
6355
6356 Value *BC0 = Builder.CreateBitCast(V: Arg0, DestTy: NewVecTy, Name: "cast");
6357 Value *BC1 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
6358
6359 NewCall = Builder.CreateCall(Callee: NewFn, Args: {BC0, BC1});
6360 break;
6361 }
6362
6363 case Intrinsic::x86_rdtscp: {
6364 // This used to take 1 arguments. If we have no arguments, it is already
6365 // upgraded.
6366 if (CI->getNumOperands() == 0)
6367 return;
6368
6369 NewCall = Builder.CreateCall(Callee: NewFn);
6370 // Extract the second result and store it.
6371 Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: 1);
6372 Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: 0), Align: Align(1));
6373 // Replace the original call result with the first result of the new call.
6374 Value *TSC = Builder.CreateExtractValue(Agg: NewCall, Idxs: 0);
6375
6376 NewCall->takeName(V: CI);
6377 CI->replaceAllUsesWith(V: TSC);
6378 CI->eraseFromParent();
6379 return;
6380 }
6381
6382 case Intrinsic::x86_sse41_insertps:
6383 case Intrinsic::x86_sse41_dppd:
6384 case Intrinsic::x86_sse41_dpps:
6385 case Intrinsic::x86_sse41_mpsadbw:
6386 case Intrinsic::x86_avx_dp_ps_256:
6387 case Intrinsic::x86_avx2_mpsadbw: {
6388 // Need to truncate the last argument from i32 to i8 -- this argument models
6389 // an inherently 8-bit immediate operand to these x86 instructions.
6390 SmallVector<Value *, 4> Args(CI->args());
6391
6392 // Replace the last argument with a trunc.
6393 Args.back() = Builder.CreateTrunc(V: Args.back(), DestTy: Type::getInt8Ty(C), Name: "trunc");
6394 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6395 break;
6396 }
6397
6398 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6399 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6400 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6401 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6402 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6403 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6404 SmallVector<Value *, 4> Args(CI->args());
6405 unsigned NumElts =
6406 cast<FixedVectorType>(Val: Args[0]->getType())->getNumElements();
6407 Args[3] = getX86MaskVec(Builder, Mask: Args[3], NumElts);
6408
6409 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6410 Value *Res = applyX86MaskOn1BitsVec(Builder, Vec: NewCall, Mask: nullptr);
6411
6412 NewCall->takeName(V: CI);
6413 CI->replaceAllUsesWith(V: Res);
6414 CI->eraseFromParent();
6415 return;
6416 }
6417
6418 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6419 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6420 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6421 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6422 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6423 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6424 SmallVector<Value *, 4> Args(CI->args());
6425 unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
6426 if (NewFn->getIntrinsicID() ==
6427 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6428 Args[1] = Builder.CreateBitCast(
6429 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6430
6431 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6432 Value *Res = Builder.CreateBitCast(
6433 V: NewCall, DestTy: FixedVectorType::get(ElementType: Builder.getInt16Ty(), NumElts));
6434
6435 NewCall->takeName(V: CI);
6436 CI->replaceAllUsesWith(V: Res);
6437 CI->eraseFromParent();
6438 return;
6439 }
6440 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6441 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6442 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6443 SmallVector<Value *, 4> Args(CI->args());
6444 unsigned NumElts =
6445 cast<FixedVectorType>(Val: CI->getType())->getNumElements() * 2;
6446 Args[1] = Builder.CreateBitCast(
6447 V: Args[1], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6448 Args[2] = Builder.CreateBitCast(
6449 V: Args[2], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
6450
6451 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6452 break;
6453 }
6454
6455 case Intrinsic::thread_pointer: {
6456 NewCall = Builder.CreateCall(Callee: NewFn, Args: {});
6457 break;
6458 }
6459
6460 case Intrinsic::memcpy:
6461 case Intrinsic::memmove:
6462 case Intrinsic::memset: {
6463 // We have to make sure that the call signature is what we're expecting.
6464 // We only want to change the old signatures by removing the alignment arg:
6465 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6466 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6467 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6468 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6469 // Note: i8*'s in the above can be any pointer type
6470 if (CI->arg_size() != 5) {
6471 DefaultCase();
6472 return;
6473 }
6474 // Remove alignment argument (3), and add alignment attributes to the
6475 // dest/src pointers.
6476 Value *Args[4] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6477 CI->getArgOperand(i: 2), CI->getArgOperand(i: 4)};
6478 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6479 AttributeList OldAttrs = CI->getAttributes();
6480 AttributeList NewAttrs = AttributeList::get(
6481 C, FnAttrs: OldAttrs.getFnAttrs(), RetAttrs: OldAttrs.getRetAttrs(),
6482 ArgAttrs: {OldAttrs.getParamAttrs(ArgNo: 0), OldAttrs.getParamAttrs(ArgNo: 1),
6483 OldAttrs.getParamAttrs(ArgNo: 2), OldAttrs.getParamAttrs(ArgNo: 4)});
6484 NewCall->setAttributes(NewAttrs);
6485 auto *MemCI = cast<MemIntrinsic>(Val: NewCall);
6486 // All mem intrinsics support dest alignment.
6487 const ConstantInt *Align = cast<ConstantInt>(Val: CI->getArgOperand(i: 3));
6488 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6489 // Memcpy/Memmove also support source alignment.
6490 if (auto *MTI = dyn_cast<MemTransferInst>(Val: MemCI))
6491 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6492 break;
6493 }
6494
6495 case Intrinsic::masked_load:
6496 case Intrinsic::masked_gather:
6497 case Intrinsic::masked_store:
6498 case Intrinsic::masked_scatter: {
6499 if (CI->arg_size() != 4) {
6500 DefaultCase();
6501 return;
6502 }
6503
6504 auto GetMaybeAlign = [](Value *Op) {
6505 if (auto *CI = dyn_cast<ConstantInt>(Val: Op)) {
6506 uint64_t Val = CI->getZExtValue();
6507 if (Val == 0)
6508 return MaybeAlign();
6509 if (isPowerOf2_64(Value: Val))
6510 return MaybeAlign(Val);
6511 }
6512 reportFatalUsageError(reason: "Invalid alignment argument");
6513 };
6514 auto GetAlign = [&](Value *Op) {
6515 MaybeAlign Align = GetMaybeAlign(Op);
6516 if (Align)
6517 return *Align;
6518 reportFatalUsageError(reason: "Invalid zero alignment argument");
6519 };
6520
6521 const DataLayout &DL = CI->getDataLayout();
6522 switch (NewFn->getIntrinsicID()) {
6523 case Intrinsic::masked_load:
6524 NewCall = Builder.CreateMaskedLoad(
6525 Ty: CI->getType(), Ptr: CI->getArgOperand(i: 0), Alignment: GetAlign(CI->getArgOperand(i: 1)),
6526 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6527 break;
6528 case Intrinsic::masked_gather:
6529 NewCall = Builder.CreateMaskedGather(
6530 Ty: CI->getType(), Ptrs: CI->getArgOperand(i: 0),
6531 Alignment: DL.getValueOrABITypeAlignment(Alignment: GetMaybeAlign(CI->getArgOperand(i: 1)),
6532 Ty: CI->getType()->getScalarType()),
6533 Mask: CI->getArgOperand(i: 2), PassThru: CI->getArgOperand(i: 3));
6534 break;
6535 case Intrinsic::masked_store:
6536 NewCall = Builder.CreateMaskedStore(
6537 Val: CI->getArgOperand(i: 0), Ptr: CI->getArgOperand(i: 1),
6538 Alignment: GetAlign(CI->getArgOperand(i: 2)), Mask: CI->getArgOperand(i: 3));
6539 break;
6540 case Intrinsic::masked_scatter:
6541 NewCall = Builder.CreateMaskedScatter(
6542 Val: CI->getArgOperand(i: 0), Ptrs: CI->getArgOperand(i: 1),
6543 Alignment: DL.getValueOrABITypeAlignment(
6544 Alignment: GetMaybeAlign(CI->getArgOperand(i: 2)),
6545 Ty: CI->getArgOperand(i: 0)->getType()->getScalarType()),
6546 Mask: CI->getArgOperand(i: 3));
6547 break;
6548 default:
6549 llvm_unreachable("Unexpected intrinsic ID");
6550 }
6551 // Previous metadata is still valid.
6552 NewCall->copyMetadata(SrcInst: *CI);
6553 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6554 break;
6555 }
6556
6557 case Intrinsic::lifetime_start:
6558 case Intrinsic::lifetime_end: {
6559 if (CI->arg_size() != 2) {
6560 DefaultCase();
6561 return;
6562 }
6563
6564 Value *Ptr = CI->getArgOperand(i: 1);
6565 // Try to strip pointer casts, such that the lifetime works on an alloca.
6566 Ptr = Ptr->stripPointerCasts();
6567 if (isa<AllocaInst>(Val: Ptr)) {
6568 // Don't use NewFn, as we might have looked through an addrspacecast.
6569 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6570 NewCall = Builder.CreateLifetimeStart(Ptr);
6571 else
6572 NewCall = Builder.CreateLifetimeEnd(Ptr);
6573 break;
6574 }
6575
6576 // Otherwise remove the lifetime marker.
6577 CI->eraseFromParent();
6578 return;
6579 }
6580
6581 case Intrinsic::x86_avx512_vpdpbusd_128:
6582 case Intrinsic::x86_avx512_vpdpbusd_256:
6583 case Intrinsic::x86_avx512_vpdpbusd_512:
6584 case Intrinsic::x86_avx512_vpdpbusds_128:
6585 case Intrinsic::x86_avx512_vpdpbusds_256:
6586 case Intrinsic::x86_avx512_vpdpbusds_512:
6587 case Intrinsic::x86_avx2_vpdpbssd_128:
6588 case Intrinsic::x86_avx2_vpdpbssd_256:
6589 case Intrinsic::x86_avx10_vpdpbssd_512:
6590 case Intrinsic::x86_avx2_vpdpbssds_128:
6591 case Intrinsic::x86_avx2_vpdpbssds_256:
6592 case Intrinsic::x86_avx10_vpdpbssds_512:
6593 case Intrinsic::x86_avx2_vpdpbsud_128:
6594 case Intrinsic::x86_avx2_vpdpbsud_256:
6595 case Intrinsic::x86_avx10_vpdpbsud_512:
6596 case Intrinsic::x86_avx2_vpdpbsuds_128:
6597 case Intrinsic::x86_avx2_vpdpbsuds_256:
6598 case Intrinsic::x86_avx10_vpdpbsuds_512:
6599 case Intrinsic::x86_avx2_vpdpbuud_128:
6600 case Intrinsic::x86_avx2_vpdpbuud_256:
6601 case Intrinsic::x86_avx10_vpdpbuud_512:
6602 case Intrinsic::x86_avx2_vpdpbuuds_128:
6603 case Intrinsic::x86_avx2_vpdpbuuds_256:
6604 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6605 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6606 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6607 CI->getArgOperand(i: 2)};
6608 Type *NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: NumElts, Scalable: false);
6609 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6610 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6611
6612 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6613 break;
6614 }
6615 case Intrinsic::x86_avx512_vpdpwssd_128:
6616 case Intrinsic::x86_avx512_vpdpwssd_256:
6617 case Intrinsic::x86_avx512_vpdpwssd_512:
6618 case Intrinsic::x86_avx512_vpdpwssds_128:
6619 case Intrinsic::x86_avx512_vpdpwssds_256:
6620 case Intrinsic::x86_avx512_vpdpwssds_512:
6621 case Intrinsic::x86_avx2_vpdpwsud_128:
6622 case Intrinsic::x86_avx2_vpdpwsud_256:
6623 case Intrinsic::x86_avx10_vpdpwsud_512:
6624 case Intrinsic::x86_avx2_vpdpwsuds_128:
6625 case Intrinsic::x86_avx2_vpdpwsuds_256:
6626 case Intrinsic::x86_avx10_vpdpwsuds_512:
6627 case Intrinsic::x86_avx2_vpdpwusd_128:
6628 case Intrinsic::x86_avx2_vpdpwusd_256:
6629 case Intrinsic::x86_avx10_vpdpwusd_512:
6630 case Intrinsic::x86_avx2_vpdpwusds_128:
6631 case Intrinsic::x86_avx2_vpdpwusds_256:
6632 case Intrinsic::x86_avx10_vpdpwusds_512:
6633 case Intrinsic::x86_avx2_vpdpwuud_128:
6634 case Intrinsic::x86_avx2_vpdpwuud_256:
6635 case Intrinsic::x86_avx10_vpdpwuud_512:
6636 case Intrinsic::x86_avx2_vpdpwuuds_128:
6637 case Intrinsic::x86_avx2_vpdpwuuds_256:
6638 case Intrinsic::x86_avx10_vpdpwuuds_512:
6639 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6640 Value *Args[] = {CI->getArgOperand(i: 0), CI->getArgOperand(i: 1),
6641 CI->getArgOperand(i: 2)};
6642 Type *NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: NumElts, Scalable: false);
6643 Args[1] = Builder.CreateBitCast(V: Args[1], DestTy: NewArgType);
6644 Args[2] = Builder.CreateBitCast(V: Args[2], DestTy: NewArgType);
6645
6646 NewCall = Builder.CreateCall(Callee: NewFn, Args);
6647 break;
6648 }
6649 assert(NewCall && "Should have either set this variable or returned through "
6650 "the default case");
6651 NewCall->takeName(V: CI);
6652 CI->replaceAllUsesWith(V: NewCall);
6653 CI->eraseFromParent();
6654}
6655
6656void llvm::UpgradeCallsToIntrinsic(Function *F) {
6657 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6658
6659 // Check if this function should be upgraded and get the replacement function
6660 // if there is one.
6661 Function *NewFn;
6662 if (UpgradeIntrinsicFunction(F, NewFn)) {
6663 // Replace all users of the old function with the new function or new
6664 // instructions. This is not a range loop because the call is deleted.
6665 for (User *U : make_early_inc_range(Range: F->users()))
6666 if (CallBase *CB = dyn_cast<CallBase>(Val: U))
6667 UpgradeIntrinsicCall(CI: CB, NewFn);
6668
6669 // Remove old function, no longer used, from the module.
6670 if (F != NewFn)
6671 F->eraseFromParent();
6672 }
6673}
6674
6675MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6676 const unsigned NumOperands = MD.getNumOperands();
6677 if (NumOperands == 0)
6678 return &MD; // Invalid, punt to a verifier error.
6679
6680 // Check if the tag uses struct-path aware TBAA format.
6681 if (isa<MDNode>(Val: MD.getOperand(I: 0)) && NumOperands >= 3)
6682 return &MD;
6683
6684 auto &Context = MD.getContext();
6685 if (NumOperands == 3) {
6686 Metadata *Elts[] = {MD.getOperand(I: 0), MD.getOperand(I: 1)};
6687 MDNode *ScalarType = MDNode::get(Context, MDs: Elts);
6688 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6689 Metadata *Elts2[] = {ScalarType, ScalarType,
6690 ConstantAsMetadata::get(
6691 C: Constant::getNullValue(Ty: Type::getInt64Ty(C&: Context))),
6692 MD.getOperand(I: 2)};
6693 return MDNode::get(Context, MDs: Elts2);
6694 }
6695 // Create a MDNode <MD, MD, offset 0>
6696 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(C: Constant::getNullValue(
6697 Ty: Type::getInt64Ty(C&: Context)))};
6698 return MDNode::get(Context, MDs: Elts);
6699}
6700
6701MDNode *llvm::UpgradeTBAAStructNode(MDNode &MD) {
6702 // !tbaa.struct is a list of (offset, size, tag) triples. Upgrade any
6703 // old-style scalar field tag to struct-path form via UpgradeTBAANode.
6704 unsigned NumOperands = MD.getNumOperands();
6705 if (NumOperands == 0 || NumOperands % 3 != 0)
6706 return &MD; // Malformed; leave it for the verifier to reject.
6707
6708 SmallVector<Metadata *, 12> Elts(MD.op_begin(), MD.op_end());
6709 bool Changed = false;
6710 for (unsigned I = 2; I < NumOperands; I += 3) {
6711 auto *Tag = dyn_cast_or_null<MDNode>(Val: Elts[I]);
6712 if (!Tag)
6713 continue;
6714 MDNode *Upgraded = UpgradeTBAANode(MD&: *Tag);
6715 if (Upgraded == Tag)
6716 continue;
6717 Elts[I] = Upgraded;
6718 Changed = true;
6719 }
6720 return Changed ? MDNode::get(Context&: MD.getContext(), MDs: Elts) : &MD;
6721}
6722
6723Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6724 Instruction *&Temp) {
6725 if (Opc != Instruction::BitCast)
6726 return nullptr;
6727
6728 Temp = nullptr;
6729 Type *SrcTy = V->getType();
6730 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6731 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6732 LLVMContext &Context = V->getContext();
6733
6734 // We have no information about target data layout, so we assume that
6735 // the maximum pointer size is 64bit.
6736 Type *MidTy = Type::getInt64Ty(C&: Context);
6737 Temp = CastInst::Create(Instruction::PtrToInt, S: V, Ty: MidTy);
6738
6739 return CastInst::Create(Instruction::IntToPtr, S: Temp, Ty: DestTy);
6740 }
6741
6742 return nullptr;
6743}
6744
6745Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6746 if (Opc != Instruction::BitCast)
6747 return nullptr;
6748
6749 Type *SrcTy = C->getType();
6750 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6751 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6752 LLVMContext &Context = C->getContext();
6753
6754 // We have no information about target data layout, so we assume that
6755 // the maximum pointer size is 64bit.
6756 Type *MidTy = Type::getInt64Ty(C&: Context);
6757
6758 return ConstantExpr::getIntToPtr(C: ConstantExpr::getPtrToInt(C, Ty: MidTy),
6759 Ty: DestTy);
6760 }
6761
6762 return nullptr;
6763}
6764
6765static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6766 if (Flag.getNumOperands() < 3)
6767 return std::nullopt;
6768 if (MDString *Name = dyn_cast_or_null<MDString>(Val: Flag.getOperand(I: 1)))
6769 return Name->getString();
6770 return std::nullopt;
6771}
6772
6773/// Check the debug info version number, if it is out-dated, drop the debug
6774/// info. Return true if module is modified.
6775bool llvm::UpgradeDebugInfo(Module &M) {
6776 if (!getIROptions(C: M.getContext()).auto_upgrade_debug_info)
6777 return false;
6778
6779 llvm::TimeTraceScope timeScope("Upgrade debug info");
6780 // We need to get metadata before the module is verified (i.e., getModuleFlag
6781 // makes assumptions that we haven't verified yet). Carefully extract the flag
6782 // from the metadata.
6783 unsigned Version = 0;
6784 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6785 auto OpIt = find_if(Range: ModFlags->operands(), P: [](const MDNode *Flag) {
6786 if (auto Name = getModuleFlagNameSafely(Flag: *Flag))
6787 return *Name == "Debug Info Version";
6788 return false;
6789 });
6790 if (OpIt != ModFlags->op_end()) {
6791 const MDOperand &ValOp = (*OpIt)->getOperand(I: 2);
6792 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(MD: ValOp))
6793 Version = CI->getZExtValue();
6794 }
6795 }
6796
6797 if (Version == DEBUG_METADATA_VERSION) {
6798 bool BrokenDebugInfo = false;
6799 if (verifyModule(M, OS: &llvm::errs(), BrokenDebugInfo: &BrokenDebugInfo))
6800 report_fatal_error(reason: "Broken module found, compilation aborted!");
6801 if (!BrokenDebugInfo)
6802 // Everything is ok.
6803 return false;
6804 else {
6805 // Diagnose malformed debug info.
6806 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6807 M.getContext().diagnose(DI: Diag);
6808 }
6809 }
6810 bool Modified = StripDebugInfo(M);
6811 if (Modified && Version != DEBUG_METADATA_VERSION) {
6812 // Diagnose a version mismatch.
6813 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6814 M.getContext().diagnose(DI: DiagVersion);
6815 }
6816 return Modified;
6817}
6818
6819static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6820 GlobalValue *GV, const Metadata *V) {
6821 Function *F = cast<Function>(Val: GV);
6822
6823 constexpr StringLiteral DefaultValue = "1";
6824 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6825 unsigned Length = 0;
6826
6827 if (F->hasFnAttribute(Kind: Attr)) {
6828 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6829 // parse these elements placing them into Vect3
6830 StringRef S = F->getFnAttribute(Kind: Attr).getValueAsString();
6831 for (; Length < 3 && !S.empty(); Length++) {
6832 auto [Part, Rest] = S.split(Separator: ',');
6833 Vect3[Length] = Part.trim();
6834 S = Rest;
6835 }
6836 }
6837
6838 const unsigned Dim = DimC - 'x';
6839 assert(Dim < 3 && "Unexpected dim char");
6840
6841 const uint64_t VInt = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6842
6843 // local variable required for StringRef in Vect3 to point to.
6844 const std::string VStr = llvm::utostr(X: VInt);
6845 Vect3[Dim] = VStr;
6846 Length = std::max(a: Length, b: Dim + 1);
6847
6848 const std::string NewAttr = llvm::join(R: ArrayRef(Vect3, Length), Separator: ",");
6849 F->addFnAttr(Kind: Attr, Val: NewAttr);
6850}
6851
6852static inline bool isXYZ(StringRef S) {
6853 return S == "x" || S == "y" || S == "z";
6854}
6855
6856bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6857 const Metadata *V) {
6858 if (K == "kernel") {
6859 if (!mdconst::extract<ConstantInt>(MD&: V)->isZero())
6860 cast<Function>(Val: GV)->setCallingConv(CallingConv::PTX_Kernel);
6861 return true;
6862 }
6863 if (K == "align") {
6864 // V is a bitfeild specifying two 16-bit values. The alignment value is
6865 // specfied in low 16-bits, The index is specified in the high bits. For the
6866 // index, 0 indicates the return value while higher values correspond to
6867 // each parameter (idx = param + 1).
6868 const uint64_t AlignIdxValuePair =
6869 mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6870 const unsigned Idx = (AlignIdxValuePair >> 16);
6871 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6872 cast<Function>(Val: GV)->addAttributeAtIndex(
6873 i: Idx, Attr: Attribute::getWithStackAlignment(Context&: GV->getContext(), Alignment: StackAlign));
6874 return true;
6875 }
6876 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6877 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6878 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: CV));
6879 return true;
6880 }
6881 if (K == "minctasm") {
6882 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6883 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MinCTASm, Val: llvm::utostr(X: CV));
6884 return true;
6885 }
6886 if (K == "maxnreg") {
6887 const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6888 cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxNReg, Val: llvm::utostr(X: CV));
6889 return true;
6890 }
6891 if (K.consume_front(Prefix: "maxntid") && isXYZ(S: K)) {
6892 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::MaxNTID, DimC: K[0], GV, V);
6893 return true;
6894 }
6895 if (K.consume_front(Prefix: "reqntid") && isXYZ(S: K)) {
6896 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ReqNTID, DimC: K[0], GV, V);
6897 return true;
6898 }
6899 if (K.consume_front(Prefix: "cluster_dim_") && isXYZ(S: K)) {
6900 upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ClusterDim, DimC: K[0], GV, V);
6901 return true;
6902 }
6903 if (K == "grid_constant") {
6904 const auto Attr = Attribute::get(Context&: GV->getContext(), Kind: NVVMAttr::GridConstant);
6905 for (const auto &Op : cast<MDNode>(Val: V)->operands()) {
6906 // For some reason, the index is 1-based in the metadata. Good thing we're
6907 // able to auto-upgrade it!
6908 const auto Index = mdconst::extract<ConstantInt>(MD: Op)->getZExtValue() - 1;
6909 cast<Function>(Val: GV)->addParamAttr(ArgNo: Index, Attr);
6910 }
6911 return true;
6912 }
6913
6914 return false;
6915}
6916
6917void llvm::UpgradeNVVMAnnotations(Module &M) {
6918 NamedMDNode *NamedMD = M.getNamedMetadata(Name: "nvvm.annotations");
6919 if (!NamedMD)
6920 return;
6921
6922 SmallVector<MDNode *, 8> NewNodes;
6923 SmallPtrSet<const MDNode *, 8> SeenNodes;
6924 for (MDNode *MD : NamedMD->operands()) {
6925 if (!SeenNodes.insert(Ptr: MD).second)
6926 continue;
6927
6928 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD: MD->getOperand(I: 0));
6929 if (!GV)
6930 continue;
6931
6932 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6933
6934 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(I: 0)};
6935 // Each nvvm.annotations metadata entry will be of the following form:
6936 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6937 // start index = 1, to skip the global variable key
6938 // increment = 2, to skip the value for each property-value pairs
6939 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6940 MDString *K = cast<MDString>(Val: MD->getOperand(I: j));
6941 const MDOperand &V = MD->getOperand(I: j + 1);
6942 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K: K->getString(), V);
6943 if (!Upgraded)
6944 NewOperands.append(IL: {K, V});
6945 }
6946
6947 if (NewOperands.size() > 1)
6948 NewNodes.push_back(Elt: MDNode::get(Context&: M.getContext(), MDs: NewOperands));
6949 }
6950
6951 NamedMD->clearOperands();
6952 for (MDNode *N : NewNodes)
6953 NamedMD->addOperand(M: N);
6954}
6955
6956/// This checks for objc retain release marker which should be upgraded. It
6957/// returns true if module is modified.
6958static bool upgradeRetainReleaseMarker(Module &M) {
6959 bool Changed = false;
6960 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6961 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(Name: MarkerKey);
6962 if (ModRetainReleaseMarker) {
6963 MDNode *Op = ModRetainReleaseMarker->getOperand(i: 0);
6964 if (Op) {
6965 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 0));
6966 if (ID) {
6967 SmallVector<StringRef, 4> ValueComp;
6968 ID->getString().split(A&: ValueComp, Separator: "#");
6969 if (ValueComp.size() == 2) {
6970 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6971 ID = MDString::get(Context&: M.getContext(), Str: NewValue);
6972 }
6973 M.addModuleFlag(Behavior: Module::Error, Key: MarkerKey, Val: ID);
6974 M.eraseNamedMetadata(NMD: ModRetainReleaseMarker);
6975 Changed = true;
6976 }
6977 }
6978 }
6979 return Changed;
6980}
6981
6982void llvm::UpgradeARCRuntime(Module &M) {
6983 // This lambda converts normal function calls to ARC runtime functions to
6984 // intrinsic calls.
6985 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6986 llvm::Intrinsic::ID IntrinsicFunc) {
6987 Function *Fn = M.getFunction(Name: OldFunc);
6988
6989 if (!Fn)
6990 return;
6991
6992 Function *NewFn =
6993 llvm::Intrinsic::getOrInsertDeclaration(M: &M, id: IntrinsicFunc);
6994
6995 for (User *U : make_early_inc_range(Range: Fn->users())) {
6996 CallInst *CI = dyn_cast<CallInst>(Val: U);
6997 if (!CI || CI->getCalledFunction() != Fn)
6998 continue;
6999
7000 IRBuilder<> Builder(CI->getIterator());
7001 FunctionType *NewFuncTy = NewFn->getFunctionType();
7002 SmallVector<Value *, 2> Args;
7003
7004 // Don't upgrade the intrinsic if it's not valid to bitcast the return
7005 // value to the return type of the old function.
7006 if (NewFuncTy->getReturnType() != CI->getType() &&
7007 !CastInst::castIsValid(op: Instruction::BitCast, S: CI,
7008 DstTy: NewFuncTy->getReturnType()))
7009 continue;
7010
7011 bool InvalidCast = false;
7012
7013 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
7014 Value *Arg = CI->getArgOperand(i: I);
7015
7016 // Bitcast argument to the parameter type of the new function if it's
7017 // not a variadic argument.
7018 if (I < NewFuncTy->getNumParams()) {
7019 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
7020 // to the parameter type of the new function.
7021 if (!CastInst::castIsValid(op: Instruction::BitCast, S: Arg,
7022 DstTy: NewFuncTy->getParamType(i: I))) {
7023 InvalidCast = true;
7024 break;
7025 }
7026 Arg = Builder.CreateBitCast(V: Arg, DestTy: NewFuncTy->getParamType(i: I));
7027 }
7028 Args.push_back(Elt: Arg);
7029 }
7030
7031 if (InvalidCast)
7032 continue;
7033
7034 // Create a call instruction that calls the new function.
7035 CallInst *NewCall = Builder.CreateCall(FTy: NewFuncTy, Callee: NewFn, Args);
7036 NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
7037 NewCall->takeName(V: CI);
7038
7039 // Bitcast the return value back to the type of the old call.
7040 Value *NewRetVal = Builder.CreateBitCast(V: NewCall, DestTy: CI->getType());
7041
7042 if (!CI->use_empty())
7043 CI->replaceAllUsesWith(V: NewRetVal);
7044 CI->eraseFromParent();
7045 }
7046
7047 if (Fn->use_empty())
7048 Fn->eraseFromParent();
7049 };
7050
7051 // Unconditionally convert a call to "clang.arc.use" to a call to
7052 // "llvm.objc.clang.arc.use".
7053 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7054
7055 // Upgrade the retain release marker. If there is no need to upgrade
7056 // the marker, that means either the module is already new enough to contain
7057 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7058 if (!upgradeRetainReleaseMarker(M))
7059 return;
7060
7061 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7062 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7063 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7064 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7065 {"objc_autoreleaseReturnValue",
7066 llvm::Intrinsic::objc_autoreleaseReturnValue},
7067 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7068 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7069 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7070 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7071 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7072 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7073 {"objc_release", llvm::Intrinsic::objc_release},
7074 {"objc_retain", llvm::Intrinsic::objc_retain},
7075 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7076 {"objc_retainAutoreleaseReturnValue",
7077 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7078 {"objc_retainAutoreleasedReturnValue",
7079 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7080 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7081 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7082 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7083 {"objc_unsafeClaimAutoreleasedReturnValue",
7084 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7085 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7086 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7087 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7088 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7089 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7090 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7091 {"objc_arc_annotation_topdown_bbstart",
7092 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7093 {"objc_arc_annotation_topdown_bbend",
7094 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7095 {"objc_arc_annotation_bottomup_bbstart",
7096 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7097 {"objc_arc_annotation_bottomup_bbend",
7098 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7099
7100 for (auto &I : RuntimeFuncs)
7101 UpgradeToIntrinsic(I.first, I.second);
7102}
7103
7104// Upgrade the way signing of pointers to init/fini functions is described.
7105//
7106// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7107// constants, if signing was requested. After the upgrade, these arrays contain
7108// plain function pointers and the desired signing schema is described via a
7109// pair of module flags.
7110//
7111// Note that the upgrade is only performed if all elements of *both* arrays
7112// agree on a common signing schema.
7113static bool upgradePtrauthInitFiniArrays(Module &M) {
7114 // As we cannot always decide whether the particular module should have
7115 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7116 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7117 // returns as soon as it spots any non-signed init/fini pointer: either we
7118 // should request non-signed pointers (safe to omit both flags) or there is
7119 // no common schema (and thus we do not modify anything).
7120 //
7121 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7122 // or whether we should request address diversity in addition to the basic
7123 // constant diversity. There is no value representing "decided not to sign"
7124 // for the reasons explained above.
7125 std::optional<bool> UseAddressDisc;
7126
7127 // Do not attempt upgrading if the new module flags already exist.
7128 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7129 for (const MDNode *Flag : ModFlags->operands()) {
7130 std::optional<StringRef> Name = getModuleFlagNameSafely(Flag: *Flag);
7131 if (Name && (*Name == "ptrauth-init-fini" ||
7132 *Name == "ptrauth-init-fini-address-discrimination"))
7133 return false;
7134 }
7135 }
7136
7137 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7138 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7139 constexpr unsigned ExpectedAddressMarker = 1;
7140
7141 auto *CPA = dyn_cast<ConstantPtrAuth>(Val: CV);
7142 if (!CPA || !CPA->getDiscriminator()->equalsInt(V: ExpectedConstDisc))
7143 return nullptr; // Nothing to upgrade or unknown pattern found.
7144
7145 bool HasAddressDisc;
7146 if (!CPA->hasAddressDiscriminator())
7147 HasAddressDisc = false;
7148 else if (CPA->hasSpecialAddressDiscriminator(Value: ExpectedAddressMarker))
7149 HasAddressDisc = true;
7150 else
7151 return nullptr; // Unknown pattern.
7152
7153 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7154 return nullptr; // Disagreement with the decided mode.
7155
7156 UseAddressDisc = HasAddressDisc;
7157 return CPA->getPointer();
7158 };
7159
7160 // Do not apply any changes until we know the upgrade is non-ambiguous.
7161 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7162 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7163
7164 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7165 auto *GV = dyn_cast_if_present<GlobalVariable>(Val: M.getNamedValue(Name));
7166 if (!GV || !GV->hasInitializer())
7167 continue; // Skip, but it is okay to upgrade the other variable.
7168
7169 auto *OldStructorsArray = dyn_cast<ConstantArray>(Val: GV->getInitializer());
7170 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7171 return false;
7172
7173 std::vector<Constant *> NewStructors;
7174 NewStructors.reserve(n: OldStructorsArray->getNumOperands());
7175
7176 for (Use &U : OldStructorsArray->operands()) {
7177 ConstantStruct *Structor = dyn_cast<ConstantStruct>(Val: U.get());
7178 if (!Structor || Structor->getNumOperands() != 3)
7179 return false;
7180
7181 Constant *Prio = Structor->getOperand(i_nocapture: 0);
7182 Constant *Func = Structor->getOperand(i_nocapture: 1);
7183 Constant *Arg = Structor->getOperand(i_nocapture: 2);
7184
7185 Func = UpgradeSinglePointer(Func);
7186 if (!Func)
7187 return false;
7188
7189 NewStructors.push_back(
7190 x: ConstantStruct::get(T: Structor->getType(), V: {Prio, Func, Arg}));
7191 }
7192
7193 Constant *NewInit =
7194 ConstantArray::get(T: OldStructorsArray->getType(), V: NewStructors);
7195 GlobalArraysToUpgrade.emplace_back(Args&: GV, Args&: NewInit);
7196 }
7197
7198 if (GlobalArraysToUpgrade.empty())
7199 return false;
7200 assert(UseAddressDisc.has_value());
7201
7202 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7203 GV->setInitializer(NewInit);
7204
7205 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini", Val: 1);
7206 M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini-address-discrimination",
7207 Val: *UseAddressDisc);
7208
7209 return true;
7210}
7211
7212bool llvm::UpgradeModuleFlags(Module &M) {
7213 bool Changed = false;
7214 Changed |= upgradePtrauthInitFiniArrays(M);
7215
7216 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7217 if (!ModFlags)
7218 return Changed;
7219
7220 bool HasObjCFlag = false, HasClassProperties = false;
7221 bool HasSwiftVersionFlag = false;
7222 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7223 uint32_t SwiftABIVersion;
7224 auto Int8Ty = Type::getInt8Ty(C&: M.getContext());
7225 auto Int32Ty = Type::getInt32Ty(C&: M.getContext());
7226
7227 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7228 MDNode *Op = ModFlags->getOperand(i: I);
7229 if (Op->getNumOperands() != 3)
7230 continue;
7231 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
7232 if (!ID)
7233 continue;
7234 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7235 Metadata *Ops[3] = {ConstantAsMetadata::get(C: ConstantInt::get(
7236 Ty: Type::getInt32Ty(C&: M.getContext()), V: B)),
7237 MDString::get(Context&: M.getContext(), Str: ID->getString()),
7238 Op->getOperand(I: 2)};
7239 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7240 Changed = true;
7241 };
7242
7243 if (ID->getString() == "Objective-C Image Info Version")
7244 HasObjCFlag = true;
7245 if (ID->getString() == "Objective-C Class Properties")
7246 HasClassProperties = true;
7247 // Upgrade PIC from Error/Max to Min.
7248 if (ID->getString() == "PIC Level") {
7249 if (auto *Behavior =
7250 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
7251 uint64_t V = Behavior->getLimitedValue();
7252 if (V == Module::Error || V == Module::Max)
7253 SetBehavior(Module::Min);
7254 }
7255 }
7256 // Upgrade "PIE Level" from Error to Max.
7257 if (ID->getString() == "PIE Level")
7258 if (auto *Behavior =
7259 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0)))
7260 if (Behavior->getLimitedValue() == Module::Error)
7261 SetBehavior(Module::Max);
7262
7263 // Upgrade branch protection and return address signing module flags. The
7264 // module flag behavior for these fields were Error and now they are Min.
7265 // The one exception is "sign-return-address-harden".
7266 if (ID->getString() == "branch-target-enforcement" ||
7267 (ID->getString().starts_with(Prefix: "sign-return-address") &&
7268 ID->getString() != "sign-return-address-harden")) {
7269 if (auto *Behavior =
7270 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 0))) {
7271 if (Behavior->getLimitedValue() == Module::Error) {
7272 Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
7273 Metadata *Ops[3] = {
7274 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Module::Min)),
7275 Op->getOperand(I: 1), Op->getOperand(I: 2)};
7276 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7277 Changed = true;
7278 }
7279 }
7280 }
7281
7282 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7283 // section name so that llvm-lto will not complain about mismatching
7284 // module flags that is functionally the same.
7285 if (ID->getString() == "Objective-C Image Info Section") {
7286 if (auto *Value = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2))) {
7287 SmallVector<StringRef, 4> ValueComp;
7288 Value->getString().split(A&: ValueComp, Separator: " ");
7289 if (ValueComp.size() != 1) {
7290 std::string NewValue;
7291 for (auto &S : ValueComp)
7292 NewValue += S.str();
7293 Metadata *Ops[3] = {Op->getOperand(I: 0), Op->getOperand(I: 1),
7294 MDString::get(Context&: M.getContext(), Str: NewValue)};
7295 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7296 Changed = true;
7297 }
7298 }
7299 }
7300
7301 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7302 // If the higher bits are set, it adds new module flag for swift info.
7303 if (ID->getString() == "Objective-C Garbage Collection") {
7304 auto Md = dyn_cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2));
7305 if (Md) {
7306 assert(Md->getValue() && "Expected non-empty metadata");
7307 auto Type = Md->getValue()->getType();
7308 if (Type == Int8Ty)
7309 continue;
7310 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7311 if ((Val & 0xff) != Val) {
7312 HasSwiftVersionFlag = true;
7313 SwiftABIVersion = (Val & 0xff00) >> 8;
7314 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7315 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7316 }
7317 Metadata *Ops[3] = {
7318 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty,V: Module::Error)),
7319 Op->getOperand(I: 1),
7320 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int8Ty,V: Val & 0xff))};
7321 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7322 Changed = true;
7323 }
7324 }
7325
7326 if (ID->getString() == "amdgpu_code_object_version") {
7327 Metadata *Ops[3] = {
7328 Op->getOperand(I: 0),
7329 MDString::get(Context&: M.getContext(), Str: "amdhsa_code_object_version"),
7330 Op->getOperand(I: 2)};
7331 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7332 Changed = true;
7333 }
7334
7335 // clang/PowerPC used to use "float-abi" to describe the long double format;
7336 // it has been renamed to "long-double-type", with its values changed to the
7337 // corresponding IR floating-point type names.
7338 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7339 StringRef Format;
7340 if (auto *S = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 2)))
7341 Format = S->getString();
7342
7343 // The "float-abi" key is now reserved for the target-independent
7344 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7345 // (including unrecognized ones, which were never valid) to the default.
7346 if (!FloatABI::parseABIType(S: Format)) {
7347 LongDoubleFormat NewFormat =
7348 StringSwitch<LongDoubleFormat>(Format)
7349 .Case(S: "ieeequad", Value: LongDoubleFormat::IEEEquad)
7350 .Case(S: "ieeedouble", Value: LongDoubleFormat::IEEEdouble)
7351 .Default(Value: LongDoubleFormat::PPCDoubleDouble);
7352 Metadata *Ops[3] = {
7353 Op->getOperand(I: 0),
7354 MDString::get(Context&: M.getContext(), Str: "long-double-type"),
7355 MDString::get(Context&: M.getContext(), Str: getLongDoubleFormatName(Format: NewFormat))};
7356 ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
7357 Changed = true;
7358 }
7359 }
7360 }
7361
7362 // "Objective-C Class Properties" is recently added for Objective-C. We
7363 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7364 // flag of value 0, so we can correclty downgrade this flag when trying to
7365 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7366 // this module flag.
7367 if (HasObjCFlag && !HasClassProperties) {
7368 M.addModuleFlag(Behavior: llvm::Module::Override, Key: "Objective-C Class Properties",
7369 Val: (uint32_t)0);
7370 Changed = true;
7371 }
7372
7373 if (HasSwiftVersionFlag) {
7374 M.addModuleFlag(Behavior: Module::Error, Key: "Swift ABI Version",
7375 Val: SwiftABIVersion);
7376 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Major Version",
7377 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMajorVersion));
7378 M.addModuleFlag(Behavior: Module::Error, Key: "Swift Minor Version",
7379 Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMinorVersion));
7380 Changed = true;
7381 }
7382
7383 return Changed;
7384}
7385
7386bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7387 NamedMDNode *CFIConsts = M.getNamedMetadata(Name: "cfi.functions");
7388 // If this metadata has operands, we expect all of them to be either from
7389 // before or from after the format change handled here, so we can bail out
7390 // fast if the first (if any) operands is of the new format.
7391 auto MatchesVersion = [](const MDNode *Op) {
7392 return Op->getNumOperands() >= 3 &&
7393 isa<ConstantAsMetadata>(Val: Op->getOperand(I: 2)) &&
7394 cast<ConstantAsMetadata>(Val: Op->getOperand(I: 2))
7395 ->getType()
7396 ->isIntegerTy(BitWidth: 64);
7397 };
7398
7399 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7400 MatchesVersion(CFIConsts->getOperand(i: 0)))
7401 return false;
7402
7403 bool Changed = false;
7404 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7405 MDNode *Op = CFIConsts->getOperand(i: I);
7406 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7407 assert(Op->getNumOperands() >= 2 &&
7408 "Expected at least 2 operands - name and linkage type");
7409 MDString *NameMD = dyn_cast<MDString>(Val: Op->getOperand(I: 0));
7410 StringRef Name = NameMD->getString();
7411 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7412 GlobalName: GlobalValue::dropLLVMManglingEscape(Name));
7413
7414 SmallVector<Metadata *, 4> Elts;
7415 Elts.push_back(Elt: Op->getOperand(I: 0));
7416 Elts.push_back(Elt: Op->getOperand(I: 1));
7417 Elts.push_back(Elt: ConstantAsMetadata::get(
7418 C: ConstantInt::get(Ty: Type::getInt64Ty(C&: M.getContext()), V: GUID)));
7419
7420 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7421 Elts.push_back(Elt: Op->getOperand(I: J));
7422
7423 CFIConsts->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Elts));
7424 Changed = true;
7425 }
7426
7427 return Changed;
7428}
7429
7430void llvm::UpgradeSectionAttributes(Module &M) {
7431 auto TrimSpaces = [](StringRef Section) -> std::string {
7432 SmallVector<StringRef, 5> Components;
7433 Section.split(A&: Components, Separator: ',');
7434
7435 SmallString<32> Buffer;
7436 raw_svector_ostream OS(Buffer);
7437
7438 for (auto Component : Components)
7439 OS << ',' << Component.trim();
7440
7441 return std::string(OS.str().substr(Start: 1));
7442 };
7443
7444 for (auto &GV : M.globals()) {
7445 if (!GV.hasSection())
7446 continue;
7447
7448 StringRef Section = GV.getSection();
7449
7450 if (!Section.starts_with(Prefix: "__DATA, __objc_catlist"))
7451 continue;
7452
7453 // __DATA, __objc_catlist, regular, no_dead_strip
7454 // __DATA,__objc_catlist,regular,no_dead_strip
7455 GV.setSection(TrimSpaces(Section));
7456 }
7457}
7458
7459namespace {
7460// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7461// callsites within a function that did not also have the strictfp attribute.
7462// Since 10.0, if strict FP semantics are needed within a function, the
7463// function must have the strictfp attribute and all calls within the function
7464// must also have the strictfp attribute. This latter restriction is
7465// necessary to prevent unwanted libcall simplification when a function is
7466// being cloned (such as for inlining).
7467//
7468// The "dangling" strictfp attribute usage was only used to prevent constant
7469// folding and other libcall simplification. The nobuiltin attribute on the
7470// callsite has the same effect.
7471struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7472 StrictFPUpgradeVisitor() = default;
7473
7474 void visitCallBase(CallBase &Call) {
7475 if (!Call.isStrictFP())
7476 return;
7477 if (isa<ConstrainedFPIntrinsic>(Val: &Call))
7478 return;
7479 // If we get here, the caller doesn't have the strictfp attribute
7480 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7481 Call.removeFnAttr(Kind: Attribute::StrictFP);
7482 Call.addFnAttr(Kind: Attribute::NoBuiltin);
7483 }
7484};
7485
7486/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7487struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7488 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7489 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7490
7491 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7492 if (!RMW.isFloatingPointOperation())
7493 return;
7494
7495 MDNode *Empty = MDNode::get(Context&: RMW.getContext(), MDs: {});
7496 RMW.setMetadata(Kind: "amdgpu.no.fine.grained.host.memory", Node: Empty);
7497 RMW.setMetadata(Kind: "amdgpu.no.remote.memory.access", Node: Empty);
7498 RMW.setMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode, Node: Empty);
7499 }
7500};
7501} // namespace
7502
7503void llvm::UpgradeFunctionAttributes(Function &F) {
7504 // If a function definition doesn't have the strictfp attribute,
7505 // convert any callsite strictfp attributes to nobuiltin.
7506 if (!F.isDeclaration() && !F.hasFnAttribute(Kind: Attribute::StrictFP)) {
7507 StrictFPUpgradeVisitor SFPV;
7508 SFPV.visit(F);
7509 }
7510
7511 // Remove all incompatibile attributes from function.
7512 F.removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
7513 Ty: F.getReturnType(), AS: F.getAttributes().getRetAttrs()));
7514 for (auto &Arg : F.args())
7515 Arg.removeAttrs(
7516 AM: AttributeFuncs::typeIncompatible(Ty: Arg.getType(), AS: Arg.getAttributes()));
7517
7518 bool AddingAttrs = false, RemovingAttrs = false;
7519 AttrBuilder AttrsToAdd(F.getContext());
7520 AttributeMask AttrsToRemove;
7521
7522 // Older versions of LLVM treated an "implicit-section-name" attribute
7523 // similarly to directly setting the section on a Function.
7524 if (Attribute A = F.getFnAttribute(Kind: "implicit-section-name");
7525 A.isValid() && A.isStringAttribute()) {
7526 F.setSection(A.getValueAsString());
7527 AttrsToRemove.addAttribute(A: "implicit-section-name");
7528 RemovingAttrs = true;
7529 }
7530
7531 if (Attribute A = F.getFnAttribute(Kind: "nooutline");
7532 A.isValid() && A.isStringAttribute()) {
7533 AttrsToRemove.addAttribute(A: "nooutline");
7534 AttrsToAdd.addAttribute(Val: Attribute::NoOutline);
7535 AddingAttrs = RemovingAttrs = true;
7536 }
7537
7538 if (Attribute A = F.getFnAttribute(Kind: "uniform-work-group-size");
7539 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7540 AttrsToRemove.addAttribute(A: "uniform-work-group-size");
7541 RemovingAttrs = true;
7542 if (A.getValueAsString() == "true") {
7543 AttrsToAdd.addAttribute(A: "uniform-work-group-size");
7544 AddingAttrs = true;
7545 }
7546 }
7547
7548 if (!F.empty()) {
7549 // For some reason this is called twice, and the first time is before any
7550 // instructions are loaded into the body.
7551
7552 if (Attribute A = F.getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics");
7553 A.isValid()) {
7554
7555 if (A.getValueAsBool()) {
7556 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7557 Visitor.visit(F);
7558 }
7559
7560 // We will leave behind dead attribute uses on external declarations, but
7561 // clang never added these to declarations anyway.
7562 AttrsToRemove.addAttribute(A: "amdgpu-unsafe-fp-atomics");
7563 RemovingAttrs = true;
7564 }
7565 }
7566
7567 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7568 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7569
7570 bool HandleDenormalMode = false;
7571
7572 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math"); Attr.isValid()) {
7573 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7574 if (ParsedMode.isValid()) {
7575 DenormalFPMath = ParsedMode;
7576 AttrsToRemove.addAttribute(A: "denormal-fp-math");
7577 AddingAttrs = RemovingAttrs = true;
7578 HandleDenormalMode = true;
7579 }
7580 }
7581
7582 if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math-f32");
7583 Attr.isValid()) {
7584 DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
7585 if (ParsedMode.isValid()) {
7586 DenormalFPMathF32 = ParsedMode;
7587 AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
7588 AddingAttrs = RemovingAttrs = true;
7589 HandleDenormalMode = true;
7590 }
7591 }
7592
7593 if (HandleDenormalMode)
7594 AttrsToAdd.addDenormalFPEnvAttr(
7595 Mode: DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7596
7597 if (RemovingAttrs)
7598 F.removeFnAttrs(Attrs: AttrsToRemove);
7599
7600 if (AddingAttrs)
7601 F.addFnAttrs(Attrs: AttrsToAdd);
7602}
7603
7604// Check if the function attribute is not present and set it.
7605static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7606 StringRef Value) {
7607 if (!F.hasFnAttribute(Kind: FnAttrName)) {
7608 F.addFnAttr(Kind: FnAttrName, Val: Value);
7609 LLVM_DEBUG(dbgs() << "Set attribute: " << FnAttrName << "=\"" << Value
7610 << "\", function: " << F.getName() << "\n");
7611 }
7612}
7613
7614// Check if the function attribute is not present and set it if needed.
7615// If the attribute is "false" then removes it.
7616// If the attribute is "true" resets it to a valueless attribute.
7617static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7618 if (!F.hasFnAttribute(Kind: FnAttrName)) {
7619 if (Set) {
7620 F.addFnAttr(Kind: FnAttrName);
7621 LLVM_DEBUG(dbgs() << "Added attribute: " << FnAttrName
7622 << ", function: " << F.getName() << "\n");
7623 }
7624 } else {
7625 auto A = F.getFnAttribute(Kind: FnAttrName);
7626 if ("false" == A.getValueAsString()) {
7627 F.removeFnAttr(Kind: FnAttrName);
7628 LLVM_DEBUG(dbgs() << "Removed attribute: " << FnAttrName
7629 << "=\"false\", function: " << F.getName() << "\n");
7630 } else if ("true" == A.getValueAsString()) {
7631 F.removeFnAttr(Kind: FnAttrName);
7632 F.addFnAttr(Kind: FnAttrName);
7633 LLVM_DEBUG(dbgs() << "Converted attribute: " << FnAttrName
7634 << "=\"true\", function: " << F.getName() << "\n");
7635 }
7636 }
7637}
7638
7639static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior,
7640 StringRef Key, uint32_t Val) {
7641 M.setModuleFlag(Behavior, Key, Val);
7642 LLVM_DEBUG(dbgs() << "Converted module flag: " << "{" << Behavior << ", "
7643 << Key << ", " << Val << "}\n");
7644}
7645
7646void llvm::copyModuleAttrToFunctions(Module &M) {
7647 Triple T(M.getTargetTriple());
7648 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7649 return;
7650
7651 uint64_t BTEValue = 0;
7652 uint64_t BPPLRValue = 0;
7653 uint64_t GCSValue = 0;
7654 uint64_t SRAValue = 0;
7655 uint64_t SRAALLValue = 0;
7656 uint64_t SRABKeyValue = 0;
7657
7658 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7659 if (ModFlags) {
7660 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7661 MDNode *Op = ModFlags->getOperand(i: I);
7662 if (Op->getNumOperands() != 3)
7663 continue;
7664
7665 MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: 1));
7666 auto *CI = mdconst::dyn_extract<ConstantInt>(MD: Op->getOperand(I: 2));
7667 if (!ID || !CI)
7668 continue;
7669
7670 StringRef IDStr = ID->getString();
7671 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7672 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7673 : IDStr == "guarded-control-stack" ? &GCSValue
7674 : IDStr == "sign-return-address" ? &SRAValue
7675 : IDStr == "sign-return-address-all" ? &SRAALLValue
7676 : IDStr == "sign-return-address-with-bkey"
7677 ? &SRABKeyValue
7678 : nullptr;
7679 if (!ValPtr)
7680 continue;
7681
7682 *ValPtr = CI->getZExtValue();
7683 if (*ValPtr == 2)
7684 return;
7685
7686 LLVM_DEBUG(dbgs() << "Found module flag: " << IDStr << "(" << *ValPtr
7687 << ")\n");
7688 }
7689 }
7690
7691 bool BTE = BTEValue == 1;
7692 bool BPPLR = BPPLRValue == 1;
7693 bool GCS = GCSValue == 1;
7694 bool SRA = SRAValue == 1;
7695
7696 StringRef SignTypeValue = "non-leaf";
7697 if (SRA && SRAALLValue == 1)
7698 SignTypeValue = "all";
7699
7700 StringRef SignKeyValue = "a_key";
7701 if (SRA && SRABKeyValue == 1)
7702 SignKeyValue = "b_key";
7703
7704 for (Function &F : M.getFunctionList()) {
7705 if (F.isDeclaration())
7706 continue;
7707
7708 if (SRA) {
7709 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address", Value: SignTypeValue);
7710 setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address-key", Value: SignKeyValue);
7711 } else {
7712 if (auto A = F.getFnAttribute(Kind: "sign-return-address");
7713 A.isValid() && "none" == A.getValueAsString()) {
7714 F.removeFnAttr(Kind: "sign-return-address");
7715 F.removeFnAttr(Kind: "sign-return-address-key");
7716 }
7717 }
7718 ConvertFunctionAttr(F, Set: BTE, FnAttrName: "branch-target-enforcement");
7719 ConvertFunctionAttr(F, Set: BPPLR, FnAttrName: "branch-protection-pauth-lr");
7720 ConvertFunctionAttr(F, Set: GCS, FnAttrName: "guarded-control-stack");
7721 }
7722
7723 if (BTE)
7724 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "branch-target-enforcement", Val: 2);
7725 if (BPPLR)
7726 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "branch-protection-pauth-lr", Val: 2);
7727 if (GCS)
7728 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "guarded-control-stack", Val: 2);
7729 if (SRA) {
7730 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "sign-return-address", Val: 2);
7731 if (SRAALLValue == 1)
7732 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "sign-return-address-all", Val: 2);
7733 if (SRABKeyValue == 1) {
7734 ConvertModuleFlag(M, Behavior: llvm::Module::Min, Key: "sign-return-address-with-bkey",
7735 Val: 2);
7736 }
7737 }
7738}
7739
7740/// Return the replacement tags if \p T still uses a removed two-operand form.
7741static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7742 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
7743 return nullptr;
7744 auto *Tag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7745 return Tag ? findBooleanLoopTags(Name: Tag->getString()) : nullptr;
7746}
7747
7748/// Build the single-operand node that replaces a boolean operand: nonzero
7749/// selects the enable tag, zero the disable tag.
7750static Metadata *makeBooleanLoopNode(LLVMContext &C,
7751 const BooleanLoopTags &Tags,
7752 const MDOperand &Op) {
7753 bool Enable = !mdconst::extract<ConstantInt>(MD: Op)->isZero();
7754 return MDTuple::get(Context&: C,
7755 MDs: {MDString::get(Context&: C, Str: Enable ? Tags.Enable : Tags.Disable)});
7756}
7757
7758static bool isOldLoopArgument(Metadata *MD) {
7759 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
7760 if (!T)
7761 return false;
7762 if (T->getNumOperands() < 1)
7763 return false;
7764 auto *S = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7765 if (!S)
7766 return false;
7767 if (S->getString().starts_with(Prefix: "llvm.vectorizer."))
7768 return true;
7769 return getOldBooleanLoopTags(T) != nullptr;
7770}
7771
7772static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7773 StringRef OldPrefix = "llvm.vectorizer.";
7774 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7775
7776 if (OldTag == "llvm.vectorizer.unroll")
7777 return MDString::get(Context&: C, Str: "llvm.loop.interleave.count");
7778
7779 return MDString::get(
7780 Context&: C, Str: (Twine("llvm.loop.vectorize.") + OldTag.drop_front(N: OldPrefix.size()))
7781 .str());
7782}
7783
7784static Metadata *upgradeLoopArgument(Metadata *MD) {
7785 auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
7786 if (!T)
7787 return MD;
7788 if (T->getNumOperands() < 1)
7789 return MD;
7790 auto *OldTag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: 0));
7791 if (!OldTag)
7792 return MD;
7793
7794 LLVMContext &C = T->getContext();
7795
7796 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7797 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7798 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
7799
7800 if (!OldTag->getString().starts_with(Prefix: "llvm.vectorizer."))
7801 return MD;
7802
7803 // This has an old tag. Upgrade it.
7804 MDString *NewTag = upgradeLoopTag(C, OldTag: OldTag->getString());
7805
7806 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7807 // vectorize.enable/disable pair, not a two-operand enable node.
7808 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(MD: T->getOperand(I: 1)))
7809 if (const BooleanLoopTags *Tags = findBooleanLoopTags(Name: NewTag->getString()))
7810 return makeBooleanLoopNode(C, Tags: *Tags, Op: T->getOperand(I: 1));
7811
7812 SmallVector<Metadata *, 8> Ops;
7813 Ops.reserve(N: T->getNumOperands());
7814 Ops.push_back(Elt: NewTag);
7815 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7816 Ops.push_back(Elt: T->getOperand(I));
7817
7818 return MDTuple::get(Context&: C, MDs: Ops);
7819}
7820
7821MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7822 auto *T = dyn_cast<MDTuple>(Val: &N);
7823 if (!T)
7824 return &N;
7825
7826 if (none_of(Range: T->operands(), P: isOldLoopArgument))
7827 return &N;
7828
7829 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7830 // any MDNode carrying those tags with more than one operand, so a leftover
7831 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7832 // In-place mutation is safe on distinct MDNodes.
7833 if (T->isDistinct()) {
7834 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7835 auto *OpT = dyn_cast_or_null<MDTuple>(Val: T->getOperand(I));
7836 if (OpT && getOldBooleanLoopTags(T: OpT))
7837 T->replaceOperandWith(I, New: upgradeLoopArgument(MD: OpT));
7838 }
7839 if (none_of(Range: T->operands(), P: isOldLoopArgument))
7840 return &N;
7841 }
7842
7843 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7844 // attachment; the original distinct loop-ID is kept as the first operand.
7845 SmallVector<Metadata *, 8> Ops;
7846 Ops.reserve(N: T->getNumOperands());
7847 for (Metadata *MD : T->operands())
7848 Ops.push_back(Elt: upgradeLoopArgument(MD));
7849
7850 return MDTuple::get(Context&: T->getContext(), MDs: Ops);
7851}
7852
7853std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7854 Triple T(TT);
7855 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7856 // the address space of globals to 1. This does not apply to SPIRV Logical.
7857 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7858 !DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G")) {
7859 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7860 }
7861
7862 if (T.isLoongArch64() || T.isRISCV64()) {
7863 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7864 auto I = DL.find(Str: "-n64-");
7865 if (I != StringRef::npos)
7866 return (DL.take_front(N: I) + "-n32:64-" + DL.drop_front(N: I + 5)).str();
7867 return DL.str();
7868 }
7869
7870 // AMDGPU data layout upgrades.
7871 std::string Res = DL.str();
7872 if (T.isAMDGPU()) {
7873 // Define address spaces for constants.
7874 if (!DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G"))
7875 Res.append(s: Res.empty() ? "G1" : "-G1");
7876
7877 // AMDGCN data layout upgrades.
7878 if (T.isAMDGCN()) {
7879
7880 // Add missing non-integral declarations.
7881 // This goes before adding new address spaces to prevent incoherent string
7882 // values.
7883 if (!DL.contains(Other: "-ni") && !DL.starts_with(Prefix: "ni"))
7884 Res.append(s: "-ni:7:8:9");
7885 // Update ni:7 to ni:7:8:9.
7886 if (DL.ends_with(Suffix: "ni:7"))
7887 Res.append(s: ":8:9");
7888 if (DL.ends_with(Suffix: "ni:7:8"))
7889 Res.append(s: ":9");
7890
7891 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7892 // resources) An empty data layout has already been upgraded to G1 by now.
7893 if (!DL.contains(Other: "-p7") && !DL.starts_with(Prefix: "p7"))
7894 Res.append(s: "-p7:160:256:256:32");
7895 if (!DL.contains(Other: "-p8") && !DL.starts_with(Prefix: "p8"))
7896 Res.append(s: "-p8:128:128:128:48");
7897 constexpr StringRef OldP8("-p8:128:128-");
7898 if (DL.contains(Other: OldP8))
7899 Res.replace(pos: Res.find(svt: OldP8), n1: OldP8.size(), s: "-p8:128:128:128:48-");
7900 if (!DL.contains(Other: "-p9") && !DL.starts_with(Prefix: "p9"))
7901 Res.append(s: "-p9:192:256:256:32");
7902
7903 // Add sizing for address space 10 through 15.
7904 // AS 10-14 are reserved and defaulted to 32:32
7905 // AS 15 is in use and is 32:32.
7906 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7907 if (!DL.contains(Other: ("-" + AS).str()) && !DL.starts_with(Prefix: AS))
7908 Res.append(str: ("-" + AS + ":32:32").str());
7909 }
7910 }
7911
7912 // Upgrade the ELF mangling mode.
7913 if (!DL.contains(Other: "m:e"))
7914 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7915
7916 return Res;
7917 }
7918
7919 if (T.isSystemZ() && !DL.empty()) {
7920 // Make sure the stack alignment is present.
7921 if (!DL.contains(Other: "-S64"))
7922 return "E-S64" + DL.drop_front(N: 1).str();
7923 return DL.str();
7924 }
7925
7926 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7927 // If the datalayout matches the expected format, add pointer size address
7928 // spaces to the datalayout.
7929 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7930 if (!DL.contains(Other: AddrSpaces)) {
7931 SmallVector<StringRef, 4> Groups;
7932 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7933 if (R.match(String: Res, Matches: &Groups))
7934 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7935 }
7936 };
7937
7938 // AArch64 data layout upgrades.
7939 if (T.isAArch64()) {
7940 // Add "-Fn32"
7941 if (!DL.empty() && !DL.contains(Other: "-Fn32"))
7942 Res.append(s: "-Fn32");
7943 AddPtr32Ptr64AddrSpaces();
7944 return Res;
7945 }
7946
7947 if (T.isSPARC() || (T.isMIPS64() && !DL.contains(Other: "m:m")) || T.isPPC64() ||
7948 T.isWasm()) {
7949 // Mips64 with o32 ABI did not add "-i128:128".
7950 // Add "-i128:128"
7951 std::string I64 = "-i64:64";
7952 std::string I128 = "-i128:128";
7953 if (!StringRef(Res).contains(Other: I128)) {
7954 size_t Pos = Res.find(str: I64);
7955 if (Pos != size_t(-1))
7956 Res.insert(pos1: Pos + I64.size(), str: I128);
7957 }
7958 }
7959
7960 if (T.isPPC() && T.isOSAIX() && !DL.contains(Other: "f64:32:64") && !DL.empty()) {
7961 size_t Pos = Res.find(s: "-S128");
7962 if (Pos == StringRef::npos)
7963 Pos = Res.size();
7964 Res.insert(pos: Pos, s: "-f64:32:64");
7965 }
7966
7967 // ARM data layout upgrades.
7968 // Add -Fi8 if a -F has not already been specified.
7969 if (T.isARM() && !DL.empty() && !DL.contains(Other: "Fi") && !DL.contains(Other: "Fn")) {
7970 const StringRef p3232 = "p:32:32";
7971 size_t Pos = Res.find(svt: p3232);
7972 if (Pos != StringRef::npos)
7973 Res.insert(pos: Pos + p3232.size(), s: "-Fi8");
7974 }
7975
7976 if (!T.isX86())
7977 return Res;
7978
7979 AddPtr32Ptr64AddrSpaces();
7980
7981 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7982 // for i128 operations prior to this being reflected in the data layout, and
7983 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7984 // boundaries, so although this is a breaking change, the upgrade is expected
7985 // to fix more IR than it breaks.
7986 // Intel MCU is an exception and uses 4-byte-alignment.
7987 if (!T.isOSIAMCU()) {
7988 std::string I128 = "-i128:128";
7989 if (StringRef Ref = Res; !Ref.contains(Other: I128)) {
7990 SmallVector<StringRef, 4> Groups;
7991 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7992 if (R.match(String: Res, Matches: &Groups))
7993 Res = (Groups[1] + I128 + Groups[3]).str();
7994 }
7995 }
7996
7997 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7998 // Raising the alignment is safe because Clang did not produce f80 values in
7999 // the MSVC environment before this upgrade was added.
8000 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
8001 StringRef Ref = Res;
8002 auto I = Ref.find(Str: "-f80:32-");
8003 if (I != StringRef::npos)
8004 Res = (Ref.take_front(N: I) + "-f80:128-" + Ref.drop_front(N: I + 8)).str();
8005 }
8006
8007 return Res;
8008}
8009
8010void llvm::UpgradeAttributes(AttrBuilder &B) {
8011 StringRef FramePointer;
8012 Attribute A = B.getAttribute(Kind: "no-frame-pointer-elim");
8013 if (A.isValid()) {
8014 // The value can be "true" or "false".
8015 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
8016 B.removeAttribute(A: "no-frame-pointer-elim");
8017 }
8018 if (B.contains(A: "no-frame-pointer-elim-non-leaf")) {
8019 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
8020 if (FramePointer != "all")
8021 FramePointer = "non-leaf";
8022 B.removeAttribute(A: "no-frame-pointer-elim-non-leaf");
8023 }
8024 if (!FramePointer.empty())
8025 B.addAttribute(A: "frame-pointer", V: FramePointer);
8026
8027 A = B.getAttribute(Kind: "null-pointer-is-valid");
8028 if (A.isValid()) {
8029 // The value can be "true" or "false".
8030 bool NullPointerIsValid = A.getValueAsString() == "true";
8031 B.removeAttribute(A: "null-pointer-is-valid");
8032 if (NullPointerIsValid)
8033 B.addAttribute(Val: Attribute::NullPointerIsValid);
8034 }
8035
8036 A = B.getAttribute(Kind: "uniform-work-group-size");
8037 if (A.isValid()) {
8038 StringRef Val = A.getValueAsString();
8039 if (!Val.empty()) {
8040 bool IsTrue = Val == "true";
8041 B.removeAttribute(A: "uniform-work-group-size");
8042 if (IsTrue)
8043 B.addAttribute(A: "uniform-work-group-size");
8044 }
8045 }
8046}
8047
8048void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
8049 // clang.arc.attachedcall bundles are now required to have an operand.
8050 // If they don't, it's okay to drop them entirely: when there is an operand,
8051 // the "attachedcall" is meaningful and required, but without an operand,
8052 // it's just a marker NOP. Dropping it merely prevents an optimization.
8053 erase_if(C&: Bundles, P: [&](OperandBundleDef &OBD) {
8054 return OBD.getTag() == "clang.arc.attachedcall" &&
8055 OBD.inputs().empty();
8056 });
8057}
8058