AutoUpgrade.cpp source code [llvm_projects/llvm/lib/IR/AutoUpgrade.cpp]

1	//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements the auto-upgrade helper functions.
10	// This is where deprecated IR intrinsics and other IR features are updated to
11	// current specifications.
12	//
13	//===----------------------------------------------------------------------===//
14
15	#include "llvm/IR/AutoUpgrade.h"
16	#include "llvm/ADT/ArrayRef.h"
17	#include "llvm/ADT/StringExtras.h"
18	#include "llvm/ADT/StringRef.h"
19	#include "llvm/ADT/StringSwitch.h"
20	#include "llvm/BinaryFormat/Dwarf.h"
21	#include "llvm/IR/AttributeMask.h"
22	#include "llvm/IR/Attributes.h"
23	#include "llvm/IR/CallingConv.h"
24	#include "llvm/IR/Constants.h"
25	#include "llvm/IR/DebugInfo.h"
26	#include "llvm/IR/DebugInfoMetadata.h"
27	#include "llvm/IR/DiagnosticInfo.h"
28	#include "llvm/IR/Function.h"
29	#include "llvm/IR/GlobalValue.h"
30	#include "llvm/IR/IRBuilder.h"
31	#include "llvm/IR/InstVisitor.h"
32	#include "llvm/IR/Instruction.h"
33	#include "llvm/IR/IntrinsicInst.h"
34	#include "llvm/IR/Intrinsics.h"
35	#include "llvm/IR/IntrinsicsAArch64.h"
36	#include "llvm/IR/IntrinsicsAMDGPU.h"
37	#include "llvm/IR/IntrinsicsARM.h"
38	#include "llvm/IR/IntrinsicsNVPTX.h"
39	#include "llvm/IR/IntrinsicsRISCV.h"
40	#include "llvm/IR/IntrinsicsWebAssembly.h"
41	#include "llvm/IR/IntrinsicsX86.h"
42	#include "llvm/IR/LLVMContext.h"
43	#include "llvm/IR/MDBuilder.h"
44	#include "llvm/IR/Metadata.h"
45	#include "llvm/IR/Module.h"
46	#include "llvm/IR/Value.h"
47	#include "llvm/IR/Verifier.h"
48	#include "llvm/Support/AMDGPUAddrSpace.h"
49	#include "llvm/Support/CommandLine.h"
50	#include "llvm/Support/ErrorHandling.h"
51	#include "llvm/Support/NVPTXAddrSpace.h"
52	#include "llvm/Support/NVVMAttributes.h"
53	#include "llvm/Support/Regex.h"
54	#include "llvm/Support/TimeProfiler.h"
55	#include "llvm/TargetParser/Triple.h"
56	#include <cstdint>
57	#include <cstring>
58	#include <numeric>
59
60	using namespace llvm;
61
62	static cl::opt<bool>
63	DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
64	cl::desc("Disable autoupgrade of debug info"));
65
66	static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
67
68	// Report a fatal error along with the
69	// Call Instruction which caused the error
70	[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
71	CallBase *CI) {
72	CI->print(O&: llvm::errs());
73	llvm::errs() << "\n";
74	reportFatalUsageError(reason);
75	}
76
77	// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
78	// changed their type from v4f32 to v2i64.
79	static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
80	Function *&NewFn) {
81	// Check whether this is an old version of the function, which received
82	// v4f32 arguments.
83	Type *Arg0Type = F->getFunctionType()->getParamType(i: `0`);
84	if (Arg0Type != FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: `4`))
85	return false;
86
87	// Yes, it's old, replace it with new version.
88	rename(GV: F);
89	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
90	return true;
91	}
92
93	// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
94	// arguments have changed their type from i32 to i8.
95	static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
96	Function *&NewFn) {
97	// Check that the last argument is an i32.
98	Type *LastArgType = F->getFunctionType()->getParamType(
99	i: F->getFunctionType()->getNumParams() - `1`);
100	if (!LastArgType->isIntegerTy(BitWidth: `32`))
101	return false;
102
103	// Move this function aside and map down.
104	rename(GV: F);
105	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
106	return true;
107	}
108
109	// Upgrade the declaration of fp compare intrinsics that change return type
110	// from scalar to vXi1 mask.
111	static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
112	Function *&NewFn) {
113	// Check if the return type is a vector.
114	if (F->getReturnType()->isVectorTy())
115	return false;
116
117	rename(GV: F);
118	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
119	return true;
120	}
121
122	// Upgrade the declaration of multiply and add bytes intrinsics whose input
123	// arguments' types have changed from vectors of i32 to vectors of i8
124	static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
125	Function *&NewFn) {
126	// check if input argument type is a vector of i8
127	Type *Arg1Type = F->getFunctionType()->getParamType(i: `1`);
128	Type *Arg2Type = F->getFunctionType()->getParamType(i: `2`);
129	if (Arg1Type->isVectorTy() &&
130	cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: `8`) &&
131	Arg2Type->isVectorTy() &&
132	cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: `8`))
133	return false;
134
135	rename(GV: F);
136	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
137	return true;
138	}
139
140	// Upgrade the declaration of multipy and add words intrinsics whose input
141	// arguments' types have changed to vectors of i32 to vectors of i16
142	static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
143	Function *&NewFn) {
144	// check if input argument type is a vector of i16
145	Type *Arg1Type = F->getFunctionType()->getParamType(i: `1`);
146	Type *Arg2Type = F->getFunctionType()->getParamType(i: `2`);
147	if (Arg1Type->isVectorTy() &&
148	cast<VectorType>(Val: Arg1Type)->getElementType()->isIntegerTy(BitWidth: `16`) &&
149	Arg2Type->isVectorTy() &&
150	cast<VectorType>(Val: Arg2Type)->getElementType()->isIntegerTy(BitWidth: `16`))
151	return false;
152
153	rename(GV: F);
154	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
155	return true;
156	}
157
158	static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
159	Function *&NewFn) {
160	if (F->getReturnType()->getScalarType()->isBFloatTy())
161	return false;
162
163	rename(GV: F);
164	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
165	return true;
166	}
167
168	static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
169	Function *&NewFn) {
170	if (F->getFunctionType()->getParamType(i: `1`)->getScalarType()->isBFloatTy())
171	return false;
172
173	rename(GV: F);
174	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
175	return true;
176	}
177
178	static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
179	// All of the intrinsics matches below should be marked with which llvm
180	// version started autoupgrading them. At some point in the future we would
181	// like to use this information to remove upgrade code for some older
182	// intrinsics. It is currently undecided how we will determine that future
183	// point.
184	if (Name.consume_front(Prefix: "avx."))
185	return (Name.starts_with(Prefix: "blend.p") \|\| // Added in 3.7
186	Name == "cvt.ps2.pd.256" \|\| // Added in 3.9
187	Name == "cvtdq2.pd.256" \|\| // Added in 3.9
188	Name == "cvtdq2.ps.256" \|\| // Added in 7.0
189	Name.starts_with(Prefix: "movnt.") \|\| // Added in 3.2
190	Name.starts_with(Prefix: "sqrt.p") \|\| // Added in 7.0
191	Name.starts_with(Prefix: "storeu.") \|\| // Added in 3.9
192	Name.starts_with(Prefix: "vbroadcast.s") \|\| // Added in 3.5
193	Name.starts_with(Prefix: "vbroadcastf128") \|\| // Added in 4.0
194	Name.starts_with(Prefix: "vextractf128.") \|\| // Added in 3.7
195	Name.starts_with(Prefix: "vinsertf128.") \|\| // Added in 3.7
196	Name.starts_with(Prefix: "vperm2f128.") \|\| // Added in 6.0
197	Name.starts_with(Prefix: "vpermil.")); // Added in 3.1
198
199	if (Name.consume_front(Prefix: "avx2."))
200	return (Name == "movntdqa" \|\| // Added in 5.0
201	Name.starts_with(Prefix: "pabs.") \|\| // Added in 6.0
202	Name.starts_with(Prefix: "padds.") \|\| // Added in 8.0
203	Name.starts_with(Prefix: "paddus.") \|\| // Added in 8.0
204	Name.starts_with(Prefix: "pblendd.") \|\| // Added in 3.7
205	Name == "pblendw" \|\| // Added in 3.7
206	Name.starts_with(Prefix: "pbroadcast") \|\| // Added in 3.8
207	Name.starts_with(Prefix: "pcmpeq.") \|\| // Added in 3.1
208	Name.starts_with(Prefix: "pcmpgt.") \|\| // Added in 3.1
209	Name.starts_with(Prefix: "pmax") \|\| // Added in 3.9
210	Name.starts_with(Prefix: "pmin") \|\| // Added in 3.9
211	Name.starts_with(Prefix: "pmovsx") \|\| // Added in 3.9
212	Name.starts_with(Prefix: "pmovzx") \|\| // Added in 3.9
213	Name == "pmul.dq" \|\| // Added in 7.0
214	Name == "pmulu.dq" \|\| // Added in 7.0
215	Name.starts_with(Prefix: "psll.dq") \|\| // Added in 3.7
216	Name.starts_with(Prefix: "psrl.dq") \|\| // Added in 3.7
217	Name.starts_with(Prefix: "psubs.") \|\| // Added in 8.0
218	Name.starts_with(Prefix: "psubus.") \|\| // Added in 8.0
219	Name.starts_with(Prefix: "vbroadcast") \|\| // Added in 3.8
220	Name == "vbroadcasti128" \|\| // Added in 3.7
221	Name == "vextracti128" \|\| // Added in 3.7
222	Name == "vinserti128" \|\| // Added in 3.7
223	Name == "vperm2i128"); // Added in 6.0
224
225	if (Name.consume_front(Prefix: "avx512.")) {
226	if (Name.consume_front(Prefix: "mask."))
227	// 'avx512.mask.'*
228	return (Name.starts_with(Prefix: "add.p") \|\| // Added in 7.0. 128/256 in 4.0
229	Name.starts_with(Prefix: "and.") \|\| // Added in 3.9
230	Name.starts_with(Prefix: "andn.") \|\| // Added in 3.9
231	Name.starts_with(Prefix: "broadcast.s") \|\| // Added in 3.9
232	Name.starts_with(Prefix: "broadcastf32x4.") \|\| // Added in 6.0
233	Name.starts_with(Prefix: "broadcastf32x8.") \|\| // Added in 6.0
234	Name.starts_with(Prefix: "broadcastf64x2.") \|\| // Added in 6.0
235	Name.starts_with(Prefix: "broadcastf64x4.") \|\| // Added in 6.0
236	Name.starts_with(Prefix: "broadcasti32x4.") \|\| // Added in 6.0
237	Name.starts_with(Prefix: "broadcasti32x8.") \|\| // Added in 6.0
238	Name.starts_with(Prefix: "broadcasti64x2.") \|\| // Added in 6.0
239	Name.starts_with(Prefix: "broadcasti64x4.") \|\| // Added in 6.0
240	Name.starts_with(Prefix: "cmp.b") \|\| // Added in 5.0
241	Name.starts_with(Prefix: "cmp.d") \|\| // Added in 5.0
242	Name.starts_with(Prefix: "cmp.q") \|\| // Added in 5.0
243	Name.starts_with(Prefix: "cmp.w") \|\| // Added in 5.0
244	Name.starts_with(Prefix: "compress.b") \|\| // Added in 9.0
245	Name.starts_with(Prefix: "compress.d") \|\| // Added in 9.0
246	Name.starts_with(Prefix: "compress.p") \|\| // Added in 9.0
247	Name.starts_with(Prefix: "compress.q") \|\| // Added in 9.0
248	Name.starts_with(Prefix: "compress.store.") \|\| // Added in 7.0
249	Name.starts_with(Prefix: "compress.w") \|\| // Added in 9.0
250	Name.starts_with(Prefix: "conflict.") \|\| // Added in 9.0
251	Name.starts_with(Prefix: "cvtdq2pd.") \|\| // Added in 4.0
252	Name.starts_with(Prefix: "cvtdq2ps.") \|\| // Added in 7.0 updated 9.0
253	Name == "cvtpd2dq.256" \|\| // Added in 7.0
254	Name == "cvtpd2ps.256" \|\| // Added in 7.0
255	Name == "cvtps2pd.128" \|\| // Added in 7.0
256	Name == "cvtps2pd.256" \|\| // Added in 7.0
257	Name.starts_with(Prefix: "cvtqq2pd.") \|\| // Added in 7.0 updated 9.0
258	Name == "cvtqq2ps.256" \|\| // Added in 9.0
259	Name == "cvtqq2ps.512" \|\| // Added in 9.0
260	Name == "cvttpd2dq.256" \|\| // Added in 7.0
261	Name == "cvttps2dq.128" \|\| // Added in 7.0
262	Name == "cvttps2dq.256" \|\| // Added in 7.0
263	Name.starts_with(Prefix: "cvtudq2pd.") \|\| // Added in 4.0
264	Name.starts_with(Prefix: "cvtudq2ps.") \|\| // Added in 7.0 updated 9.0
265	Name.starts_with(Prefix: "cvtuqq2pd.") \|\| // Added in 7.0 updated 9.0
266	Name == "cvtuqq2ps.256" \|\| // Added in 9.0
267	Name == "cvtuqq2ps.512" \|\| // Added in 9.0
268	Name.starts_with(Prefix: "dbpsadbw.") \|\| // Added in 7.0
269	Name.starts_with(Prefix: "div.p") \|\| // Added in 7.0. 128/256 in 4.0
270	Name.starts_with(Prefix: "expand.b") \|\| // Added in 9.0
271	Name.starts_with(Prefix: "expand.d") \|\| // Added in 9.0
272	Name.starts_with(Prefix: "expand.load.") \|\| // Added in 7.0
273	Name.starts_with(Prefix: "expand.p") \|\| // Added in 9.0
274	Name.starts_with(Prefix: "expand.q") \|\| // Added in 9.0
275	Name.starts_with(Prefix: "expand.w") \|\| // Added in 9.0
276	Name.starts_with(Prefix: "fpclass.p") \|\| // Added in 7.0
277	Name.starts_with(Prefix: "insert") \|\| // Added in 4.0
278	Name.starts_with(Prefix: "load.") \|\| // Added in 3.9
279	Name.starts_with(Prefix: "loadu.") \|\| // Added in 3.9
280	Name.starts_with(Prefix: "lzcnt.") \|\| // Added in 5.0
281	Name.starts_with(Prefix: "max.p") \|\| // Added in 7.0. 128/256 in 5.0
282	Name.starts_with(Prefix: "min.p") \|\| // Added in 7.0. 128/256 in 5.0
283	Name.starts_with(Prefix: "movddup") \|\| // Added in 3.9
284	Name.starts_with(Prefix: "move.s") \|\| // Added in 4.0
285	Name.starts_with(Prefix: "movshdup") \|\| // Added in 3.9
286	Name.starts_with(Prefix: "movsldup") \|\| // Added in 3.9
287	Name.starts_with(Prefix: "mul.p") \|\| // Added in 7.0. 128/256 in 4.0
288	Name.starts_with(Prefix: "or.") \|\| // Added in 3.9
289	Name.starts_with(Prefix: "pabs.") \|\| // Added in 6.0
290	Name.starts_with(Prefix: "packssdw.") \|\| // Added in 5.0
291	Name.starts_with(Prefix: "packsswb.") \|\| // Added in 5.0
292	Name.starts_with(Prefix: "packusdw.") \|\| // Added in 5.0
293	Name.starts_with(Prefix: "packuswb.") \|\| // Added in 5.0
294	Name.starts_with(Prefix: "padd.") \|\| // Added in 4.0
295	Name.starts_with(Prefix: "padds.") \|\| // Added in 8.0
296	Name.starts_with(Prefix: "paddus.") \|\| // Added in 8.0
297	Name.starts_with(Prefix: "palignr.") \|\| // Added in 3.9
298	Name.starts_with(Prefix: "pand.") \|\| // Added in 3.9
299	Name.starts_with(Prefix: "pandn.") \|\| // Added in 3.9
300	Name.starts_with(Prefix: "pavg") \|\| // Added in 6.0
301	Name.starts_with(Prefix: "pbroadcast") \|\| // Added in 6.0
302	Name.starts_with(Prefix: "pcmpeq.") \|\| // Added in 3.9
303	Name.starts_with(Prefix: "pcmpgt.") \|\| // Added in 3.9
304	Name.starts_with(Prefix: "perm.df.") \|\| // Added in 3.9
305	Name.starts_with(Prefix: "perm.di.") \|\| // Added in 3.9
306	Name.starts_with(Prefix: "permvar.") \|\| // Added in 7.0
307	Name.starts_with(Prefix: "pmaddubs.w.") \|\| // Added in 7.0
308	Name.starts_with(Prefix: "pmaddw.d.") \|\| // Added in 7.0
309	Name.starts_with(Prefix: "pmax") \|\| // Added in 4.0
310	Name.starts_with(Prefix: "pmin") \|\| // Added in 4.0
311	Name == "pmov.qd.256" \|\| // Added in 9.0
312	Name == "pmov.qd.512" \|\| // Added in 9.0
313	Name == "pmov.wb.256" \|\| // Added in 9.0
314	Name == "pmov.wb.512" \|\| // Added in 9.0
315	Name.starts_with(Prefix: "pmovsx") \|\| // Added in 4.0
316	Name.starts_with(Prefix: "pmovzx") \|\| // Added in 4.0
317	Name.starts_with(Prefix: "pmul.dq.") \|\| // Added in 4.0
318	Name.starts_with(Prefix: "pmul.hr.sw.") \|\| // Added in 7.0
319	Name.starts_with(Prefix: "pmulh.w.") \|\| // Added in 7.0
320	Name.starts_with(Prefix: "pmulhu.w.") \|\| // Added in 7.0
321	Name.starts_with(Prefix: "pmull.") \|\| // Added in 4.0
322	Name.starts_with(Prefix: "pmultishift.qb.") \|\| // Added in 8.0
323	Name.starts_with(Prefix: "pmulu.dq.") \|\| // Added in 4.0
324	Name.starts_with(Prefix: "por.") \|\| // Added in 3.9
325	Name.starts_with(Prefix: "prol.") \|\| // Added in 8.0
326	Name.starts_with(Prefix: "prolv.") \|\| // Added in 8.0
327	Name.starts_with(Prefix: "pror.") \|\| // Added in 8.0
328	Name.starts_with(Prefix: "prorv.") \|\| // Added in 8.0
329	Name.starts_with(Prefix: "pshuf.b.") \|\| // Added in 4.0
330	Name.starts_with(Prefix: "pshuf.d.") \|\| // Added in 3.9
331	Name.starts_with(Prefix: "pshufh.w.") \|\| // Added in 3.9
332	Name.starts_with(Prefix: "pshufl.w.") \|\| // Added in 3.9
333	Name.starts_with(Prefix: "psll.d") \|\| // Added in 4.0
334	Name.starts_with(Prefix: "psll.q") \|\| // Added in 4.0
335	Name.starts_with(Prefix: "psll.w") \|\| // Added in 4.0
336	Name.starts_with(Prefix: "pslli") \|\| // Added in 4.0
337	Name.starts_with(Prefix: "psllv") \|\| // Added in 4.0
338	Name.starts_with(Prefix: "psra.d") \|\| // Added in 4.0
339	Name.starts_with(Prefix: "psra.q") \|\| // Added in 4.0
340	Name.starts_with(Prefix: "psra.w") \|\| // Added in 4.0
341	Name.starts_with(Prefix: "psrai") \|\| // Added in 4.0
342	Name.starts_with(Prefix: "psrav") \|\| // Added in 4.0
343	Name.starts_with(Prefix: "psrl.d") \|\| // Added in 4.0
344	Name.starts_with(Prefix: "psrl.q") \|\| // Added in 4.0
345	Name.starts_with(Prefix: "psrl.w") \|\| // Added in 4.0
346	Name.starts_with(Prefix: "psrli") \|\| // Added in 4.0
347	Name.starts_with(Prefix: "psrlv") \|\| // Added in 4.0
348	Name.starts_with(Prefix: "psub.") \|\| // Added in 4.0
349	Name.starts_with(Prefix: "psubs.") \|\| // Added in 8.0
350	Name.starts_with(Prefix: "psubus.") \|\| // Added in 8.0
351	Name.starts_with(Prefix: "pternlog.") \|\| // Added in 7.0
352	Name.starts_with(Prefix: "punpckh") \|\| // Added in 3.9
353	Name.starts_with(Prefix: "punpckl") \|\| // Added in 3.9
354	Name.starts_with(Prefix: "pxor.") \|\| // Added in 3.9
355	Name.starts_with(Prefix: "shuf.f") \|\| // Added in 6.0
356	Name.starts_with(Prefix: "shuf.i") \|\| // Added in 6.0
357	Name.starts_with(Prefix: "shuf.p") \|\| // Added in 4.0
358	Name.starts_with(Prefix: "sqrt.p") \|\| // Added in 7.0
359	Name.starts_with(Prefix: "store.b.") \|\| // Added in 3.9
360	Name.starts_with(Prefix: "store.d.") \|\| // Added in 3.9
361	Name.starts_with(Prefix: "store.p") \|\| // Added in 3.9
362	Name.starts_with(Prefix: "store.q.") \|\| // Added in 3.9
363	Name.starts_with(Prefix: "store.w.") \|\| // Added in 3.9
364	Name == "store.ss" \|\| // Added in 7.0
365	Name.starts_with(Prefix: "storeu.") \|\| // Added in 3.9
366	Name.starts_with(Prefix: "sub.p") \|\| // Added in 7.0. 128/256 in 4.0
367	Name.starts_with(Prefix: "ucmp.") \|\| // Added in 5.0
368	Name.starts_with(Prefix: "unpckh.") \|\| // Added in 3.9
369	Name.starts_with(Prefix: "unpckl.") \|\| // Added in 3.9
370	Name.starts_with(Prefix: "valign.") \|\| // Added in 4.0
371	Name == "vcvtph2ps.128" \|\| // Added in 11.0
372	Name == "vcvtph2ps.256" \|\| // Added in 11.0
373	Name.starts_with(Prefix: "vextract") \|\| // Added in 4.0
374	Name.starts_with(Prefix: "vfmadd.") \|\| // Added in 7.0
375	Name.starts_with(Prefix: "vfmaddsub.") \|\| // Added in 7.0
376	Name.starts_with(Prefix: "vfnmadd.") \|\| // Added in 7.0
377	Name.starts_with(Prefix: "vfnmsub.") \|\| // Added in 7.0
378	Name.starts_with(Prefix: "vpdpbusd.") \|\| // Added in 7.0
379	Name.starts_with(Prefix: "vpdpbusds.") \|\| // Added in 7.0
380	Name.starts_with(Prefix: "vpdpwssd.") \|\| // Added in 7.0
381	Name.starts_with(Prefix: "vpdpwssds.") \|\| // Added in 7.0
382	Name.starts_with(Prefix: "vpermi2var.") \|\| // Added in 7.0
383	Name.starts_with(Prefix: "vpermil.p") \|\| // Added in 3.9
384	Name.starts_with(Prefix: "vpermilvar.") \|\| // Added in 4.0
385	Name.starts_with(Prefix: "vpermt2var.") \|\| // Added in 7.0
386	Name.starts_with(Prefix: "vpmadd52") \|\| // Added in 7.0
387	Name.starts_with(Prefix: "vpshld.") \|\| // Added in 7.0
388	Name.starts_with(Prefix: "vpshldv.") \|\| // Added in 8.0
389	Name.starts_with(Prefix: "vpshrd.") \|\| // Added in 7.0
390	Name.starts_with(Prefix: "vpshrdv.") \|\| // Added in 8.0
391	Name.starts_with(Prefix: "vpshufbitqmb.") \|\| // Added in 8.0
392	Name.starts_with(Prefix: "xor.")); // Added in 3.9
393
394	if (Name.consume_front(Prefix: "mask3."))
395	// 'avx512.mask3.'*
396	return (Name.starts_with(Prefix: "vfmadd.") \|\| // Added in 7.0
397	Name.starts_with(Prefix: "vfmaddsub.") \|\| // Added in 7.0
398	Name.starts_with(Prefix: "vfmsub.") \|\| // Added in 7.0
399	Name.starts_with(Prefix: "vfmsubadd.") \|\| // Added in 7.0
400	Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
401
402	if (Name.consume_front(Prefix: "maskz."))
403	// 'avx512.maskz.'*
404	return (Name.starts_with(Prefix: "pternlog.") \|\| // Added in 7.0
405	Name.starts_with(Prefix: "vfmadd.") \|\| // Added in 7.0
406	Name.starts_with(Prefix: "vfmaddsub.") \|\| // Added in 7.0
407	Name.starts_with(Prefix: "vpdpbusd.") \|\| // Added in 7.0
408	Name.starts_with(Prefix: "vpdpbusds.") \|\| // Added in 7.0
409	Name.starts_with(Prefix: "vpdpwssd.") \|\| // Added in 7.0
410	Name.starts_with(Prefix: "vpdpwssds.") \|\| // Added in 7.0
411	Name.starts_with(Prefix: "vpermt2var.") \|\| // Added in 7.0
412	Name.starts_with(Prefix: "vpmadd52") \|\| // Added in 7.0
413	Name.starts_with(Prefix: "vpshldv.") \|\| // Added in 8.0
414	Name.starts_with(Prefix: "vpshrdv.")); // Added in 8.0
415
416	// 'avx512.'*
417	return (Name == "movntdqa" \|\| // Added in 5.0
418	Name == "pmul.dq.512" \|\| // Added in 7.0
419	Name == "pmulu.dq.512" \|\| // Added in 7.0
420	Name.starts_with(Prefix: "broadcastm") \|\| // Added in 6.0
421	Name.starts_with(Prefix: "cmp.p") \|\| // Added in 12.0
422	Name.starts_with(Prefix: "cvtb2mask.") \|\| // Added in 7.0
423	Name.starts_with(Prefix: "cvtd2mask.") \|\| // Added in 7.0
424	Name.starts_with(Prefix: "cvtmask2") \|\| // Added in 5.0
425	Name.starts_with(Prefix: "cvtq2mask.") \|\| // Added in 7.0
426	Name == "cvtusi2sd" \|\| // Added in 7.0
427	Name.starts_with(Prefix: "cvtw2mask.") \|\| // Added in 7.0
428	Name == "kand.w" \|\| // Added in 7.0
429	Name == "kandn.w" \|\| // Added in 7.0
430	Name == "knot.w" \|\| // Added in 7.0
431	Name == "kor.w" \|\| // Added in 7.0
432	Name == "kortestc.w" \|\| // Added in 7.0
433	Name == "kortestz.w" \|\| // Added in 7.0
434	Name.starts_with(Prefix: "kunpck") \|\| // added in 6.0
435	Name == "kxnor.w" \|\| // Added in 7.0
436	Name == "kxor.w" \|\| // Added in 7.0
437	Name.starts_with(Prefix: "padds.") \|\| // Added in 8.0
438	Name.starts_with(Prefix: "pbroadcast") \|\| // Added in 3.9
439	Name.starts_with(Prefix: "prol") \|\| // Added in 8.0
440	Name.starts_with(Prefix: "pror") \|\| // Added in 8.0
441	Name.starts_with(Prefix: "psll.dq") \|\| // Added in 3.9
442	Name.starts_with(Prefix: "psrl.dq") \|\| // Added in 3.9
443	Name.starts_with(Prefix: "psubs.") \|\| // Added in 8.0
444	Name.starts_with(Prefix: "ptestm") \|\| // Added in 6.0
445	Name.starts_with(Prefix: "ptestnm") \|\| // Added in 6.0
446	Name.starts_with(Prefix: "storent.") \|\| // Added in 3.9
447	Name.starts_with(Prefix: "vbroadcast.s") \|\| // Added in 7.0
448	Name.starts_with(Prefix: "vpshld.") \|\| // Added in 8.0
449	Name.starts_with(Prefix: "vpshrd.")); // Added in 8.0
450	}
451
452	if (Name.consume_front(Prefix: "fma."))
453	return (Name.starts_with(Prefix: "vfmadd.") \|\| // Added in 7.0
454	Name.starts_with(Prefix: "vfmsub.") \|\| // Added in 7.0
455	Name.starts_with(Prefix: "vfmsubadd.") \|\| // Added in 7.0
456	Name.starts_with(Prefix: "vfnmadd.") \|\| // Added in 7.0
457	Name.starts_with(Prefix: "vfnmsub.")); // Added in 7.0
458
459	if (Name.consume_front(Prefix: "fma4."))
460	return Name.starts_with(Prefix: "vfmadd.s"); // Added in 7.0
461
462	if (Name.consume_front(Prefix: "sse."))
463	return (Name == "add.ss" \|\| // Added in 4.0
464	Name == "cvtsi2ss" \|\| // Added in 7.0
465	Name == "cvtsi642ss" \|\| // Added in 7.0
466	Name == "div.ss" \|\| // Added in 4.0
467	Name == "mul.ss" \|\| // Added in 4.0
468	Name.starts_with(Prefix: "sqrt.p") \|\| // Added in 7.0
469	Name == "sqrt.ss" \|\| // Added in 7.0
470	Name.starts_with(Prefix: "storeu.") \|\| // Added in 3.9
471	Name == "sub.ss"); // Added in 4.0
472
473	if (Name.consume_front(Prefix: "sse2."))
474	return (Name == "add.sd" \|\| // Added in 4.0
475	Name == "cvtdq2pd" \|\| // Added in 3.9
476	Name == "cvtdq2ps" \|\| // Added in 7.0
477	Name == "cvtps2pd" \|\| // Added in 3.9
478	Name == "cvtsi2sd" \|\| // Added in 7.0
479	Name == "cvtsi642sd" \|\| // Added in 7.0
480	Name == "cvtss2sd" \|\| // Added in 7.0
481	Name == "div.sd" \|\| // Added in 4.0
482	Name == "mul.sd" \|\| // Added in 4.0
483	Name.starts_with(Prefix: "padds.") \|\| // Added in 8.0
484	Name.starts_with(Prefix: "paddus.") \|\| // Added in 8.0
485	Name.starts_with(Prefix: "pcmpeq.") \|\| // Added in 3.1
486	Name.starts_with(Prefix: "pcmpgt.") \|\| // Added in 3.1
487	Name == "pmaxs.w" \|\| // Added in 3.9
488	Name == "pmaxu.b" \|\| // Added in 3.9
489	Name == "pmins.w" \|\| // Added in 3.9
490	Name == "pminu.b" \|\| // Added in 3.9
491	Name == "pmulu.dq" \|\| // Added in 7.0
492	Name.starts_with(Prefix: "pshuf") \|\| // Added in 3.9
493	Name.starts_with(Prefix: "psll.dq") \|\| // Added in 3.7
494	Name.starts_with(Prefix: "psrl.dq") \|\| // Added in 3.7
495	Name.starts_with(Prefix: "psubs.") \|\| // Added in 8.0
496	Name.starts_with(Prefix: "psubus.") \|\| // Added in 8.0
497	Name.starts_with(Prefix: "sqrt.p") \|\| // Added in 7.0
498	Name == "sqrt.sd" \|\| // Added in 7.0
499	Name == "storel.dq" \|\| // Added in 3.9
500	Name.starts_with(Prefix: "storeu.") \|\| // Added in 3.9
501	Name == "sub.sd"); // Added in 4.0
502
503	if (Name.consume_front(Prefix: "sse41."))
504	return (Name.starts_with(Prefix: "blendp") \|\| // Added in 3.7
505	Name == "movntdqa" \|\| // Added in 5.0
506	Name == "pblendw" \|\| // Added in 3.7
507	Name == "pmaxsb" \|\| // Added in 3.9
508	Name == "pmaxsd" \|\| // Added in 3.9
509	Name == "pmaxud" \|\| // Added in 3.9
510	Name == "pmaxuw" \|\| // Added in 3.9
511	Name == "pminsb" \|\| // Added in 3.9
512	Name == "pminsd" \|\| // Added in 3.9
513	Name == "pminud" \|\| // Added in 3.9
514	Name == "pminuw" \|\| // Added in 3.9
515	Name.starts_with(Prefix: "pmovsx") \|\| // Added in 3.8
516	Name.starts_with(Prefix: "pmovzx") \|\| // Added in 3.9
517	Name == "pmuldq"); // Added in 7.0
518
519	if (Name.consume_front(Prefix: "sse42."))
520	return Name == "crc32.64.8"; // Added in 3.4
521
522	if (Name.consume_front(Prefix: "sse4a."))
523	return Name.starts_with(Prefix: "movnt."); // Added in 3.9
524
525	if (Name.consume_front(Prefix: "ssse3."))
526	return (Name == "pabs.b.128" \|\| // Added in 6.0
527	Name == "pabs.d.128" \|\| // Added in 6.0
528	Name == "pabs.w.128"); // Added in 6.0
529
530	if (Name.consume_front(Prefix: "xop."))
531	return (Name == "vpcmov" \|\| // Added in 3.8
532	Name == "vpcmov.256" \|\| // Added in 5.0
533	Name.starts_with(Prefix: "vpcom") \|\| // Added in 3.2, Updated in 9.0
534	Name.starts_with(Prefix: "vprot")); // Added in 8.0
535
536	if (Name.consume_front(Prefix: "bmi."))
537	return (Name.starts_with(Prefix: "pdep.") \|\| // Added in 23.0
538	Name.starts_with(Prefix: "pext.")); // Added in 23.0
539
540	return (Name == "addcarry.u32" \|\| // Added in 8.0
541	Name == "addcarry.u64" \|\| // Added in 8.0
542	Name == "addcarryx.u32" \|\| // Added in 8.0
543	Name == "addcarryx.u64" \|\| // Added in 8.0
544	Name == "subborrow.u32" \|\| // Added in 8.0
545	Name == "subborrow.u64" \|\| // Added in 8.0
546	Name.starts_with(Prefix: "vcvtph2ps.")); // Added in 11.0
547	}
548
549	static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
550	Function *&NewFn) {
551	// Only handle intrinsics that start with "x86.".
552	if (!Name.consume_front(Prefix: "x86."))
553	return false;
554
555	if (shouldUpgradeX86Intrinsic(F, Name)) {
556	NewFn = nullptr;
557	return true;
558	}
559
560	if (Name == "rdtscp") { // Added in 8.0
561	// If this intrinsic has 0 operands, it's the new version.
562	if (F->getFunctionType()->getNumParams() == `0`)
563	return false;
564
565	rename(GV: F);
566	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
567	id: Intrinsic::x86_rdtscp);
568	return true;
569	}
570
571	Intrinsic::ID ID;
572
573	// SSE4.1 ptest functions may have an old signature.
574	if (Name.consume_front(Prefix: "sse41.ptest")) { // Added in 3.2
575	ID = StringSwitch<Intrinsic::ID>(Name)
576	.Case(S: "c", Value: Intrinsic::x86_sse41_ptestc)
577	.Case(S: "z", Value: Intrinsic::x86_sse41_ptestz)
578	.Case(S: "nzc", Value: Intrinsic::x86_sse41_ptestnzc)
579	.Default(Value: Intrinsic::not_intrinsic);
580	if (ID != Intrinsic::not_intrinsic)
581	return upgradePTESTIntrinsic(F, IID: ID, NewFn);
582
583	return false;
584	}
585
586	// Several blend and other instructions with masks used the wrong number of
587	// bits.
588
589	// Added in 3.6
590	ID = StringSwitch<Intrinsic::ID>(Name)
591	.Case(S: "sse41.insertps", Value: Intrinsic::x86_sse41_insertps)
592	.Case(S: "sse41.dppd", Value: Intrinsic::x86_sse41_dppd)
593	.Case(S: "sse41.dpps", Value: Intrinsic::x86_sse41_dpps)
594	.Case(S: "sse41.mpsadbw", Value: Intrinsic::x86_sse41_mpsadbw)
595	.Case(S: "avx.dp.ps.256", Value: Intrinsic::x86_avx_dp_ps_256)
596	.Case(S: "avx2.mpsadbw", Value: Intrinsic::x86_avx2_mpsadbw)
597	.Default(Value: Intrinsic::not_intrinsic);
598	if (ID != Intrinsic::not_intrinsic)
599	return upgradeX86IntrinsicsWith8BitMask(F, IID: ID, NewFn);
600
601	if (Name.consume_front(Prefix: "avx512.")) {
602	if (Name.consume_front(Prefix: "mask.cmp.")) {
603	// Added in 7.0
604	ID = StringSwitch<Intrinsic::ID>(Name)
605	.Case(S: "pd.128", Value: Intrinsic::x86_avx512_mask_cmp_pd_128)
606	.Case(S: "pd.256", Value: Intrinsic::x86_avx512_mask_cmp_pd_256)
607	.Case(S: "pd.512", Value: Intrinsic::x86_avx512_mask_cmp_pd_512)
608	.Case(S: "ps.128", Value: Intrinsic::x86_avx512_mask_cmp_ps_128)
609	.Case(S: "ps.256", Value: Intrinsic::x86_avx512_mask_cmp_ps_256)
610	.Case(S: "ps.512", Value: Intrinsic::x86_avx512_mask_cmp_ps_512)
611	.Default(Value: Intrinsic::not_intrinsic);
612	if (ID != Intrinsic::not_intrinsic)
613	return upgradeX86MaskedFPCompare(F, IID: ID, NewFn);
614	} else if (Name.starts_with(Prefix: "vpdpbusd.") \|\|
615	Name.starts_with(Prefix: "vpdpbusds.")) {
616	// Added in 21.1
617	ID = StringSwitch<Intrinsic::ID>(Name)
618	.Case(S: "vpdpbusd.128", Value: Intrinsic::x86_avx512_vpdpbusd_128)
619	.Case(S: "vpdpbusd.256", Value: Intrinsic::x86_avx512_vpdpbusd_256)
620	.Case(S: "vpdpbusd.512", Value: Intrinsic::x86_avx512_vpdpbusd_512)
621	.Case(S: "vpdpbusds.128", Value: Intrinsic::x86_avx512_vpdpbusds_128)
622	.Case(S: "vpdpbusds.256", Value: Intrinsic::x86_avx512_vpdpbusds_256)
623	.Case(S: "vpdpbusds.512", Value: Intrinsic::x86_avx512_vpdpbusds_512)
624	.Default(Value: Intrinsic::not_intrinsic);
625	if (ID != Intrinsic::not_intrinsic)
626	return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
627	} else if (Name.starts_with(Prefix: "vpdpwssd.") \|\|
628	Name.starts_with(Prefix: "vpdpwssds.")) {
629	// Added in 21.1
630	ID = StringSwitch<Intrinsic::ID>(Name)
631	.Case(S: "vpdpwssd.128", Value: Intrinsic::x86_avx512_vpdpwssd_128)
632	.Case(S: "vpdpwssd.256", Value: Intrinsic::x86_avx512_vpdpwssd_256)
633	.Case(S: "vpdpwssd.512", Value: Intrinsic::x86_avx512_vpdpwssd_512)
634	.Case(S: "vpdpwssds.128", Value: Intrinsic::x86_avx512_vpdpwssds_128)
635	.Case(S: "vpdpwssds.256", Value: Intrinsic::x86_avx512_vpdpwssds_256)
636	.Case(S: "vpdpwssds.512", Value: Intrinsic::x86_avx512_vpdpwssds_512)
637	.Default(Value: Intrinsic::not_intrinsic);
638	if (ID != Intrinsic::not_intrinsic)
639	return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
640	}
641	return false; // No other 'x86.avx512.'.*
642	}
643
644	if (Name.consume_front(Prefix: "avx2.")) {
645	if (Name.consume_front(Prefix: "vpdpb")) {
646	// Added in 21.1
647	ID = StringSwitch<Intrinsic::ID>(Name)
648	.Case(S: "ssd.128", Value: Intrinsic::x86_avx2_vpdpbssd_128)
649	.Case(S: "ssd.256", Value: Intrinsic::x86_avx2_vpdpbssd_256)
650	.Case(S: "ssds.128", Value: Intrinsic::x86_avx2_vpdpbssds_128)
651	.Case(S: "ssds.256", Value: Intrinsic::x86_avx2_vpdpbssds_256)
652	.Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpbsud_128)
653	.Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpbsud_256)
654	.Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpbsuds_128)
655	.Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpbsuds_256)
656	.Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpbuud_128)
657	.Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpbuud_256)
658	.Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpbuuds_128)
659	.Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpbuuds_256)
660	.Default(Value: Intrinsic::not_intrinsic);
661	if (ID != Intrinsic::not_intrinsic)
662	return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
663	} else if (Name.consume_front(Prefix: "vpdpw")) {
664	// Added in 21.1
665	ID = StringSwitch<Intrinsic::ID>(Name)
666	.Case(S: "sud.128", Value: Intrinsic::x86_avx2_vpdpwsud_128)
667	.Case(S: "sud.256", Value: Intrinsic::x86_avx2_vpdpwsud_256)
668	.Case(S: "suds.128", Value: Intrinsic::x86_avx2_vpdpwsuds_128)
669	.Case(S: "suds.256", Value: Intrinsic::x86_avx2_vpdpwsuds_256)
670	.Case(S: "usd.128", Value: Intrinsic::x86_avx2_vpdpwusd_128)
671	.Case(S: "usd.256", Value: Intrinsic::x86_avx2_vpdpwusd_256)
672	.Case(S: "usds.128", Value: Intrinsic::x86_avx2_vpdpwusds_128)
673	.Case(S: "usds.256", Value: Intrinsic::x86_avx2_vpdpwusds_256)
674	.Case(S: "uud.128", Value: Intrinsic::x86_avx2_vpdpwuud_128)
675	.Case(S: "uud.256", Value: Intrinsic::x86_avx2_vpdpwuud_256)
676	.Case(S: "uuds.128", Value: Intrinsic::x86_avx2_vpdpwuuds_128)
677	.Case(S: "uuds.256", Value: Intrinsic::x86_avx2_vpdpwuuds_256)
678	.Default(Value: Intrinsic::not_intrinsic);
679	if (ID != Intrinsic::not_intrinsic)
680	return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
681	}
682	return false; // No other 'x86.avx2.'*
683	}
684
685	if (Name.consume_front(Prefix: "avx10.")) {
686	if (Name.consume_front(Prefix: "vpdpb")) {
687	// Added in 21.1
688	ID = StringSwitch<Intrinsic::ID>(Name)
689	.Case(S: "ssd.512", Value: Intrinsic::x86_avx10_vpdpbssd_512)
690	.Case(S: "ssds.512", Value: Intrinsic::x86_avx10_vpdpbssds_512)
691	.Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpbsud_512)
692	.Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpbsuds_512)
693	.Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpbuud_512)
694	.Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpbuuds_512)
695	.Default(Value: Intrinsic::not_intrinsic);
696	if (ID != Intrinsic::not_intrinsic)
697	return upgradeX86MultiplyAddBytes(F, IID: ID, NewFn);
698	} else if (Name.consume_front(Prefix: "vpdpw")) {
699	ID = StringSwitch<Intrinsic::ID>(Name)
700	.Case(S: "sud.512", Value: Intrinsic::x86_avx10_vpdpwsud_512)
701	.Case(S: "suds.512", Value: Intrinsic::x86_avx10_vpdpwsuds_512)
702	.Case(S: "usd.512", Value: Intrinsic::x86_avx10_vpdpwusd_512)
703	.Case(S: "usds.512", Value: Intrinsic::x86_avx10_vpdpwusds_512)
704	.Case(S: "uud.512", Value: Intrinsic::x86_avx10_vpdpwuud_512)
705	.Case(S: "uuds.512", Value: Intrinsic::x86_avx10_vpdpwuuds_512)
706	.Default(Value: Intrinsic::not_intrinsic);
707	if (ID != Intrinsic::not_intrinsic)
708	return upgradeX86MultiplyAddWords(F, IID: ID, NewFn);
709	}
710	return false; // No other 'x86.avx10.'*
711	}
712
713	if (Name.consume_front(Prefix: "avx512bf16.")) {
714	// Added in 9.0
715	ID = StringSwitch<Intrinsic::ID>(Name)
716	.Case(S: "cvtne2ps2bf16.128",
717	Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
718	.Case(S: "cvtne2ps2bf16.256",
719	Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
720	.Case(S: "cvtne2ps2bf16.512",
721	Value: Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
722	.Case(S: "mask.cvtneps2bf16.128",
723	Value: Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
724	.Case(S: "cvtneps2bf16.256",
725	Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
726	.Case(S: "cvtneps2bf16.512",
727	Value: Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
728	.Default(Value: Intrinsic::not_intrinsic);
729	if (ID != Intrinsic::not_intrinsic)
730	return upgradeX86BF16Intrinsic(F, IID: ID, NewFn);
731
732	// Added in 9.0
733	ID = StringSwitch<Intrinsic::ID>(Name)
734	.Case(S: "dpbf16ps.128", Value: Intrinsic::x86_avx512bf16_dpbf16ps_128)
735	.Case(S: "dpbf16ps.256", Value: Intrinsic::x86_avx512bf16_dpbf16ps_256)
736	.Case(S: "dpbf16ps.512", Value: Intrinsic::x86_avx512bf16_dpbf16ps_512)
737	.Default(Value: Intrinsic::not_intrinsic);
738	if (ID != Intrinsic::not_intrinsic)
739	return upgradeX86BF16DPIntrinsic(F, IID: ID, NewFn);
740	return false; // No other 'x86.avx512bf16.'.*
741	}
742
743	if (Name.consume_front(Prefix: "xop.")) {
744	Intrinsic::ID ID = Intrinsic::not_intrinsic;
745	if (Name.starts_with(Prefix: "vpermil2")) { // Added in 3.9
746	// Upgrade any XOP PERMIL2 index operand still using a float/double
747	// vector.
748	auto Idx = F->getFunctionType()->getParamType(i: `2`);
749	if (Idx->isFPOrFPVectorTy()) {
750	unsigned IdxSize = Idx->getPrimitiveSizeInBits();
751	unsigned EltSize = Idx->getScalarSizeInBits();
752	if (EltSize == `64` && IdxSize == `128`)
753	ID = Intrinsic::x86_xop_vpermil2pd;
754	else if (EltSize == `32` && IdxSize == `128`)
755	ID = Intrinsic::x86_xop_vpermil2ps;
756	else if (EltSize == `64` && IdxSize == `256`)
757	ID = Intrinsic::x86_xop_vpermil2pd_256;
758	else
759	ID = Intrinsic::x86_xop_vpermil2ps_256;
760	}
761	} else if (F->arg_size() == `2`)
762	// frcz.ss/sd may need to have an argument dropped. Added in 3.2
763	ID = StringSwitch<Intrinsic::ID>(Name)
764	.Case(S: "vfrcz.ss", Value: Intrinsic::x86_xop_vfrcz_ss)
765	.Case(S: "vfrcz.sd", Value: Intrinsic::x86_xop_vfrcz_sd)
766	.Default(Value: Intrinsic::not_intrinsic);
767
768	if (ID != Intrinsic::not_intrinsic) {
769	rename(GV: F);
770	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
771	return true;
772	}
773	return false; // No other 'x86.xop.'*
774	}
775
776	if (Name == "seh.recoverfp") {
777	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
778	id: Intrinsic::eh_recoverfp);
779	return true;
780	}
781
782	return false;
783	}
784
785	// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
786	// IsArm: 'arm.', !IsArm: 'aarch64.'.
787	static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
788	StringRef Name,
789	Function *&NewFn) {
790	if (Name.starts_with(Prefix: "rbit")) {
791	// '(arm\|aarch64).rbit'.
792	NewFn = Intrinsic::getOrInsertDeclaration(
793	M: F->getParent(), id: Intrinsic::bitreverse, OverloadTys: F->arg_begin()->getType());
794	return true;
795	}
796
797	if (Name == "thread.pointer") {
798	// '(arm\|aarch64).thread.pointer'.
799	NewFn = Intrinsic::getOrInsertDeclaration(
800	M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
801	return true;
802	}
803
804	bool Neon = Name.consume_front(Prefix: "neon.");
805	if (Neon) {
806	// '(arm\|aarch64).neon.'.*
807	// Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
808	// v16i8 respectively.
809	if (Name.consume_front(Prefix: "bfdot.")) {
810	// (arm\|aarch64).neon.bfdot.'.*
811	Intrinsic::ID ID =
812	StringSwitch<Intrinsic::ID>(Name)
813	.Cases(CaseStrings: {"v2f32.v8i8", "v4f32.v16i8"},
814	Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
815	: (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
816	.Default(Value: Intrinsic::not_intrinsic);
817	if (ID != Intrinsic::not_intrinsic) {
818	size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
819	assert((OperandWidth == `64` \|\| OperandWidth == `128`) &&
820	"Unexpected operand width");
821	LLVMContext &Ctx = F->getParent()->getContext();
822	std::array<Type *, `2`> Tys{
823	._M_elems: {F->getReturnType(),
824	FixedVectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), NumElts: OperandWidth / `16`)}};
825	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
826	return true;
827	}
828	return false; // No other '(arm\|aarch64).neon.bfdot.'.*
829	}
830
831	// Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
832	// anymore and accept v8bf16 instead of v16i8.
833	if (Name.consume_front(Prefix: "bfm")) {
834	// (arm\|aarch64).neon.bfm'.*
835	if (Name.consume_back(Suffix: ".v4f32.v16i8")) {
836	// (arm\|aarch64).neon.bfm.v4f32.v16i8'.*
837	Intrinsic::ID ID =
838	StringSwitch<Intrinsic::ID>(Name)
839	.Case(S: "mla",
840	Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
841	: (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
842	.Case(S: "lalb",
843	Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
844	: (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
845	.Case(S: "lalt",
846	Value: IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
847	: (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
848	.Default(Value: Intrinsic::not_intrinsic);
849	if (ID != Intrinsic::not_intrinsic) {
850	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
851	return true;
852	}
853	return false; // No other '(arm\|aarch64).neon.bfm.v16i8'.*
854	}
855	return false; // No other '(arm\|aarch64).neon.bfm.*
856	}
857	// Continue on to Aarch64 Neon or Arm Neon.
858	}
859	// Continue on to Arm or Aarch64.
860
861	if (IsArm) {
862	// 'arm.'.*
863	if (Neon) {
864	// 'arm.neon.'.*
865	Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
866	.StartsWith(S: "vclz.", Value: Intrinsic::ctlz)
867	.StartsWith(S: "vcnt.", Value: Intrinsic::ctpop)
868	.StartsWith(S: "vqadds.", Value: Intrinsic::sadd_sat)
869	.StartsWith(S: "vqaddu.", Value: Intrinsic::uadd_sat)
870	.StartsWith(S: "vqsubs.", Value: Intrinsic::ssub_sat)
871	.StartsWith(S: "vqsubu.", Value: Intrinsic::usub_sat)
872	.StartsWith(S: "vrinta.", Value: Intrinsic::round)
873	.StartsWith(S: "vrintn.", Value: Intrinsic::roundeven)
874	.StartsWith(S: "vrintm.", Value: Intrinsic::floor)
875	.StartsWith(S: "vrintp.", Value: Intrinsic::ceil)
876	.StartsWith(S: "vrintx.", Value: Intrinsic::rint)
877	.StartsWith(S: "vrintz.", Value: Intrinsic::trunc)
878	.Default(Value: Intrinsic::not_intrinsic);
879	if (ID != Intrinsic::not_intrinsic) {
880	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
881	OverloadTys: F->arg_begin()->getType());
882	return true;
883	}
884
885	if (Name.consume_front(Prefix: "vst")) {
886	// 'arm.neon.vst'.*
887	static const Regex vstRegex("^([1234]\|[234]lane)\\.v[a-z0-9]*$");
888	SmallVector<StringRef, `2`> Groups;
889	if (vstRegex.match(String: Name, Matches: &Groups)) {
890	static const Intrinsic::ID StoreInts[] = {
891	Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
892	Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
893
894	static const Intrinsic::ID StoreLaneInts[] = {
895	Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
896	Intrinsic::arm_neon_vst4lane};
897
898	auto fArgs = F->getFunctionType()->params();
899	Type *Tys[] = {fArgs [`0`], fArgs [`1`]};
900	if (Groups [`1`].size() == `1`)
901	NewFn = Intrinsic::getOrInsertDeclaration(
902	M: F->getParent(), id: StoreInts[fArgs.size() - `3`], OverloadTys: Tys);
903	else
904	NewFn = Intrinsic::getOrInsertDeclaration(
905	M: F->getParent(), id: StoreLaneInts[fArgs.size() - `5`], OverloadTys: Tys);
906	return true;
907	}
908	return false; // No other 'arm.neon.vst'.*
909	}
910
911	return false; // No other 'arm.neon.'.*
912	}
913
914	if (Name.consume_front(Prefix: "mve.")) {
915	// 'arm.mve.'.*
916	if (Name == "vctp64") {
917	if (cast<FixedVectorType>(Val: F->getReturnType())->getNumElements() == `4`) {
918	// A vctp64 returning a v4i1 is converted to return a v2i1. Rename
919	// the function and deal with it below in UpgradeIntrinsicCall.
920	rename(GV: F);
921	return true;
922	}
923	return false; // Not 'arm.mve.vctp64'.
924	}
925
926	if (Name.starts_with(Prefix: "vrintn.v")) {
927	NewFn = Intrinsic::getOrInsertDeclaration(
928	M: F->getParent(), id: Intrinsic::roundeven, OverloadTys: F->arg_begin()->getType());
929	return true;
930	}
931
932	// These too are changed to accept a v2i1 instead of the old v4i1.
933	if (Name.consume_back(Suffix: ".v4i1")) {
934	// 'arm.mve..v4i1'.*
935	if (Name.consume_back(Suffix: ".predicated.v2i64.v4i32"))
936	// 'arm.mve..predicated.v2i64.v4i32.v4i1'*
937	return Name == "mull.int" \|\| Name == "vqdmull";
938
939	if (Name.consume_back(Suffix: ".v2i64")) {
940	// 'arm.mve..v2i64.v4i1'*
941	bool IsGather = Name.consume_front(Prefix: "vldr.gather.");
942	if (IsGather \|\| Name.consume_front(Prefix: "vstr.scatter.")) {
943	if (Name.consume_front(Prefix: "base.")) {
944	// Optional 'wb.' prefix.
945	Name.consume_front(Prefix: "wb.");
946	// 'arm.mve.(vldr.gather\|vstr.scatter).base.(wb.)?
947	// predicated.v2i64.v2i64.v4i1'.
948	return Name == "predicated.v2i64";
949	}
950
951	if (Name.consume_front(Prefix: "offset.predicated."))
952	return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") \|\|
953	Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
954
955	// No other 'arm.mve.(vldr.gather\|vstr.scatter)..v2i64.v4i1'.*
956	return false;
957	}
958
959	return false; // No other 'arm.mve..v2i64.v4i1'.*
960	}
961	return false; // No other 'arm.mve..v4i1'.*
962	}
963	return false; // No other 'arm.mve.'.*
964	}
965
966	if (Name.consume_front(Prefix: "cde.vcx")) {
967	// 'arm.cde.vcx'.*
968	if (Name.consume_back(Suffix: ".predicated.v2i64.v4i1"))
969	// 'arm.cde.vcx.predicated.v2i64.v4i1'.*
970	return Name == "1q" \|\| Name == "1qa" \|\| Name == "2q" \|\| Name == "2qa" \|\|
971	Name == "3q" \|\| Name == "3qa";
972
973	return false; // No other 'arm.cde.vcx'.*
974	}
975	} else {
976	// 'aarch64.'.*
977	if (Neon) {
978	// 'aarch64.neon.'.*
979	Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
980	.StartsWith(S: "frintn", Value: Intrinsic::roundeven)
981	.StartsWith(S: "rbit", Value: Intrinsic::bitreverse)
982	.Default(Value: Intrinsic::not_intrinsic);
983	if (ID != Intrinsic::not_intrinsic) {
984	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
985	OverloadTys: F->arg_begin()->getType());
986	return true;
987	}
988
989	if (Name.starts_with(Prefix: "addp")) {
990	// 'aarch64.neon.addp'.*
991	if (F->arg_size() != `2`)
992	return false; // Invalid IR.
993	VectorType *Ty = dyn_cast<VectorType>(Val: F->getReturnType());
994	if (Ty && Ty->getElementType()->isFloatingPointTy()) {
995	NewFn = Intrinsic::getOrInsertDeclaration(
996	M: F->getParent(), id: Intrinsic::aarch64_neon_faddp, OverloadTys: Ty);
997	return true;
998	}
999	}
1000
1001	// Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1002	if (Name.starts_with(Prefix: "bfcvt")) {
1003	NewFn = nullptr;
1004	return true;
1005	}
1006
1007	// vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1008	if (Name == "vcvtfp2hf" \|\| Name == "vcvthf2fp") {
1009	NewFn = nullptr;
1010	return true;
1011	}
1012
1013	return false; // No other 'aarch64.neon.'.*
1014	}
1015	if (Name.consume_front(Prefix: "sve.")) {
1016	// 'aarch64.sve.'.*
1017	if (Name.consume_front(Prefix: "bf")) {
1018	if (Name == "mmla") {
1019	Type *Tys[] = {F->getReturnType(),
1020	std::next(x: F->arg_begin())->getType()};
1021	NewFn = Intrinsic::getOrInsertDeclaration(
1022	M: F->getParent(), id: Intrinsic::aarch64_sve_fmmla, OverloadTys: Tys);
1023	return true;
1024	}
1025	if (Name.consume_back(Suffix: ".lane")) {
1026	// 'aarch64.sve.bf.lane'.*
1027	Intrinsic::ID ID =
1028	StringSwitch<Intrinsic::ID>(Name)
1029	.Case(S: "dot", Value: Intrinsic::aarch64_sve_bfdot_lane_v2)
1030	.Case(S: "mlalb", Value: Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1031	.Case(S: "mlalt", Value: Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1032	.Default(Value: Intrinsic::not_intrinsic);
1033	if (ID != Intrinsic::not_intrinsic) {
1034	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1035	return true;
1036	}
1037	return false; // No other 'aarch64.sve.bf.lane'.*
1038	}
1039	return false; // No other 'aarch64.sve.bf'.*
1040	}
1041
1042	// 'aarch64.sve.fcvt.bf16f32' \|\| 'aarch64.sve.fcvtnt.bf16f32'
1043	if (Name == "fcvt.bf16f32" \|\| Name == "fcvtnt.bf16f32") {
1044	NewFn = nullptr;
1045	return true;
1046	}
1047
1048	if (Name.consume_front(Prefix: "addqv")) {
1049	// 'aarch64.sve.addqv'.
1050	if (!F->getReturnType()->isFPOrFPVectorTy())
1051	return false;
1052
1053	auto Args = F->getFunctionType()->params();
1054	Type *Tys[] = {F->getReturnType(), Args [`1`]};
1055	NewFn = Intrinsic::getOrInsertDeclaration(
1056	M: F->getParent(), id: Intrinsic::aarch64_sve_faddqv, OverloadTys: Tys);
1057	return true;
1058	}
1059
1060	if (Name.consume_front(Prefix: "ld")) {
1061	// 'aarch64.sve.ld'.*
1062	static const Regex LdRegex("^[234](.nxv[a-z0-9]+\|$)");
1063	if (LdRegex.match(String: Name)) {
1064	Type *ScalarTy =
1065	cast<VectorType>(Val: F->getReturnType())->getElementType();
1066	ElementCount EC =
1067	cast<VectorType>(Val: F->arg_begin()->getType())->getElementCount();
1068	assert(F->arg_size() == `2` &&
1069	"Expected 2 arguments for ld* intrinsic.");
1070	Type *PtrTy = F->getArg(i: `1`)->getType();
1071	Type *Ty = VectorType::get(ElementType: ScalarTy, EC);
1072	static const Intrinsic::ID LoadIDs[] = {
1073	Intrinsic::aarch64_sve_ld2_sret,
1074	Intrinsic::aarch64_sve_ld3_sret,
1075	Intrinsic::aarch64_sve_ld4_sret,
1076	};
1077	NewFn = Intrinsic::getOrInsertDeclaration(
1078	M: F->getParent(), id: LoadIDs[Name [`0`] - `'2'`], OverloadTys: {Ty, PtrTy});
1079	return true;
1080	}
1081	return false; // No other 'aarch64.sve.ld'.*
1082	}
1083
1084	if (Name.consume_front(Prefix: "tuple.")) {
1085	// 'aarch64.sve.tuple.'.*
1086	if (Name.starts_with(Prefix: "get")) {
1087	// 'aarch64.sve.tuple.get'.*
1088	Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1089	NewFn = Intrinsic::getOrInsertDeclaration(
1090	M: F->getParent(), id: Intrinsic::vector_extract, OverloadTys: Tys);
1091	return true;
1092	}
1093
1094	if (Name.starts_with(Prefix: "set")) {
1095	// 'aarch64.sve.tuple.set'.*
1096	auto Args = F->getFunctionType()->params();
1097	Type *Tys[] = {Args [`0`], Args [`2`], Args [`1`]};
1098	NewFn = Intrinsic::getOrInsertDeclaration(
1099	M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1100	return true;
1101	}
1102
1103	static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+\|$)");
1104	if (CreateTupleRegex.match(String: Name)) {
1105	// 'aarch64.sve.tuple.create'.*
1106	auto Args = F->getFunctionType()->params();
1107	Type *Tys[] = {F->getReturnType(), Args [`1`]};
1108	NewFn = Intrinsic::getOrInsertDeclaration(
1109	M: F->getParent(), id: Intrinsic::vector_insert, OverloadTys: Tys);
1110	return true;
1111	}
1112	return false; // No other 'aarch64.sve.tuple.'.*
1113	}
1114
1115	if (Name.starts_with(Prefix: "rev.nxv")) {
1116	// 'aarch64.sve.rev.<Ty>'
1117	NewFn = Intrinsic::getOrInsertDeclaration(
1118	M: F->getParent(), id: Intrinsic::vector_reverse, OverloadTys: F->getReturnType());
1119	return true;
1120	}
1121
1122	return false; // No other 'aarch64.sve.'.*
1123	}
1124	if (Name.consume_front(Prefix: "sme.")) {
1125	// 'aarch64.sme.'.*
1126	if (Name.consume_front(Prefix: "ftmopa.")) {
1127	// The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1128	// intrinsics to model their FPMR dependency.
1129	Intrinsic::ID ID =
1130	StringSwitch<Intrinsic::ID>(Name)
1131	.Case(S: "za16.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1132	.Case(S: "za32.nxv16i8", Value: Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1133	.Default(Value: Intrinsic::not_intrinsic);
1134	if (ID != Intrinsic::not_intrinsic) {
1135	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1136	return true;
1137	}
1138	return false; // No other 'aarch64.sme.ftmopa.'.*
1139	}
1140
1141	return false; // No other 'aarch64.sme.'.*
1142	}
1143	}
1144	return false; // No other 'arm.', 'aarch64.'.
1145	}
1146
1147	static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1148	StringRef Name) {
1149	if (Name.consume_front(Prefix: "cp.async.bulk.tensor.g2s.")) {
1150	Intrinsic::ID ID =
1151	StringSwitch<Intrinsic::ID>(Name)
1152	.Case(S: "im2col.3d",
1153	Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1154	.Case(S: "im2col.4d",
1155	Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1156	.Case(S: "im2col.5d",
1157	Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1158	.Case(S: "tile.1d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1159	.Case(S: "tile.2d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1160	.Case(S: "tile.3d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1161	.Case(S: "tile.4d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1162	.Case(S: "tile.5d", Value: Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1163	.Default(Value: Intrinsic::not_intrinsic);
1164
1165	if (ID == Intrinsic::not_intrinsic)
1166	return ID;
1167
1168	// These intrinsics may need upgrade for two reasons:
1169	// (1) When the address-space of the first argument is shared[AS=3]
1170	// (and we upgrade it to use shared_cluster address-space[AS=7])
1171	if (F->getArg(i: `0`)->getType()->getPointerAddressSpace() ==
1172	NVPTXAS::ADDRESS_SPACE_SHARED)
1173	return ID;
1174
1175	// (2) When there are only two boolean flag arguments at the end:
1176	//
1177	// The last three parameters of the older version of these
1178	// intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1179	//
1180	// The newer version reads as:
1181	// arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1182	//
1183	// So, when the type of the [N-3]rd argument is "not i1", then
1184	// it is the older version and we need to upgrade.
1185	size_t FlagStartIndex = F->getFunctionType()->getNumParams() - `3`;
1186	Type *ArgType = F->getFunctionType()->getParamType(i: FlagStartIndex);
1187	if (!ArgType->isIntegerTy(BitWidth: `1`))
1188	return ID;
1189	}
1190
1191	return Intrinsic::not_intrinsic;
1192	}
1193
1194	static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1195	StringRef Name) {
1196	if (Name.consume_front(Prefix: "mapa.shared.cluster"))
1197	if (F->getReturnType()->getPointerAddressSpace() ==
1198	NVPTXAS::ADDRESS_SPACE_SHARED)
1199	return Intrinsic::nvvm_mapa_shared_cluster;
1200
1201	if (Name.consume_front(Prefix: "cp.async.bulk.")) {
1202	Intrinsic::ID ID =
1203	StringSwitch<Intrinsic::ID>(Name)
1204	.Case(S: "global.to.shared.cluster",
1205	Value: Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1206	.Case(S: "shared.cta.to.cluster",
1207	Value: Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1208	.Default(Value: Intrinsic::not_intrinsic);
1209
1210	if (ID != Intrinsic::not_intrinsic)
1211	if (F->getArg(i: `0`)->getType()->getPointerAddressSpace() ==
1212	NVPTXAS::ADDRESS_SPACE_SHARED)
1213	return ID;
1214	}
1215
1216	return Intrinsic::not_intrinsic;
1217	}
1218
1219	static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1220	if (Name.consume_front(Prefix: "fma.rn."))
1221	return StringSwitch<Intrinsic::ID>(Name)
1222	.Case(S: "bf16", Value: Intrinsic::nvvm_fma_rn_bf16)
1223	.Case(S: "bf16x2", Value: Intrinsic::nvvm_fma_rn_bf16x2)
1224	.Case(S: "relu.bf16", Value: Intrinsic::nvvm_fma_rn_relu_bf16)
1225	.Case(S: "relu.bf16x2", Value: Intrinsic::nvvm_fma_rn_relu_bf16x2)
1226	.Default(Value: Intrinsic::not_intrinsic);
1227
1228	if (Name.consume_front(Prefix: "fmax."))
1229	return StringSwitch<Intrinsic::ID>(Name)
1230	.Case(S: "bf16", Value: Intrinsic::nvvm_fmax_bf16)
1231	.Case(S: "bf16x2", Value: Intrinsic::nvvm_fmax_bf16x2)
1232	.Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmax_ftz_bf16)
1233	.Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_bf16x2)
1234	.Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16)
1235	.Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1236	.Case(S: "ftz.nan.xorsign.abs.bf16",
1237	Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1238	.Case(S: "ftz.nan.xorsign.abs.bf16x2",
1239	Value: Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1240	.Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1241	.Case(S: "ftz.xorsign.abs.bf16x2",
1242	Value: Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1243	.Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmax_nan_bf16)
1244	.Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmax_nan_bf16x2)
1245	.Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1246	.Case(S: "nan.xorsign.abs.bf16x2",
1247	Value: Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1248	.Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1249	.Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1250	.Default(Value: Intrinsic::not_intrinsic);
1251
1252	if (Name.consume_front(Prefix: "fmin."))
1253	return StringSwitch<Intrinsic::ID>(Name)
1254	.Case(S: "bf16", Value: Intrinsic::nvvm_fmin_bf16)
1255	.Case(S: "bf16x2", Value: Intrinsic::nvvm_fmin_bf16x2)
1256	.Case(S: "ftz.bf16", Value: Intrinsic::nvvm_fmin_ftz_bf16)
1257	.Case(S: "ftz.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_bf16x2)
1258	.Case(S: "ftz.nan.bf16", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16)
1259	.Case(S: "ftz.nan.bf16x2", Value: Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1260	.Case(S: "ftz.nan.xorsign.abs.bf16",
1261	Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1262	.Case(S: "ftz.nan.xorsign.abs.bf16x2",
1263	Value: Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1264	.Case(S: "ftz.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1265	.Case(S: "ftz.xorsign.abs.bf16x2",
1266	Value: Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1267	.Case(S: "nan.bf16", Value: Intrinsic::nvvm_fmin_nan_bf16)
1268	.Case(S: "nan.bf16x2", Value: Intrinsic::nvvm_fmin_nan_bf16x2)
1269	.Case(S: "nan.xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1270	.Case(S: "nan.xorsign.abs.bf16x2",
1271	Value: Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1272	.Case(S: "xorsign.abs.bf16", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1273	.Case(S: "xorsign.abs.bf16x2", Value: Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1274	.Default(Value: Intrinsic::not_intrinsic);
1275
1276	if (Name.consume_front(Prefix: "neg."))
1277	return StringSwitch<Intrinsic::ID>(Name)
1278	.Case(S: "bf16", Value: Intrinsic::nvvm_neg_bf16)
1279	.Case(S: "bf16x2", Value: Intrinsic::nvvm_neg_bf16x2)
1280	.Default(Value: Intrinsic::not_intrinsic);
1281
1282	return Intrinsic::not_intrinsic;
1283	}
1284
1285	static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1286	return Name.consume_front(Prefix: "local") \|\| Name.consume_front(Prefix: "shared") \|\|
1287	Name.consume_front(Prefix: "global") \|\| Name.consume_front(Prefix: "constant") \|\|
1288	Name.consume_front(Prefix: "param");
1289	}
1290
1291	static bool convertIntrinsicValidType(StringRef Name,
1292	const FunctionType *FuncTy) {
1293	Type *HalfTy = Type::getHalfTy(C&: FuncTy->getContext());
1294	if (Name.starts_with(Prefix: "to.fp16")) {
1295	return CastInst::castIsValid(op: Instruction::FPTrunc, SrcTy: FuncTy->getParamType(i: `0`),
1296	DstTy: HalfTy) &&
1297	CastInst::castIsValid(op: Instruction::BitCast, SrcTy: HalfTy,
1298	DstTy: FuncTy->getReturnType());
1299	}
1300
1301	if (Name.starts_with(Prefix: "from.fp16")) {
1302	return CastInst::castIsValid(op: Instruction::BitCast, SrcTy: FuncTy->getParamType(i: `0`),
1303	DstTy: HalfTy) &&
1304	CastInst::castIsValid(op: Instruction::FPExt, SrcTy: HalfTy,
1305	DstTy: FuncTy->getReturnType());
1306	}
1307
1308	return false;
1309	}
1310
1311	static bool upgradeIntrinsicDeclWithDefaultArgs(Function F, Function &NewFn) {
1312	Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(Name: F->getName());
1313	if (IID == Intrinsic::not_intrinsic)
1314	return false;
1315
1316	auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1317	if (Defaults.empty())
1318	return false;
1319
1320	// Overloaded intrinsics are out of scope for the default-arg feature
1321	// and will be supported in a follow-up.
1322	if (Intrinsic::isOverloaded(id: IID))
1323	return false;
1324
1325	// Get the canonical full declaration for this intrinsic.
1326	Function *FullDecl = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1327
1328	// If the existing declaration already has all args, nothing to upgrade
1329	if (F->arg_size() >= FullDecl->arg_size())
1330	return false;
1331
1332	// Defaults are a contiguous trailing block, so checking the first missing
1333	// argument is enough.
1334	if (F->arg_size() < FirstDefault)
1335	return false;
1336
1337	NewFn = FullDecl;
1338	return true;
1339	}
1340
1341	static bool upgradeIntrinsicFunction1(Function F, Function &NewFn,
1342	bool CanUpgradeDebugIntrinsicsToRecords) {
1343	assert(F && "Illegal to upgrade a non-existent Function.");
1344
1345	StringRef Name = F->getName();
1346
1347	// Quickly eliminate it, if it's not a candidate.
1348	if (!Name.consume_front(Prefix: "llvm.") \|\| Name.empty())
1349	return false;
1350
1351	switch (Name [`0`]) {
1352	default: break;
1353	case `'a'`: {
1354	bool IsArm = Name.consume_front(Prefix: "arm.");
1355	if (IsArm \|\| Name.consume_front(Prefix: "aarch64.")) {
1356	if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1357	return true;
1358	break;
1359	}
1360
1361	if (Name.consume_front(Prefix: "amdgcn.")) {
1362	if (Name == "alignbit") {
1363	// Target specific intrinsic became redundant
1364	NewFn = Intrinsic::getOrInsertDeclaration(
1365	M: F->getParent(), id: Intrinsic::fshr, OverloadTys: {F->getReturnType()});
1366	return true;
1367	}
1368
1369	if (Name.consume_front(Prefix: "atomic.")) {
1370	if (Name.starts_with(Prefix: "inc") \|\| Name.starts_with(Prefix: "dec") \|\|
1371	Name.starts_with(Prefix: "cond.sub") \|\| Name.starts_with(Prefix: "csub")) {
1372	// These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1373	// and usub_sat so there's no new declaration.
1374	NewFn = nullptr;
1375	return true;
1376	}
1377	break; // No other 'amdgcn.atomic.'*
1378	}
1379
1380	switch (F->getIntrinsicID()) {
1381	default:
1382	break;
1383	// Legacy wmma iu intrinsics without the optional clamp operand.
1384	case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1385	if (F->arg_size() == `7`) {
1386	NewFn = nullptr;
1387	return true;
1388	}
1389	break;
1390	case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1391	case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1392	case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1393	case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1394	case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1395	case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1396	case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1397	if (F->arg_size() == `8`) {
1398	NewFn = nullptr;
1399	return true;
1400	}
1401	break;
1402	}
1403
1404	if (Name.consume_front(Prefix: "ds.") \|\| Name.consume_front(Prefix: "global.atomic.") \|\|
1405	Name.consume_front(Prefix: "flat.atomic.")) {
1406	if (Name.starts_with(Prefix: "fadd") \|\|
1407	// FIXME: We should also remove fmin.num and fmax.num intrinsics.
1408	(Name.starts_with(Prefix: "fmin") && !Name.starts_with(Prefix: "fmin.num")) \|\|
1409	(Name.starts_with(Prefix: "fmax") && !Name.starts_with(Prefix: "fmax.num"))) {
1410	// Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1411	// declaration.
1412	NewFn = nullptr;
1413	return true;
1414	}
1415	}
1416
1417	if (Name.starts_with(Prefix: "ldexp.")) {
1418	// Target specific intrinsic became redundant
1419	NewFn = Intrinsic::getOrInsertDeclaration(
1420	M: F->getParent(), id: Intrinsic::ldexp,
1421	OverloadTys: {F->getReturnType(), F->getArg(i: `1`)->getType()});
1422	return true;
1423	}
1424	break; // No other 'amdgcn.'*
1425	}
1426
1427	break;
1428	}
1429	case `'c'`: {
1430	if (F->arg_size() == `1`) {
1431	if (Name.consume_front(Prefix: "convert.")) {
1432	if (convertIntrinsicValidType(Name, FuncTy: F->getFunctionType())) {
1433	NewFn = nullptr;
1434	return true;
1435	}
1436	}
1437
1438	Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1439	.StartsWith(S: "ctlz.", Value: Intrinsic::ctlz)
1440	.StartsWith(S: "cttz.", Value: Intrinsic::cttz)
1441	.Default(Value: Intrinsic::not_intrinsic);
1442	if (ID != Intrinsic::not_intrinsic) {
1443	rename(GV: F);
1444	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1445	OverloadTys: F->arg_begin()->getType());
1446	return true;
1447	}
1448	}
1449
1450	if (F->arg_size() == `2` && Name == "coro.end") {
1451	rename(GV: F);
1452	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1453	id: Intrinsic::coro_end);
1454	return true;
1455	}
1456
1457	break;
1458	}
1459	case `'d'`:
1460	if (Name.consume_front(Prefix: "dbg.")) {
1461	// Mark debug intrinsics for upgrade to new debug format.
1462	if (CanUpgradeDebugIntrinsicsToRecords) {
1463	if (Name == "addr" \|\| Name == "value" \|\| Name == "assign" \|\|
1464	Name == "declare" \|\| Name == "label") {
1465	// There's no function to replace these with.
1466	NewFn = nullptr;
1467	// But we do want these to get upgraded.
1468	return true;
1469	}
1470	}
1471	// Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1472	// converted to DbgVariableRecords later.
1473	if (Name == "addr" \|\| (Name == "value" && F->arg_size() == `4`)) {
1474	rename(GV: F);
1475	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1476	id: Intrinsic::dbg_value);
1477	return true;
1478	}
1479	break; // No other 'dbg.'.*
1480	}
1481	break;
1482	case `'e'`:
1483	if (Name.consume_front(Prefix: "experimental.vector.")) {
1484	Intrinsic::ID ID =
1485	StringSwitch<Intrinsic::ID>(Name)
1486	// Skip over extract.last.active, otherwise it will be 'upgraded'
1487	// to a regular vector extract which is a different operation.
1488	.StartsWith(S: "extract.last.active.", Value: Intrinsic::not_intrinsic)
1489	.StartsWith(S: "extract.", Value: Intrinsic::vector_extract)
1490	.StartsWith(S: "insert.", Value: Intrinsic::vector_insert)
1491	.StartsWith(S: "reverse.", Value: Intrinsic::vector_reverse)
1492	.StartsWith(S: "interleave2.", Value: Intrinsic::vector_interleave2)
1493	.StartsWith(S: "deinterleave2.", Value: Intrinsic::vector_deinterleave2)
1494	.StartsWith(S: "partial.reduce.add",
1495	Value: Intrinsic::vector_partial_reduce_add)
1496	.Default(Value: Intrinsic::not_intrinsic);
1497	if (ID != Intrinsic::not_intrinsic) {
1498	const auto *FT = F->getFunctionType();
1499	SmallVector<Type *, `2`> Tys;
1500	if (ID == Intrinsic::vector_extract \|\|
1501	ID == Intrinsic::vector_interleave2)
1502	// Extracting overloads the return type.
1503	Tys.push_back(Elt: FT->getReturnType());
1504	if (ID != Intrinsic::vector_interleave2)
1505	Tys.push_back(Elt: FT->getParamType(i: `0`));
1506	if (ID == Intrinsic::vector_insert \|\|
1507	ID == Intrinsic::vector_partial_reduce_add)
1508	// Inserting overloads the inserted type.
1509	Tys.push_back(Elt: FT->getParamType(i: `1`));
1510	rename(GV: F);
1511	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: Tys);
1512	return true;
1513	}
1514
1515	if (Name.consume_front(Prefix: "reduce.")) {
1516	SmallVector<StringRef, `2`> Groups;
1517	static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1518	if (R.match(String: Name, Matches: &Groups))
1519	ID = StringSwitch<Intrinsic::ID>(Groups [`1`])
1520	.Case(S: "add", Value: Intrinsic::vector_reduce_add)
1521	.Case(S: "mul", Value: Intrinsic::vector_reduce_mul)
1522	.Case(S: "and", Value: Intrinsic::vector_reduce_and)
1523	.Case(S: "or", Value: Intrinsic::vector_reduce_or)
1524	.Case(S: "xor", Value: Intrinsic::vector_reduce_xor)
1525	.Case(S: "smax", Value: Intrinsic::vector_reduce_smax)
1526	.Case(S: "smin", Value: Intrinsic::vector_reduce_smin)
1527	.Case(S: "umax", Value: Intrinsic::vector_reduce_umax)
1528	.Case(S: "umin", Value: Intrinsic::vector_reduce_umin)
1529	.Case(S: "fmax", Value: Intrinsic::vector_reduce_fmax)
1530	.Case(S: "fmin", Value: Intrinsic::vector_reduce_fmin)
1531	.Default(Value: Intrinsic::not_intrinsic);
1532
1533	bool V2 = false;
1534	if (ID == Intrinsic::not_intrinsic) {
1535	static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1536	Groups.clear();
1537	V2 = true;
1538	if (R2.match(String: Name, Matches: &Groups))
1539	ID = StringSwitch<Intrinsic::ID>(Groups [`1`])
1540	.Case(S: "fadd", Value: Intrinsic::vector_reduce_fadd)
1541	.Case(S: "fmul", Value: Intrinsic::vector_reduce_fmul)
1542	.Default(Value: Intrinsic::not_intrinsic);
1543	}
1544	if (ID != Intrinsic::not_intrinsic) {
1545	rename(GV: F);
1546	auto Args = F->getFunctionType()->params();
1547	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1548	OverloadTys: {Args [V2 ? `1` : `0`]});
1549	return true;
1550	}
1551	break; // No other 'expermental.vector.reduce.'.*
1552	}
1553
1554	if (Name.consume_front(Prefix: "splice"))
1555	return true;
1556	break; // No other 'experimental.vector.'.*
1557	}
1558	if (Name.consume_front(Prefix: "experimental.stepvector.")) {
1559	Intrinsic::ID ID = Intrinsic::stepvector;
1560	rename(GV: F);
1561	NewFn = Intrinsic::getOrInsertDeclaration(
1562	M: F->getParent(), id: ID, OverloadTys: F->getFunctionType()->getReturnType());
1563	return true;
1564	}
1565	break; // No other 'e'.*
1566	case `'f'`:
1567	if (Name.starts_with(Prefix: "flt.rounds")) {
1568	rename(GV: F);
1569	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1570	id: Intrinsic::get_rounding);
1571	return true;
1572	}
1573	break;
1574	case `'i'`:
1575	if (Name.starts_with(Prefix: "invariant.group.barrier")) {
1576	// Rename invariant.group.barrier to launder.invariant.group
1577	auto Args = F->getFunctionType()->params();
1578	Type* ObjectPtr[`1`] = {Args [`0`]};
1579	rename(GV: F);
1580	NewFn = Intrinsic::getOrInsertDeclaration(
1581	M: F->getParent(), id: Intrinsic::launder_invariant_group, OverloadTys: ObjectPtr);
1582	return true;
1583	}
1584	break;
1585	case `'l'`: {
1586	bool IsLifetimeStart = Name.consume_front(Prefix: "lifetime.start");
1587	bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front(Prefix: "lifetime.end");
1588	if (IsLifetimeStart \|\| IsLifetimeEnd) {
1589	if (F->arg_size() == `2`) {
1590	Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1591	: Intrinsic::lifetime_end;
1592	rename(GV: F);
1593	// Old 2 argument form of these intrinsics have [Size, Ptr] as
1594	// arguments. Use the Ptr argument to create new declaration.
1595	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1596	OverloadTys: F->getArg(i: `1`)->getType());
1597	return true;
1598	} else if (F->arg_size() == `1` && Name == ".i64") {
1599	// Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1600	// Autoupgrade prior to
1601	// https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1602	// intrinsic with no expected calls. To allow auto-upgrade process to
1603	// delete such invalid intrinsic declaration, set NewFn = nullptr
1604	// and return true here. If there are actual calls to this intrinsic
1605	// (which is not expected), they will be deleted in
1606	// UpgradeIntrinsicCall.
1607	NewFn = nullptr;
1608	return true;
1609	}
1610	}
1611	break;
1612	}
1613	case `'m'`: {
1614	// Updating the memory intrinsics (memcpy/memmove/memset) that have an
1615	// alignment parameter to embedding the alignment as an attribute of
1616	// the pointer args.
1617	if (unsigned ID = StringSwitch<unsigned>(Name)
1618	.StartsWith(S: "memcpy.", Value: Intrinsic::memcpy)
1619	.StartsWith(S: "memmove.", Value: Intrinsic::memmove)
1620	.Default(Value: `0`)) {
1621	if (F->arg_size() == `5`) {
1622	rename(GV: F);
1623	// Get the types of dest, src, and len
1624	ArrayRef<Type *> ParamTypes =
1625	F->getFunctionType()->params().slice(N: `0`, M: `3`);
1626	NewFn =
1627	Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys: ParamTypes);
1628	return true;
1629	}
1630	}
1631	if (Name.starts_with(Prefix: "memset.") && F->arg_size() == `5`) {
1632	rename(GV: F);
1633	// Get the types of dest, and len
1634	const auto *FT = F->getFunctionType();
1635	Type *ParamTypes[`2`] = {
1636	FT->getParamType(i: `0`), // Dest
1637	FT->getParamType(i: `2`) // len
1638	};
1639	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1640	id: Intrinsic::memset, OverloadTys: ParamTypes);
1641	return true;
1642	}
1643
1644	unsigned MaskedID =
1645	StringSwitch<unsigned>(Name)
1646	.StartsWith(S: "masked.load", Value: Intrinsic::masked_load)
1647	.StartsWith(S: "masked.gather", Value: Intrinsic::masked_gather)
1648	.StartsWith(S: "masked.store", Value: Intrinsic::masked_store)
1649	.StartsWith(S: "masked.scatter", Value: Intrinsic::masked_scatter)
1650	.Default(Value: `0`);
1651	if (MaskedID && F->arg_size() == `4`) {
1652	rename(GV: F);
1653	if (MaskedID == Intrinsic::masked_load \|\|
1654	MaskedID == Intrinsic::masked_gather) {
1655	NewFn = Intrinsic::getOrInsertDeclaration(
1656	M: F->getParent(), id: MaskedID,
1657	OverloadTys: {F->getReturnType(), F->getArg(i: `0`)->getType()});
1658	return true;
1659	}
1660	NewFn = Intrinsic::getOrInsertDeclaration(
1661	M: F->getParent(), id: MaskedID,
1662	OverloadTys: {F->getArg(i: `0`)->getType(), F->getArg(i: `1`)->getType()});
1663	return true;
1664	}
1665	break;
1666	}
1667	case `'n'`: {
1668	if (Name.consume_front(Prefix: "nvvm.")) {
1669	// Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1670	if (F->arg_size() == `1`) {
1671	Intrinsic::ID IID =
1672	StringSwitch<Intrinsic::ID>(Name)
1673	.Cases(CaseStrings: {"brev32", "brev64"}, Value: Intrinsic::bitreverse)
1674	.Case(S: "clz.i", Value: Intrinsic::ctlz)
1675	.Case(S: "popc.i", Value: Intrinsic::ctpop)
1676	.Default(Value: Intrinsic::not_intrinsic);
1677	if (IID != Intrinsic::not_intrinsic) {
1678	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1679	OverloadTys: {F->getReturnType()});
1680	return true;
1681	}
1682	} else if (F->arg_size() == `2`) {
1683	Intrinsic::ID IID =
1684	StringSwitch<Intrinsic::ID>(Name)
1685	.Cases(CaseStrings: {"max.s", "max.i", "max.ll"}, Value: Intrinsic::smax)
1686	.Cases(CaseStrings: {"min.s", "min.i", "min.ll"}, Value: Intrinsic::smin)
1687	.Cases(CaseStrings: {"max.us", "max.ui", "max.ull"}, Value: Intrinsic::umax)
1688	.Cases(CaseStrings: {"min.us", "min.ui", "min.ull"}, Value: Intrinsic::umin)
1689	.Default(Value: Intrinsic::not_intrinsic);
1690	if (IID != Intrinsic::not_intrinsic) {
1691	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID,
1692	OverloadTys: {F->getReturnType()});
1693	return true;
1694	}
1695	}
1696
1697	// Check for nvvm intrinsics that need a return type adjustment.
1698	if (!F->getReturnType()->getScalarType()->isBFloatTy()) {
1699	Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1700	if (IID != Intrinsic::not_intrinsic) {
1701	NewFn = nullptr;
1702	return true;
1703	}
1704	}
1705
1706	// Upgrade Distributed Shared Memory Intrinsics
1707	Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1708	if (IID != Intrinsic::not_intrinsic) {
1709	rename(GV: F);
1710	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1711	return true;
1712	}
1713
1714	// Upgrade TMA copy G2S Intrinsics
1715	IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1716	if (IID != Intrinsic::not_intrinsic) {
1717	rename(GV: F);
1718	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
1719	return true;
1720	}
1721
1722	// The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1723	// not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1724	//
1725	// TODO: We could add lohi.i2d.
1726	bool Expand = false;
1727	if (Name.consume_front(Prefix: "abs."))
1728	// nvvm.abs.{i,ii}
1729	Expand =
1730	Name == "i" \|\| Name == "ll" \|\| Name == "bf16" \|\| Name == "bf16x2";
1731	else if (Name.consume_front(Prefix: "fabs."))
1732	// nvvm.fabs.{f,ftz.f,d}
1733	Expand = Name == "f" \|\| Name == "ftz.f" \|\| Name == "d";
1734	else if (Name.consume_front(Prefix: "ex2.approx."))
1735	// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
1736	Expand =
1737	Name == "f" \|\| Name == "ftz.f" \|\| Name == "d" \|\| Name == "f16x2";
1738	else if (Name.consume_front(Prefix: "atomic.load."))
1739	// nvvm.atomic.load.add.{f32,f64}.p
1740	// nvvm.atomic.load.{inc,dec}.32.p
1741	Expand = StringSwitch<bool>(Name)
1742	.StartsWith(S: "add.f32.p", Value: true)
1743	.StartsWith(S: "add.f64.p", Value: true)
1744	.StartsWith(S: "inc.32.p", Value: true)
1745	.StartsWith(S: "dec.32.p", Value: true)
1746	.Default(Value: false);
1747	else if (Name.consume_front(Prefix: "atomic."))
1748	// nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
1749	// nvvm.atomic.cas.gen.i.{cta,sys}
1750	Expand = StringSwitch<bool>(Name)
1751	.StartsWith(S: "add.gen.", Value: true)
1752	.StartsWith(S: "exch.gen.", Value: true)
1753	.StartsWith(S: "max.gen.", Value: true)
1754	.StartsWith(S: "min.gen.", Value: true)
1755	.StartsWith(S: "inc.gen.", Value: true)
1756	.StartsWith(S: "dec.gen.", Value: true)
1757	.StartsWith(S: "and.gen.", Value: true)
1758	.StartsWith(S: "or.gen.", Value: true)
1759	.StartsWith(S: "xor.gen.", Value: true)
1760	.StartsWith(S: "cas.gen.", Value: true)
1761	.Default(Value: false);
1762	else if (Name.consume_front(Prefix: "bitcast."))
1763	// nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
1764	Expand =
1765	Name == "f2i" \|\| Name == "i2f" \|\| Name == "ll2d" \|\| Name == "d2ll";
1766	else if (Name.consume_front(Prefix: "rotate."))
1767	// nvvm.rotate.{b32,b64,right.b64}
1768	Expand = Name == "b32" \|\| Name == "b64" \|\| Name == "right.b64";
1769	else if (Name.consume_front(Prefix: "ptr.gen.to."))
1770	// nvvm.ptr.gen.to.{local,shared,global,constant,param}
1771	Expand = consumeNVVMPtrAddrSpace(Name);
1772	else if (Name.consume_front(Prefix: "ptr."))
1773	// nvvm.ptr.{local,shared,global,constant,param}.to.gen
1774	Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(Prefix: ".to.gen");
1775	else if (Name.consume_front(Prefix: "ldg.global."))
1776	// nvvm.ldg.global.{i,p,f}
1777	Expand = (Name.starts_with(Prefix: "i.") \|\| Name.starts_with(Prefix: "f.") \|\|
1778	Name.starts_with(Prefix: "p."));
1779	else
1780	Expand = StringSwitch<bool>(Name)
1781	.Case(S: "barrier0", Value: true)
1782	.Case(S: "barrier.n", Value: true)
1783	.Case(S: "barrier.sync.cnt", Value: true)
1784	.Case(S: "barrier.sync", Value: true)
1785	.Case(S: "barrier", Value: true)
1786	.Case(S: "bar.sync", Value: true)
1787	.Case(S: "barrier0.popc", Value: true)
1788	.Case(S: "barrier0.and", Value: true)
1789	.Case(S: "barrier0.or", Value: true)
1790	.Case(S: "clz.ll", Value: true)
1791	.Case(S: "popc.ll", Value: true)
1792	.Case(S: "h2f", Value: true)
1793	.Case(S: "swap.lo.hi.b64", Value: true)
1794	.Case(S: "tanh.approx.f32", Value: true)
1795	.Default(Value: false);
1796
1797	if (Expand) {
1798	NewFn = nullptr;
1799	return true;
1800	}
1801	break; // No other 'nvvm.'.*
1802	}
1803	break;
1804	}
1805	case `'o'`:
1806	if (Name.starts_with(Prefix: "objectsize.")) {
1807	Type *Tys[`2`] = { F->getReturnType(), F->arg_begin()->getType() };
1808	if (F->arg_size() == `2` \|\| F->arg_size() == `3`) {
1809	rename(GV: F);
1810	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(),
1811	id: Intrinsic::objectsize, OverloadTys: Tys);
1812	return true;
1813	}
1814	}
1815	break;
1816
1817	case `'p'`:
1818	if (Name.starts_with(Prefix: "ptr.annotation.") && F->arg_size() == `4`) {
1819	rename(GV: F);
1820	NewFn = Intrinsic::getOrInsertDeclaration(
1821	M: F->getParent(), id: Intrinsic::ptr_annotation,
1822	OverloadTys: {F->arg_begin()->getType(), F->getArg(i: `1`)->getType()});
1823	return true;
1824	}
1825	break;
1826
1827	case `'r'`: {
1828	if (Name.consume_front(Prefix: "riscv.")) {
1829	Intrinsic::ID ID;
1830	ID = StringSwitch<Intrinsic::ID>(Name)
1831	.Case(S: "aes32dsi", Value: Intrinsic::riscv_aes32dsi)
1832	.Case(S: "aes32dsmi", Value: Intrinsic::riscv_aes32dsmi)
1833	.Case(S: "aes32esi", Value: Intrinsic::riscv_aes32esi)
1834	.Case(S: "aes32esmi", Value: Intrinsic::riscv_aes32esmi)
1835	.Default(Value: Intrinsic::not_intrinsic);
1836	if (ID != Intrinsic::not_intrinsic) {
1837	if (!F->getFunctionType()->getParamType(i: `2`)->isIntegerTy(BitWidth: `32`)) {
1838	rename(GV: F);
1839	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1840	return true;
1841	}
1842	break; // No other applicable upgrades.
1843	}
1844
1845	ID = StringSwitch<Intrinsic::ID>(Name)
1846	.StartsWith(S: "sm4ks", Value: Intrinsic::riscv_sm4ks)
1847	.StartsWith(S: "sm4ed", Value: Intrinsic::riscv_sm4ed)
1848	.Default(Value: Intrinsic::not_intrinsic);
1849	if (ID != Intrinsic::not_intrinsic) {
1850	if (!F->getFunctionType()->getParamType(i: `2`)->isIntegerTy(BitWidth: `32`) \|\|
1851	F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: `64`)) {
1852	rename(GV: F);
1853	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1854	return true;
1855	}
1856	break; // No other applicable upgrades.
1857	}
1858
1859	ID = StringSwitch<Intrinsic::ID>(Name)
1860	.StartsWith(S: "sha256sig0", Value: Intrinsic::riscv_sha256sig0)
1861	.StartsWith(S: "sha256sig1", Value: Intrinsic::riscv_sha256sig1)
1862	.StartsWith(S: "sha256sum0", Value: Intrinsic::riscv_sha256sum0)
1863	.StartsWith(S: "sha256sum1", Value: Intrinsic::riscv_sha256sum1)
1864	.StartsWith(S: "sm3p0", Value: Intrinsic::riscv_sm3p0)
1865	.StartsWith(S: "sm3p1", Value: Intrinsic::riscv_sm3p1)
1866	.Default(Value: Intrinsic::not_intrinsic);
1867	if (ID != Intrinsic::not_intrinsic) {
1868	if (F->getFunctionType()->getReturnType()->isIntegerTy(BitWidth: `64`)) {
1869	rename(GV: F);
1870	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1871	return true;
1872	}
1873	break; // No other applicable upgrades.
1874	}
1875
1876	// Replace llvm.riscv.clmul with llvm.clmul.
1877	if (Name == "clmul.i32" \|\| Name == "clmul.i64") {
1878	NewFn = Intrinsic::getOrInsertDeclaration(
1879	M: F->getParent(), id: Intrinsic::clmul, OverloadTys: {F->getReturnType()});
1880	return true;
1881	}
1882
1883	break; // No other 'riscv.' intrinsics*
1884	}
1885	} break;
1886
1887	case `'s'`:
1888	if (Name == "stackprotectorcheck") {
1889	NewFn = nullptr;
1890	return true;
1891	}
1892	break;
1893
1894	case `'t'`:
1895	if (Name == "thread.pointer") {
1896	NewFn = Intrinsic::getOrInsertDeclaration(
1897	M: F->getParent(), id: Intrinsic::thread_pointer, OverloadTys: F->getReturnType());
1898	return true;
1899	}
1900	break;
1901
1902	case `'v'`: {
1903	if (Name == "var.annotation" && F->arg_size() == `4`) {
1904	rename(GV: F);
1905	NewFn = Intrinsic::getOrInsertDeclaration(
1906	M: F->getParent(), id: Intrinsic::var_annotation,
1907	OverloadTys: {{F->arg_begin()->getType(), F->getArg(i: `1`)->getType()}});
1908	return true;
1909	}
1910	if (Name.consume_front(Prefix: "vector.splice")) {
1911	if (Name.starts_with(Prefix: ".left") \|\| Name.starts_with(Prefix: ".right"))
1912	break;
1913	return true;
1914	}
1915	break;
1916	}
1917
1918	case `'w'`:
1919	if (Name.consume_front(Prefix: "wasm.")) {
1920	Intrinsic::ID ID =
1921	StringSwitch<Intrinsic::ID>(Name)
1922	.StartsWith(S: "fma.", Value: Intrinsic::wasm_relaxed_madd)
1923	.StartsWith(S: "fms.", Value: Intrinsic::wasm_relaxed_nmadd)
1924	.StartsWith(S: "laneselect.", Value: Intrinsic::wasm_relaxed_laneselect)
1925	.Default(Value: Intrinsic::not_intrinsic);
1926	if (ID != Intrinsic::not_intrinsic) {
1927	rename(GV: F);
1928	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID,
1929	OverloadTys: F->getReturnType());
1930	return true;
1931	}
1932
1933	if (Name.consume_front(Prefix: "dot.i8x16.i7x16.")) {
1934	ID = StringSwitch<Intrinsic::ID>(Name)
1935	.Case(S: "signed", Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1936	.Case(S: "add.signed",
1937	Value: Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1938	.Default(Value: Intrinsic::not_intrinsic);
1939	if (ID != Intrinsic::not_intrinsic) {
1940	rename(GV: F);
1941	NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID);
1942	return true;
1943	}
1944	break; // No other 'wasm.dot.i8x16.i7x16.'.*
1945	}
1946	break; // No other 'wasm.'.*
1947	}
1948	break;
1949
1950	case `'x'`:
1951	if (upgradeX86IntrinsicFunction(F, Name, NewFn))
1952	return true;
1953	}
1954
1955	auto *ST = dyn_cast<StructType>(Val: F->getReturnType());
1956	if (ST && (!ST->isLiteral() \|\| ST->isPacked()) &&
1957	F->getIntrinsicID() != Intrinsic::not_intrinsic) {
1958	// Replace return type with literal non-packed struct. Only do this for
1959	// intrinsics declared to return a struct, not for intrinsics with
1960	// overloaded return type, in which case the exact struct type will be
1961	// mangled into the name.
1962	if (Intrinsic::hasStructReturnType(id: F->getIntrinsicID())) {
1963	FunctionType *FT = F->getFunctionType();
1964	auto *NewST = StructType::get(Context&: ST->getContext(), Elements: ST->elements());
1965	auto *NewFT = FunctionType::get(Result: NewST, Params: FT->params(), isVarArg: FT->isVarArg());
1966	std::string Name = F->getName().str();
1967	rename(GV: F);
1968	NewFn = Function::Create(Ty: NewFT, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
1969	N: Name, M: F->getParent());
1970
1971	// The new function may also need remangling.
1972	if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F: NewFn))
1973	NewFn = *Result;
1974	return true;
1975	}
1976	}
1977
1978	// Remangle our intrinsic since we upgrade the mangling
1979	auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
1980	if (Result != std::nullopt) {
1981	NewFn = *Result;
1982	return true;
1983	}
1984
1985	// This may not belong here. This function is effectively being overloaded
1986	// to both detect an intrinsic which needs upgrading, and to provide the
1987	// upgraded form of the intrinsic. We should perhaps have two separate
1988	// functions for this.
1989	if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
1990	return true;
1991
1992	return false;
1993	}
1994
1995	bool llvm::UpgradeIntrinsicFunction(Function F, Function &NewFn,
1996	bool CanUpgradeDebugIntrinsicsToRecords) {
1997	NewFn = nullptr;
1998	bool Upgraded =
1999	upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2000
2001	// Upgrade intrinsic attributes. This does not change the function.
2002	if (NewFn)
2003	F = NewFn;
2004	if (Intrinsic::ID id = F->getIntrinsicID()) {
2005	// Only do this if the intrinsic signature is valid.
2006	SmallVector<Type *> OverloadTys;
2007	if (Intrinsic::isSignatureValid(ID: id, FT: F->getFunctionType(), OverloadTys))
2008	F->setAttributes(
2009	Intrinsic::getAttributes(C&: F->getContext(), id, FT: F->getFunctionType()));
2010	}
2011	return Upgraded;
2012	}
2013
2014	GlobalVariable llvm::UpgradeGlobalVariable(GlobalVariable GV) {
2015	if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" \|\|
2016	GV->getName() == "llvm.global_dtors")) \|\|
2017	!GV->hasInitializer())
2018	return nullptr;
2019	ArrayType *ATy = dyn_cast<ArrayType>(Val: GV->getValueType());
2020	if (!ATy)
2021	return nullptr;
2022	StructType *STy = dyn_cast<StructType>(Val: ATy->getElementType());
2023	if (!STy \|\| STy->getNumElements() != `2`)
2024	return nullptr;
2025
2026	LLVMContext &C = GV->getContext();
2027	IRBuilder<> IRB(C);
2028	auto EltTy = StructType::get(elt1: STy->getElementType(N: `0`), elts: STy->getElementType(N: `1`),
2029	elts: IRB.getPtrTy());
2030	Constant *Init = GV->getInitializer();
2031	unsigned N = Init->getNumOperands();
2032	std::vector<Constant *> NewCtors(N);
2033	for (unsigned i = `0`; i != N; ++i) {
2034	auto Ctor = cast<Constant>(Val: Init->getOperand(i));
2035	NewCtors [i] = ConstantStruct::get(T: EltTy, Vs: Ctor->getAggregateElement(Elt: `0u`),
2036	Vs: Ctor->getAggregateElement(Elt: `1`),
2037	Vs: ConstantPointerNull::get(T: IRB.getPtrTy()));
2038	}
2039	Constant *NewInit = ConstantArray::get(T: ArrayType::get(ElementType: EltTy, NumElements: N), V: NewCtors);
2040
2041	return new GlobalVariable (NewInit->getType(), false, GV->getLinkage(),
2042	NewInit, GV->getName());
2043	}
2044
2045	// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2046	// to byte shuffles.
2047	static Value upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value Op,
2048	unsigned Shift) {
2049	auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2050	unsigned NumElts = ResultTy->getNumElements() * `8`;
2051
2052	// Bitcast from a 64-bit element type to a byte element type.
2053	Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2054	Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2055
2056	// We'll be shuffling in zeroes.
2057	Value *Res = Constant::getNullValue(Ty: VecTy);
2058
2059	// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2060	// we'll just return the zero vector.
2061	if (Shift < `16`) {
2062	int Idxs[`64`];
2063	// 256/512-bit version is split into 2/4 16-byte lanes.
2064	for (unsigned l = `0`; l != NumElts; l += `16`)
2065	for (unsigned i = `0`; i != `16`; ++i) {
2066	unsigned Idx = NumElts + i - Shift;
2067	if (Idx < NumElts)
2068	Idx -= NumElts - `16`; // end of lane, switch operand.
2069	Idxs[l + i] = Idx + l;
2070	}
2071
2072	Res = Builder.CreateShuffleVector(V1: Res, V2: Op, Mask: ArrayRef(Idxs, NumElts));
2073	}
2074
2075	// Bitcast back to a 64-bit element type.
2076	return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2077	}
2078
2079	// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2080	// to byte shuffles.
2081	static Value upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value Op,
2082	unsigned Shift) {
2083	auto *ResultTy = cast<FixedVectorType>(Val: Op->getType());
2084	unsigned NumElts = ResultTy->getNumElements() * `8`;
2085
2086	// Bitcast from a 64-bit element type to a byte element type.
2087	Type *VecTy = FixedVectorType::get(ElementType: Builder.getInt8Ty(), NumElts);
2088	Op = Builder.CreateBitCast(V: Op, DestTy: VecTy, Name: "cast");
2089
2090	// We'll be shuffling in zeroes.
2091	Value *Res = Constant::getNullValue(Ty: VecTy);
2092
2093	// If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2094	// we'll just return the zero vector.
2095	if (Shift < `16`) {
2096	int Idxs[`64`];
2097	// 256/512-bit version is split into 2/4 16-byte lanes.
2098	for (unsigned l = `0`; l != NumElts; l += `16`)
2099	for (unsigned i = `0`; i != `16`; ++i) {
2100	unsigned Idx = i + Shift;
2101	if (Idx >= `16`)
2102	Idx += NumElts - `16`; // end of lane, switch operand.
2103	Idxs[l + i] = Idx + l;
2104	}
2105
2106	Res = Builder.CreateShuffleVector(V1: Op, V2: Res, Mask: ArrayRef(Idxs, NumElts));
2107	}
2108
2109	// Bitcast back to a 64-bit element type.
2110	return Builder.CreateBitCast(V: Res, DestTy: ResultTy, Name: "cast");
2111	}
2112
2113	static Value getX86MaskVec(IRBuilder<> &Builder, Value Mask,
2114	unsigned NumElts) {
2115	assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2116	llvm::VectorType *MaskTy = FixedVectorType::get(
2117	ElementType: Builder.getInt1Ty(), NumElts: cast<IntegerType>(Val: Mask->getType())->getBitWidth());
2118	Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2119
2120	// If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2121	// i8 and we need to extract down to the right number of elements.
2122	if (NumElts <= `4`) {
2123	int Indices[`4`];
2124	for (unsigned i = `0`; i != NumElts; ++i)
2125	Indices[i] = i;
2126	Mask = Builder.CreateShuffleVector(V1: Mask, V2: Mask, Mask: ArrayRef(Indices, NumElts),
2127	Name: "extract");
2128	}
2129
2130	return Mask;
2131	}
2132
2133	static Value emitX86Select(IRBuilder<> &Builder, Value Mask, Value *Op0,
2134	Value *Op1) {
2135	// If the mask is all ones just emit the first operation.
2136	if (const auto *C = dyn_cast<Constant>(Val: Mask))
2137	if (C->isAllOnesValue())
2138	return Op0;
2139
2140	Mask = getX86MaskVec(Builder, Mask,
2141	NumElts: cast<FixedVectorType>(Val: Op0->getType())->getNumElements());
2142	return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2143	}
2144
2145	static Value emitX86ScalarSelect(IRBuilder<> &Builder, Value Mask, Value *Op0,
2146	Value *Op1) {
2147	// If the mask is all ones just emit the first operation.
2148	if (const auto *C = dyn_cast<Constant>(Val: Mask))
2149	if (C->isAllOnesValue())
2150	return Op0;
2151
2152	auto *MaskTy = FixedVectorType::get(ElementType: Builder.getInt1Ty(),
2153	NumElts: Mask->getType()->getIntegerBitWidth());
2154	Mask = Builder.CreateBitCast(V: Mask, DestTy: MaskTy);
2155	Mask = Builder.CreateExtractElement(Vec: Mask, Idx: (uint64_t)`0`);
2156	return Builder.CreateSelect(C: Mask, True: Op0, False: Op1);
2157	}
2158
2159	// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2160	// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2161	// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2162	static Value upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value Op0,
2163	Value Op1, Value Shift,
2164	Value Passthru, Value Mask,
2165	bool IsVALIGN) {
2166	unsigned ShiftVal = cast<llvm::ConstantInt>(Val: Shift)->getZExtValue();
2167
2168	unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2169	assert((IsVALIGN \|\| NumElts % `16` == `0`) && "Illegal NumElts for PALIGNR!");
2170	assert((!IsVALIGN \|\| NumElts <= `16`) && "NumElts too large for VALIGN!");
2171	assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2172
2173	// Mask the immediate for VALIGN.
2174	if (IsVALIGN)
2175	ShiftVal &= (NumElts - `1`);
2176
2177	// If palignr is shifting the pair of vectors more than the size of two
2178	// lanes, emit zero.
2179	if (ShiftVal >= `32`)
2180	return llvm::Constant::getNullValue(Ty: Op0->getType());
2181
2182	// If palignr is shifting the pair of input vectors more than one lane,
2183	// but less than two lanes, convert to shifting in zeroes.
2184	if (ShiftVal > `16`) {
2185	ShiftVal -= `16`;
2186	Op1 = Op0;
2187	Op0 = llvm::Constant::getNullValue(Ty: Op0->getType());
2188	}
2189
2190	int Indices[`64`];
2191	// 256-bit palignr operates on 128-bit lanes so we need to handle that
2192	for (unsigned l = `0`; l < NumElts; l += `16`) {
2193	for (unsigned i = `0`; i != `16`; ++i) {
2194	unsigned Idx = ShiftVal + i;
2195	if (!IsVALIGN && Idx >= `16`) // Disable wrap for VALIGN.
2196	Idx += NumElts - `16`; // End of lane, switch operand.
2197	Indices[l + i] = Idx + l;
2198	}
2199	}
2200
2201	Value *Align = Builder.CreateShuffleVector(
2202	V1: Op1, V2: Op0, Mask: ArrayRef(Indices, NumElts), Name: "palignr");
2203
2204	return emitX86Select(Builder, Mask, Op0: Align, Op1: Passthru);
2205	}
2206
2207	static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2208	bool ZeroMask, bool IndexForm) {
2209	Type *Ty = CI.getType();
2210	unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2211	unsigned EltWidth = Ty->getScalarSizeInBits();
2212	bool IsFloat = Ty->isFPOrFPVectorTy();
2213	Intrinsic::ID IID;
2214	if (VecWidth == `128` && EltWidth == `32` && IsFloat)
2215	IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2216	else if (VecWidth == `128` && EltWidth == `32` && !IsFloat)
2217	IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2218	else if (VecWidth == `128` && EltWidth == `64` && IsFloat)
2219	IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2220	else if (VecWidth == `128` && EltWidth == `64` && !IsFloat)
2221	IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2222	else if (VecWidth == `256` && EltWidth == `32` && IsFloat)
2223	IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2224	else if (VecWidth == `256` && EltWidth == `32` && !IsFloat)
2225	IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2226	else if (VecWidth == `256` && EltWidth == `64` && IsFloat)
2227	IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2228	else if (VecWidth == `256` && EltWidth == `64` && !IsFloat)
2229	IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2230	else if (VecWidth == `512` && EltWidth == `32` && IsFloat)
2231	IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2232	else if (VecWidth == `512` && EltWidth == `32` && !IsFloat)
2233	IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2234	else if (VecWidth == `512` && EltWidth == `64` && IsFloat)
2235	IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2236	else if (VecWidth == `512` && EltWidth == `64` && !IsFloat)
2237	IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2238	else if (VecWidth == `128` && EltWidth == `16`)
2239	IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2240	else if (VecWidth == `256` && EltWidth == `16`)
2241	IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2242	else if (VecWidth == `512` && EltWidth == `16`)
2243	IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2244	else if (VecWidth == `128` && EltWidth == `8`)
2245	IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2246	else if (VecWidth == `256` && EltWidth == `8`)
2247	IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2248	else if (VecWidth == `512` && EltWidth == `8`)
2249	IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2250	else
2251	llvm_unreachable("Unexpected intrinsic");
2252
2253	Value *Args[] = { CI.getArgOperand(i: `0`) , CI.getArgOperand(i: `1`),
2254	CI.getArgOperand(i: `2`) };
2255
2256	// If this isn't index form we need to swap operand 0 and 1.
2257	if (!IndexForm)
2258	std::swap(a&: Args[`0`], b&: Args[`1`]);
2259
2260	Value *V = Builder.CreateIntrinsic(ID: IID, Args);
2261	Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2262	: Builder.CreateBitCast(V: CI.getArgOperand(i: `1`),
2263	DestTy: Ty);
2264	return emitX86Select(Builder, Mask: CI.getArgOperand(i: `3`), Op0: V, Op1: PassThru);
2265	}
2266
2267	static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2268	Intrinsic::ID IID) {
2269	Type *Ty = CI.getType();
2270	Value *Op0 = CI.getOperand(i_nocapture: `0`);
2271	Value *Op1 = CI.getOperand(i_nocapture: `1`);
2272	Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1});
2273
2274	if (CI.arg_size() == `4`) { // For masked intrinsics.
2275	Value *VecSrc = CI.getOperand(i_nocapture: `2`);
2276	Value *Mask = CI.getOperand(i_nocapture: `3`);
2277	Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2278	}
2279	return Res;
2280	}
2281
2282	static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2283	bool IsRotateRight) {
2284	Type *Ty = CI.getType();
2285	Value *Src = CI.getArgOperand(i: `0`);
2286	Value *Amt = CI.getArgOperand(i: `1`);
2287
2288	// Amount may be scalar immediate, in which case create a splat vector.
2289	// Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2290	// we only care about the lowest log2 bits anyway.
2291	if (Amt->getType() != Ty) {
2292	unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2293	Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2294	Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2295	}
2296
2297	Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2298	Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Src, Src, Amt});
2299
2300	if (CI.arg_size() == `4`) { // For masked intrinsics.
2301	Value *VecSrc = CI.getOperand(i_nocapture: `2`);
2302	Value *Mask = CI.getOperand(i_nocapture: `3`);
2303	Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2304	}
2305	return Res;
2306	}
2307
2308	static Value upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned* Imm,
2309	bool IsSigned) {
2310	Type *Ty = CI.getType();
2311	Value *LHS = CI.getArgOperand(i: `0`);
2312	Value *RHS = CI.getArgOperand(i: `1`);
2313
2314	CmpInst::Predicate Pred;
2315	switch (Imm) {
2316	case `0x0`:
2317	Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2318	break;
2319	case `0x1`:
2320	Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2321	break;
2322	case `0x2`:
2323	Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2324	break;
2325	case `0x3`:
2326	Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2327	break;
2328	case `0x4`:
2329	Pred = ICmpInst::ICMP_EQ;
2330	break;
2331	case `0x5`:
2332	Pred = ICmpInst::ICMP_NE;
2333	break;
2334	case `0x6`:
2335	return Constant::getNullValue(Ty); // FALSE
2336	case `0x7`:
2337	return Constant::getAllOnesValue(Ty); // TRUE
2338	default:
2339	llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2340	}
2341
2342	Value *Cmp = Builder.CreateICmp(P: Pred, LHS, RHS);
2343	Value *Ext = Builder.CreateSExt(V: Cmp, DestTy: Ty);
2344	return Ext;
2345	}
2346
2347	static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2348	bool IsShiftRight, bool ZeroMask) {
2349	Type *Ty = CI.getType();
2350	Value *Op0 = CI.getArgOperand(i: `0`);
2351	Value *Op1 = CI.getArgOperand(i: `1`);
2352	Value *Amt = CI.getArgOperand(i: `2`);
2353
2354	if (IsShiftRight)
2355	std::swap(a&: Op0, b&: Op1);
2356
2357	// Amount may be scalar immediate, in which case create a splat vector.
2358	// Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2359	// we only care about the lowest log2 bits anyway.
2360	if (Amt->getType() != Ty) {
2361	unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
2362	Amt = Builder.CreateIntCast(V: Amt, DestTy: Ty->getScalarType(), isSigned: false);
2363	Amt = Builder.CreateVectorSplat(NumElts, V: Amt);
2364	}
2365
2366	Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2367	Value *Res = Builder.CreateIntrinsic(ID: IID, OverloadTypes: Ty, Args: {Op0, Op1, Amt});
2368
2369	unsigned NumArgs = CI.arg_size();
2370	if (NumArgs >= `4`) { // For masked intrinsics.
2371	Value *VecSrc = NumArgs == `5` ? CI.getArgOperand(i: `3`) :
2372	ZeroMask ? ConstantAggregateZero::get(Ty: CI.getType()) :
2373	CI.getArgOperand(i: `0`);
2374	Value *Mask = CI.getOperand(i_nocapture: NumArgs - `1`);
2375	Res = emitX86Select(Builder, Mask, Op0: Res, Op1: VecSrc);
2376	}
2377	return Res;
2378	}
2379
2380	static Value upgradeMaskedStore(IRBuilder<> &Builder, Value Ptr, Value *Data,
2381	Value Mask, bool* Aligned) {
2382	const Align Alignment =
2383	Aligned
2384	? Align (Data->getType()->getPrimitiveSizeInBits().getFixedValue() / `8`)
2385	: Align (`1`);
2386
2387	// If the mask is all ones just emit a regular store.
2388	if (const auto *C = dyn_cast<Constant>(Val: Mask))
2389	if (C->isAllOnesValue())
2390	return Builder.CreateAlignedStore(Val: Data, Ptr, Align: Alignment);
2391
2392	// Convert the mask from an integer type to a vector of i1.
2393	unsigned NumElts = cast<FixedVectorType>(Val: Data->getType())->getNumElements();
2394	Mask = getX86MaskVec(Builder, Mask, NumElts);
2395	return Builder.CreateMaskedStore(Val: Data, Ptr, Alignment, Mask);
2396	}
2397
2398	static Value upgradeMaskedLoad(IRBuilder<> &Builder, Value Ptr,
2399	Value Passthru, Value Mask, bool Aligned) {
2400	Type *ValTy = Passthru->getType();
2401	const Align Alignment =
2402	Aligned
2403	? Align (
2404	Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2405	`8`)
2406	: Align (`1`);
2407
2408	// If the mask is all ones just emit a regular store.
2409	if (const auto *C = dyn_cast<Constant>(Val: Mask))
2410	if (C->isAllOnesValue())
2411	return Builder.CreateAlignedLoad(Ty: ValTy, Ptr, Align: Alignment);
2412
2413	// Convert the mask from an integer type to a vector of i1.
2414	unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
2415	Mask = getX86MaskVec(Builder, Mask, NumElts);
2416	return Builder.CreateMaskedLoad(Ty: ValTy, Ptr, Alignment, Mask, PassThru: Passthru);
2417	}
2418
2419	static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2420	Type *Ty = CI.getType();
2421	Value *Op0 = CI.getArgOperand(i: `0`);
2422	Value *Res = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: Ty,
2423	Args: {Op0, Builder.getInt1(V: false)});
2424	if (CI.arg_size() == `3`)
2425	Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: `2`), Op0: Res, Op1: CI.getArgOperand(i: `1`));
2426	return Res;
2427	}
2428
2429	static Value upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool* IsSigned) {
2430	Type *Ty = CI.getType();
2431
2432	// Arguments have a vXi32 type so cast to vXi64.
2433	Value *LHS = Builder.CreateBitCast(V: CI.getArgOperand(i: `0`), DestTy: Ty);
2434	Value *RHS = Builder.CreateBitCast(V: CI.getArgOperand(i: `1`), DestTy: Ty);
2435
2436	if (IsSigned) {
2437	// Shift left then arithmetic shift right.
2438	Constant *ShiftAmt = ConstantInt::get(Ty, V: `32`);
2439	LHS = Builder.CreateShl(LHS, RHS: ShiftAmt);
2440	LHS = Builder.CreateAShr(LHS, RHS: ShiftAmt);
2441	RHS = Builder.CreateShl(LHS: RHS, RHS: ShiftAmt);
2442	RHS = Builder.CreateAShr(LHS: RHS, RHS: ShiftAmt);
2443	} else {
2444	// Clear the upper bits.
2445	Constant *Mask = ConstantInt::get(Ty, V: `0xffffffff`);
2446	LHS = Builder.CreateAnd(LHS, RHS: Mask);
2447	RHS = Builder.CreateAnd(LHS: RHS, RHS: Mask);
2448	}
2449
2450	Value *Res = Builder.CreateMul(LHS, RHS);
2451
2452	if (CI.arg_size() == `4`)
2453	Res = emitX86Select(Builder, Mask: CI.getArgOperand(i: `3`), Op0: Res, Op1: CI.getArgOperand(i: `2`));
2454
2455	return Res;
2456	}
2457
2458	// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2459	static Value applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value Vec,
2460	Value *Mask) {
2461	unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2462	if (Mask) {
2463	const auto *C = dyn_cast<Constant>(Val: Mask);
2464	if (!C \|\| !C->isAllOnesValue())
2465	Vec = Builder.CreateAnd(LHS: Vec, RHS: getX86MaskVec(Builder, Mask, NumElts));
2466	}
2467
2468	if (NumElts < `8`) {
2469	int Indices[`8`];
2470	for (unsigned i = `0`; i != NumElts; ++i)
2471	Indices[i] = i;
2472	for (unsigned i = NumElts; i != `8`; ++i)
2473	Indices[i] = NumElts + i % NumElts;
2474	Vec = Builder.CreateShuffleVector(V1: Vec,
2475	V2: Constant::getNullValue(Ty: Vec->getType()),
2476	Mask: Indices);
2477	}
2478	return Builder.CreateBitCast(V: Vec, DestTy: Builder.getIntNTy(N: std::max(a: NumElts, b: `8U`)));
2479	}
2480
2481	static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2482	unsigned CC, bool Signed) {
2483	Value *Op0 = CI.getArgOperand(i: `0`);
2484	unsigned NumElts = cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
2485
2486	Value *Cmp;
2487	if (CC == `3`) {
2488	Cmp = Constant::getNullValue(
2489	Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2490	} else if (CC == `7`) {
2491	Cmp = Constant::getAllOnesValue(
2492	Ty: FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts));
2493	} else {
2494	ICmpInst::Predicate Pred;
2495	switch (CC) {
2496	default: llvm_unreachable("Unknown condition code");
2497	case `0`: Pred = ICmpInst::ICMP_EQ; break;
2498	case `1`: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2499	case `2`: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2500	case `4`: Pred = ICmpInst::ICMP_NE; break;
2501	case `5`: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2502	case `6`: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2503	}
2504	Cmp = Builder.CreateICmp(P: Pred, LHS: Op0, RHS: CI.getArgOperand(i: `1`));
2505	}
2506
2507	Value *Mask = CI.getArgOperand(i: CI.arg_size() - `1`);
2508
2509	return applyX86MaskOn1BitsVec(Builder, Vec: Cmp, Mask);
2510	}
2511
2512	// Replace a masked intrinsic with an older unmasked intrinsic.
2513	static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2514	Intrinsic::ID IID) {
2515	Value *Rep =
2516	Builder.CreateIntrinsic(ID: IID, Args: {CI.getArgOperand(i: `0`), CI.getArgOperand(i: `1`)});
2517	return emitX86Select(Builder, Mask: CI.getArgOperand(i: `3`), Op0: Rep, Op1: CI.getArgOperand(i: `2`));
2518	}
2519
2520	static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2521	Value* A = CI.getArgOperand(i: `0`);
2522	Value* B = CI.getArgOperand(i: `1`);
2523	Value* Src = CI.getArgOperand(i: `2`);
2524	Value* Mask = CI.getArgOperand(i: `3`);
2525
2526	Value* AndNode = Builder.CreateAnd(LHS: Mask, RHS: APInt (`8`, `1`));
2527	Value* Cmp = Builder.CreateIsNotNull(Arg: AndNode);
2528	Value* Extract1 = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)`0`);
2529	Value* Extract2 = Builder.CreateExtractElement(Vec: Src, Idx: (uint64_t)`0`);
2530	Value* Select = Builder.CreateSelect(C: Cmp, True: Extract1, False: Extract2);
2531	return Builder.CreateInsertElement(Vec: A, NewElt: Select, Idx: (uint64_t)`0`);
2532	}
2533
2534	static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2535	Value* Op = CI.getArgOperand(i: `0`);
2536	Type* ReturnOp = CI.getType();
2537	unsigned NumElts = cast<FixedVectorType>(Val: CI.getType())->getNumElements();
2538	Value *Mask = getX86MaskVec(Builder, Mask: Op, NumElts);
2539	return Builder.CreateSExt(V: Mask, DestTy: ReturnOp, Name: "vpmovm2");
2540	}
2541
2542	// Replace intrinsic with unmasked version and a select.
2543	static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2544	CallBase &CI, Value *&Rep) {
2545	Name = Name.substr(Start: `12`); // Remove avx512.mask.
2546
2547	unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2548	unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2549	Intrinsic::ID IID;
2550	if (Name.starts_with(Prefix: "max.p")) {
2551	if (VecWidth == `128` && EltWidth == `32`)
2552	IID = Intrinsic::x86_sse_max_ps;
2553	else if (VecWidth == `128` && EltWidth == `64`)
2554	IID = Intrinsic::x86_sse2_max_pd;
2555	else if (VecWidth == `256` && EltWidth == `32`)
2556	IID = Intrinsic::x86_avx_max_ps_256;
2557	else if (VecWidth == `256` && EltWidth == `64`)
2558	IID = Intrinsic::x86_avx_max_pd_256;
2559	else
2560	llvm_unreachable("Unexpected intrinsic");
2561	} else if (Name.starts_with(Prefix: "min.p")) {
2562	if (VecWidth == `128` && EltWidth == `32`)
2563	IID = Intrinsic::x86_sse_min_ps;
2564	else if (VecWidth == `128` && EltWidth == `64`)
2565	IID = Intrinsic::x86_sse2_min_pd;
2566	else if (VecWidth == `256` && EltWidth == `32`)
2567	IID = Intrinsic::x86_avx_min_ps_256;
2568	else if (VecWidth == `256` && EltWidth == `64`)
2569	IID = Intrinsic::x86_avx_min_pd_256;
2570	else
2571	llvm_unreachable("Unexpected intrinsic");
2572	} else if (Name.starts_with(Prefix: "pshuf.b.")) {
2573	if (VecWidth == `128`)
2574	IID = Intrinsic::x86_ssse3_pshuf_b_128;
2575	else if (VecWidth == `256`)
2576	IID = Intrinsic::x86_avx2_pshuf_b;
2577	else if (VecWidth == `512`)
2578	IID = Intrinsic::x86_avx512_pshuf_b_512;
2579	else
2580	llvm_unreachable("Unexpected intrinsic");
2581	} else if (Name.starts_with(Prefix: "pmul.hr.sw.")) {
2582	if (VecWidth == `128`)
2583	IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2584	else if (VecWidth == `256`)
2585	IID = Intrinsic::x86_avx2_pmul_hr_sw;
2586	else if (VecWidth == `512`)
2587	IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2588	else
2589	llvm_unreachable("Unexpected intrinsic");
2590	} else if (Name.starts_with(Prefix: "pmulh.w.")) {
2591	if (VecWidth == `128`)
2592	IID = Intrinsic::x86_sse2_pmulh_w;
2593	else if (VecWidth == `256`)
2594	IID = Intrinsic::x86_avx2_pmulh_w;
2595	else if (VecWidth == `512`)
2596	IID = Intrinsic::x86_avx512_pmulh_w_512;
2597	else
2598	llvm_unreachable("Unexpected intrinsic");
2599	} else if (Name.starts_with(Prefix: "pmulhu.w.")) {
2600	if (VecWidth == `128`)
2601	IID = Intrinsic::x86_sse2_pmulhu_w;
2602	else if (VecWidth == `256`)
2603	IID = Intrinsic::x86_avx2_pmulhu_w;
2604	else if (VecWidth == `512`)
2605	IID = Intrinsic::x86_avx512_pmulhu_w_512;
2606	else
2607	llvm_unreachable("Unexpected intrinsic");
2608	} else if (Name.starts_with(Prefix: "pmaddw.d.")) {
2609	if (VecWidth == `128`)
2610	IID = Intrinsic::x86_sse2_pmadd_wd;
2611	else if (VecWidth == `256`)
2612	IID = Intrinsic::x86_avx2_pmadd_wd;
2613	else if (VecWidth == `512`)
2614	IID = Intrinsic::x86_avx512_pmaddw_d_512;
2615	else
2616	llvm_unreachable("Unexpected intrinsic");
2617	} else if (Name.starts_with(Prefix: "pmaddubs.w.")) {
2618	if (VecWidth == `128`)
2619	IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2620	else if (VecWidth == `256`)
2621	IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2622	else if (VecWidth == `512`)
2623	IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2624	else
2625	llvm_unreachable("Unexpected intrinsic");
2626	} else if (Name.starts_with(Prefix: "packsswb.")) {
2627	if (VecWidth == `128`)
2628	IID = Intrinsic::x86_sse2_packsswb_128;
2629	else if (VecWidth == `256`)
2630	IID = Intrinsic::x86_avx2_packsswb;
2631	else if (VecWidth == `512`)
2632	IID = Intrinsic::x86_avx512_packsswb_512;
2633	else
2634	llvm_unreachable("Unexpected intrinsic");
2635	} else if (Name.starts_with(Prefix: "packssdw.")) {
2636	if (VecWidth == `128`)
2637	IID = Intrinsic::x86_sse2_packssdw_128;
2638	else if (VecWidth == `256`)
2639	IID = Intrinsic::x86_avx2_packssdw;
2640	else if (VecWidth == `512`)
2641	IID = Intrinsic::x86_avx512_packssdw_512;
2642	else
2643	llvm_unreachable("Unexpected intrinsic");
2644	} else if (Name.starts_with(Prefix: "packuswb.")) {
2645	if (VecWidth == `128`)
2646	IID = Intrinsic::x86_sse2_packuswb_128;
2647	else if (VecWidth == `256`)
2648	IID = Intrinsic::x86_avx2_packuswb;
2649	else if (VecWidth == `512`)
2650	IID = Intrinsic::x86_avx512_packuswb_512;
2651	else
2652	llvm_unreachable("Unexpected intrinsic");
2653	} else if (Name.starts_with(Prefix: "packusdw.")) {
2654	if (VecWidth == `128`)
2655	IID = Intrinsic::x86_sse41_packusdw;
2656	else if (VecWidth == `256`)
2657	IID = Intrinsic::x86_avx2_packusdw;
2658	else if (VecWidth == `512`)
2659	IID = Intrinsic::x86_avx512_packusdw_512;
2660	else
2661	llvm_unreachable("Unexpected intrinsic");
2662	} else if (Name.starts_with(Prefix: "vpermilvar.")) {
2663	if (VecWidth == `128` && EltWidth == `32`)
2664	IID = Intrinsic::x86_avx_vpermilvar_ps;
2665	else if (VecWidth == `128` && EltWidth == `64`)
2666	IID = Intrinsic::x86_avx_vpermilvar_pd;
2667	else if (VecWidth == `256` && EltWidth == `32`)
2668	IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2669	else if (VecWidth == `256` && EltWidth == `64`)
2670	IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2671	else if (VecWidth == `512` && EltWidth == `32`)
2672	IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2673	else if (VecWidth == `512` && EltWidth == `64`)
2674	IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2675	else
2676	llvm_unreachable("Unexpected intrinsic");
2677	} else if (Name == "cvtpd2dq.256") {
2678	IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2679	} else if (Name == "cvtpd2ps.256") {
2680	IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2681	} else if (Name == "cvttpd2dq.256") {
2682	IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2683	} else if (Name == "cvttps2dq.128") {
2684	IID = Intrinsic::x86_sse2_cvttps2dq;
2685	} else if (Name == "cvttps2dq.256") {
2686	IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2687	} else if (Name.starts_with(Prefix: "permvar.")) {
2688	bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2689	if (VecWidth == `256` && EltWidth == `32` && IsFloat)
2690	IID = Intrinsic::x86_avx2_permps;
2691	else if (VecWidth == `256` && EltWidth == `32` && !IsFloat)
2692	IID = Intrinsic::x86_avx2_permd;
2693	else if (VecWidth == `256` && EltWidth == `64` && IsFloat)
2694	IID = Intrinsic::x86_avx512_permvar_df_256;
2695	else if (VecWidth == `256` && EltWidth == `64` && !IsFloat)
2696	IID = Intrinsic::x86_avx512_permvar_di_256;
2697	else if (VecWidth == `512` && EltWidth == `32` && IsFloat)
2698	IID = Intrinsic::x86_avx512_permvar_sf_512;
2699	else if (VecWidth == `512` && EltWidth == `32` && !IsFloat)
2700	IID = Intrinsic::x86_avx512_permvar_si_512;
2701	else if (VecWidth == `512` && EltWidth == `64` && IsFloat)
2702	IID = Intrinsic::x86_avx512_permvar_df_512;
2703	else if (VecWidth == `512` && EltWidth == `64` && !IsFloat)
2704	IID = Intrinsic::x86_avx512_permvar_di_512;
2705	else if (VecWidth == `128` && EltWidth == `16`)
2706	IID = Intrinsic::x86_avx512_permvar_hi_128;
2707	else if (VecWidth == `256` && EltWidth == `16`)
2708	IID = Intrinsic::x86_avx512_permvar_hi_256;
2709	else if (VecWidth == `512` && EltWidth == `16`)
2710	IID = Intrinsic::x86_avx512_permvar_hi_512;
2711	else if (VecWidth == `128` && EltWidth == `8`)
2712	IID = Intrinsic::x86_avx512_permvar_qi_128;
2713	else if (VecWidth == `256` && EltWidth == `8`)
2714	IID = Intrinsic::x86_avx512_permvar_qi_256;
2715	else if (VecWidth == `512` && EltWidth == `8`)
2716	IID = Intrinsic::x86_avx512_permvar_qi_512;
2717	else
2718	llvm_unreachable("Unexpected intrinsic");
2719	} else if (Name.starts_with(Prefix: "dbpsadbw.")) {
2720	if (VecWidth == `128`)
2721	IID = Intrinsic::x86_avx512_dbpsadbw_128;
2722	else if (VecWidth == `256`)
2723	IID = Intrinsic::x86_avx512_dbpsadbw_256;
2724	else if (VecWidth == `512`)
2725	IID = Intrinsic::x86_avx512_dbpsadbw_512;
2726	else
2727	llvm_unreachable("Unexpected intrinsic");
2728	} else if (Name.starts_with(Prefix: "pmultishift.qb.")) {
2729	if (VecWidth == `128`)
2730	IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2731	else if (VecWidth == `256`)
2732	IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2733	else if (VecWidth == `512`)
2734	IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2735	else
2736	llvm_unreachable("Unexpected intrinsic");
2737	} else if (Name.starts_with(Prefix: "conflict.")) {
2738	if (Name [`9`] == `'d'` && VecWidth == `128`)
2739	IID = Intrinsic::x86_avx512_conflict_d_128;
2740	else if (Name [`9`] == `'d'` && VecWidth == `256`)
2741	IID = Intrinsic::x86_avx512_conflict_d_256;
2742	else if (Name [`9`] == `'d'` && VecWidth == `512`)
2743	IID = Intrinsic::x86_avx512_conflict_d_512;
2744	else if (Name [`9`] == `'q'` && VecWidth == `128`)
2745	IID = Intrinsic::x86_avx512_conflict_q_128;
2746	else if (Name [`9`] == `'q'` && VecWidth == `256`)
2747	IID = Intrinsic::x86_avx512_conflict_q_256;
2748	else if (Name [`9`] == `'q'` && VecWidth == `512`)
2749	IID = Intrinsic::x86_avx512_conflict_q_512;
2750	else
2751	llvm_unreachable("Unexpected intrinsic");
2752	} else if (Name.starts_with(Prefix: "pavg.")) {
2753	if (Name [`5`] == `'b'` && VecWidth == `128`)
2754	IID = Intrinsic::x86_sse2_pavg_b;
2755	else if (Name [`5`] == `'b'` && VecWidth == `256`)
2756	IID = Intrinsic::x86_avx2_pavg_b;
2757	else if (Name [`5`] == `'b'` && VecWidth == `512`)
2758	IID = Intrinsic::x86_avx512_pavg_b_512;
2759	else if (Name [`5`] == `'w'` && VecWidth == `128`)
2760	IID = Intrinsic::x86_sse2_pavg_w;
2761	else if (Name [`5`] == `'w'` && VecWidth == `256`)
2762	IID = Intrinsic::x86_avx2_pavg_w;
2763	else if (Name [`5`] == `'w'` && VecWidth == `512`)
2764	IID = Intrinsic::x86_avx512_pavg_w_512;
2765	else
2766	llvm_unreachable("Unexpected intrinsic");
2767	} else
2768	return false;
2769
2770	SmallVector<Value *, `4`> Args(CI.args());
2771	Args.pop_back();
2772	Args.pop_back();
2773	Rep = Builder.CreateIntrinsic(ID: IID, Args);
2774	unsigned NumArgs = CI.arg_size();
2775	Rep = emitX86Select(Builder, Mask: CI.getArgOperand(i: NumArgs - `1`), Op0: Rep,
2776	Op1: CI.getArgOperand(i: NumArgs - `2`));
2777	return true;
2778	}
2779
2780	/// Upgrade comment in call to inline asm that represents an objc retain release
2781	/// marker.
2782	void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
2783	size_t Pos;
2784	if (AsmStr->find(s: "mov\tfp") == `0` &&
2785	AsmStr->find(s: "objc_retainAutoreleaseReturnValue") != std::string::npos &&
2786	(Pos = AsmStr->find(s: "# marker")) != std::string::npos) {
2787	AsmStr->replace(pos: Pos, n1: `1`, s: ";");
2788	}
2789	}
2790
2791	static Value upgradeNVVMIntrinsicCall(StringRef Name, CallBase CI,
2792	Function *F, IRBuilder<> &Builder) {
2793	Value Rep = nullptr*;
2794
2795	if (Name == "abs.i" \|\| Name == "abs.ll") {
2796	Value *Arg = CI->getArgOperand(i: `0`);
2797	Rep = Builder.CreateIntrinsic(ID: Intrinsic::abs, OverloadTypes: {Arg->getType()},
2798	Args: {Arg, Builder.getTrue()},
2799	/FMFSource=/nullptr, Name: "abs");
2800	} else if (Name == "abs.bf16" \|\| Name == "abs.bf16x2") {
2801	Type *Ty = (Name == "abs.bf16")
2802	? Builder.getBFloatTy()
2803	: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts: `2`);
2804	Value *Arg = Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: Ty);
2805	Value *Abs = Builder.CreateUnaryIntrinsic(ID: Intrinsic::nvvm_fabs, Op: Arg);
2806	Rep = Builder.CreateBitCast(V: Abs, DestTy: CI->getType());
2807	} else if (Name == "fabs.f" \|\| Name == "fabs.ftz.f" \|\| Name == "fabs.d") {
2808	Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
2809	: Intrinsic::nvvm_fabs;
2810	Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: `0`));
2811	} else if (Name.consume_front(Prefix: "ex2.approx.")) {
2812	// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2813	Intrinsic::ID IID = Name.starts_with(Prefix: "ftz") ? Intrinsic::nvvm_ex2_approx_ftz
2814	: Intrinsic::nvvm_ex2_approx;
2815	Rep = Builder.CreateUnaryIntrinsic(ID: IID, Op: CI->getArgOperand(i: `0`));
2816	} else if (Name.starts_with(Prefix: "atomic.load.add.f32.p") \|\|
2817	Name.starts_with(Prefix: "atomic.load.add.f64.p")) {
2818	Value *Ptr = CI->getArgOperand(i: `0`);
2819	Value *Val = CI->getArgOperand(i: `1`);
2820	Rep = Builder.CreateAtomicRMW(
2821	Op: AtomicRMWInst::FAdd, Ptr, Val, Align: MaybeAlign (), Ordering: AtomicOrdering::Monotonic,
2822	SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
2823	// The default scope for atomic.load. intrinsics is device*
2824	// (= gpu scope in ptx), but the default LLVM atomic scope is
2825	// "system"
2826	} else if (Name.starts_with(Prefix: "atomic.load.inc.32.p") \|\|
2827	Name.starts_with(Prefix: "atomic.load.dec.32.p")) {
2828	Value *Ptr = CI->getArgOperand(i: `0`);
2829	Value *Val = CI->getArgOperand(i: `1`);
2830	auto Op = Name.starts_with(Prefix: "atomic.load.inc") ? AtomicRMWInst::UIncWrap
2831	: AtomicRMWInst::UDecWrap;
2832	Rep = Builder.CreateAtomicRMW(
2833	Op, Ptr, Val, Align: MaybeAlign (), Ordering: AtomicOrdering::Monotonic,
2834	SSID: CI->getContext().getOrInsertSyncScopeID(SSN: "device"));
2835	// See comment above.
2836	} else if (Name.starts_with(Prefix: "atomic.") && Name.contains(Other: ".gen.")) {
2837	// nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
2838	StringRef Op = Name.substr(Start: StringRef ("atomic.").size());
2839	Value *Ptr = CI->getArgOperand(i: `0`);
2840	Value *Val = CI->getArgOperand(i: `1`);
2841	SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
2842	SSN: Op.contains(Other: ".cta.") ? "block" : "");
2843	if (Op.starts_with(Prefix: "cas.")) {
2844	Value *New = CI->getArgOperand(i: `2`);
2845	Value *Pair = Builder.CreateAtomicCmpXchg(
2846	Ptr, Cmp: Val, New, Align: MaybeAlign (), SuccessOrdering: AtomicOrdering::Monotonic,
2847	FailureOrdering: AtomicOrdering::Monotonic, SSID);
2848	Rep = Builder.CreateExtractValue(Agg: Pair, Idxs: `0`);
2849	} else {
2850	// Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
2851	// because we were actually missing those intrinsics!
2852	AtomicRMWInst::BinOp BinOp =
2853	StringSwitch<AtomicRMWInst::BinOp>(Op)
2854	.StartsWith(S: "add.gen.f", Value: AtomicRMWInst::FAdd)
2855	.StartsWith(S: "add.gen.i", Value: AtomicRMWInst::Add)
2856	.StartsWith(S: "exch.", Value: AtomicRMWInst::Xchg)
2857	.StartsWith(S: "max.", Value: AtomicRMWInst::Max)
2858	.StartsWith(S: "min.", Value: AtomicRMWInst::Min)
2859	.StartsWith(S: "inc.", Value: AtomicRMWInst::UIncWrap)
2860	.StartsWith(S: "dec.", Value: AtomicRMWInst::UDecWrap)
2861	.StartsWith(S: "and.", Value: AtomicRMWInst::And)
2862	.StartsWith(S: "or.", Value: AtomicRMWInst::Or)
2863	.StartsWith(S: "xor.", Value: AtomicRMWInst::Xor)
2864	.Default(Value: AtomicRMWInst::BAD_BINOP);
2865	assert(BinOp != AtomicRMWInst::BAD_BINOP &&
2866	"unexpected nvvm scoped atomic intrinsic");
2867	Rep = Builder.CreateAtomicRMW(Op: BinOp, Ptr, Val, Align: MaybeAlign (),
2868	Ordering: AtomicOrdering::Monotonic, SSID);
2869	}
2870	} else if (Name == "clz.ll") {
2871	// llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
2872	Value *Arg = CI->getArgOperand(i: `0`);
2873	Value *Ctlz = Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: {Arg->getType()},
2874	Args: {Arg, Builder.getFalse()},
2875	/FMFSource=/nullptr, Name: "ctlz");
2876	Rep = Builder.CreateTrunc(V: Ctlz, DestTy: Builder.getInt32Ty(), Name: "ctlz.trunc");
2877	} else if (Name == "popc.ll") {
2878	// llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
2879	// i64.
2880	Value *Arg = CI->getArgOperand(i: `0`);
2881	Value *Popc = Builder.CreateIntrinsic(ID: Intrinsic::ctpop, OverloadTypes: {Arg->getType()},
2882	Args: Arg, /FMFSource=/nullptr, Name: "ctpop");
2883	Rep = Builder.CreateTrunc(V: Popc, DestTy: Builder.getInt32Ty(), Name: "ctpop.trunc");
2884	} else if (Name == "h2f") {
2885	Value *Cast =
2886	Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: Builder.getHalfTy());
2887	Rep = Builder.CreateFPExt(V: Cast, DestTy: Builder.getFloatTy());
2888	} else if (Name.consume_front(Prefix: "bitcast.") &&
2889	(Name == "f2i" \|\| Name == "i2f" \|\| Name == "ll2d" \|\|
2890	Name == "d2ll")) {
2891	Rep = Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: CI->getType());
2892	} else if (Name == "rotate.b32") {
2893	Value *Arg = CI->getOperand(i_nocapture: `0`);
2894	Value *ShiftAmt = CI->getOperand(i_nocapture: `1`);
2895	Rep = Builder.CreateIntrinsic(RetTy: Builder.getInt32Ty(), ID: Intrinsic::fshl,
2896	Args: {Arg, Arg, ShiftAmt});
2897	} else if (Name == "rotate.b64") {
2898	Type *Int64Ty = Builder.getInt64Ty();
2899	Value *Arg = CI->getOperand(i_nocapture: `0`);
2900	Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: `1`), DestTy: Int64Ty);
2901	Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
2902	Args: {Arg, Arg, ZExtShiftAmt});
2903	} else if (Name == "rotate.right.b64") {
2904	Type *Int64Ty = Builder.getInt64Ty();
2905	Value *Arg = CI->getOperand(i_nocapture: `0`);
2906	Value *ZExtShiftAmt = Builder.CreateZExt(V: CI->getOperand(i_nocapture: `1`), DestTy: Int64Ty);
2907	Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshr,
2908	Args: {Arg, Arg, ZExtShiftAmt});
2909	} else if (Name == "swap.lo.hi.b64") {
2910	Type *Int64Ty = Builder.getInt64Ty();
2911	Value *Arg = CI->getOperand(i_nocapture: `0`);
2912	Rep = Builder.CreateIntrinsic(RetTy: Int64Ty, ID: Intrinsic::fshl,
2913	Args: {Arg, Arg, Builder.getInt64(C: `32`)});
2914	} else if ((Name.consume_front(Prefix: "ptr.gen.to.") &&
2915	consumeNVVMPtrAddrSpace(Name)) \|\|
2916	(Name.consume_front(Prefix: "ptr.") && consumeNVVMPtrAddrSpace(Name) &&
2917	Name.starts_with(Prefix: ".to.gen"))) {
2918	Rep = Builder.CreateAddrSpaceCast(V: CI->getArgOperand(i: `0`), DestTy: CI->getType());
2919	} else if (Name.consume_front(Prefix: "ldg.global")) {
2920	Value *Ptr = CI->getArgOperand(i: `0`);
2921	Align PtrAlign = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getAlignValue();
2922	// Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
2923	Value *ASC = Builder.CreateAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: `1`));
2924	Instruction *LD = Builder.CreateAlignedLoad(Ty: CI->getType(), Ptr: ASC, Align: PtrAlign);
2925	MDNode *MD = MDNode::get(Context&: Builder.getContext(), MDs: {});
2926	LD->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MD);
2927	return LD;
2928	} else if (Name == "tanh.approx.f32") {
2929	// nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
2930	FastMathFlags FMF;
2931	FMF.setApproxFunc();
2932	Rep = Builder.CreateUnaryIntrinsic(ID: Intrinsic::tanh, Op: CI->getArgOperand(i: `0`),
2933	FMFSource: FMF);
2934	} else if (Name == "barrier0" \|\| Name == "barrier.n" \|\| Name == "bar.sync") {
2935	Value *Arg =
2936	Name.ends_with(Suffix: `'0'`) ? Builder.getInt32(C: `0`) : CI->getArgOperand(i: `0`);
2937	Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_aligned_all,
2938	OverloadTypes: {}, Args: {Arg});
2939	} else if (Name == "barrier") {
2940	Rep = Builder.CreateIntrinsic(
2941	ID: Intrinsic::nvvm_barrier_cta_sync_aligned_count, OverloadTypes: {},
2942	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`)});
2943	} else if (Name == "barrier.sync") {
2944	Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_all, OverloadTypes: {},
2945	Args: {CI->getArgOperand(i: `0`)});
2946	} else if (Name == "barrier.sync.cnt") {
2947	Rep = Builder.CreateIntrinsic(ID: Intrinsic::nvvm_barrier_cta_sync_count, OverloadTypes: {},
2948	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`)});
2949	} else if (Name == "barrier0.popc" \|\| Name == "barrier0.and" \|\|
2950	Name == "barrier0.or") {
2951	Value *C = CI->getArgOperand(i: `0`);
2952	C = Builder.CreateICmpNE(LHS: C, RHS: Builder.getInt32(C: `0`));
2953
2954	Intrinsic::ID IID =
2955	StringSwitch<Intrinsic::ID>(Name)
2956	.Case(S: "barrier0.popc",
2957	Value: Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
2958	.Case(S: "barrier0.and",
2959	Value: Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
2960	.Case(S: "barrier0.or",
2961	Value: Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
2962	Value *Bar = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {}, Args: {Builder.getInt32(C: `0`), C});
2963	Rep = Builder.CreateZExt(V: Bar, DestTy: CI->getType());
2964	} else {
2965	Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2966	if (IID != Intrinsic::not_intrinsic &&
2967	!F->getReturnType()->getScalarType()->isBFloatTy()) {
2968	rename(GV: F);
2969	Function *NewFn = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: IID);
2970	SmallVector<Value *, `2`> Args;
2971	for (size_t I = `0`; I < NewFn->arg_size(); ++I) {
2972	Value *Arg = CI->getArgOperand(i: I);
2973	Type *OldType = Arg->getType();
2974	Type *NewType = NewFn->getArg(i: I)->getType();
2975	Args.push_back(
2976	Elt: (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
2977	? Builder.CreateBitCast(V: Arg, DestTy: NewType)
2978	: Arg);
2979	}
2980	Rep = Builder.CreateCall(Callee: NewFn, Args);
2981	if (F->getReturnType()->isIntegerTy())
2982	Rep = Builder.CreateBitCast(V: Rep, DestTy: F->getReturnType());
2983	}
2984	}
2985
2986	return Rep;
2987	}
2988
2989	static Value upgradeX86IntrinsicCall(StringRef Name, CallBase CI, Function *F,
2990	IRBuilder<> &Builder) {
2991	LLVMContext &C = F->getContext();
2992	Value Rep = nullptr*;
2993
2994	if (Name.starts_with(Prefix: "sse4a.movnt.")) {
2995	SmallVector<Metadata *, `1`> Elts;
2996	Elts.push_back(
2997	Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: `1`)));
2998	MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
2999
3000	Value *Arg0 = CI->getArgOperand(i: `0`);
3001	Value *Arg1 = CI->getArgOperand(i: `1`);
3002
3003	// Nontemporal (unaligned) store of the 0'th element of the float/double
3004	// vector.
3005	Value *Extract =
3006	Builder.CreateExtractElement(Vec: Arg1, Idx: (uint64_t)`0`, Name: "extractelement");
3007
3008	StoreInst *SI = Builder.CreateAlignedStore(Val: Extract, Ptr: Arg0, Align: Align (`1`));
3009	SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3010	} else if (Name.starts_with(Prefix: "avx.movnt.") \|\|
3011	Name.starts_with(Prefix: "avx512.storent.")) {
3012	SmallVector<Metadata *, `1`> Elts;
3013	Elts.push_back(
3014	Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: `1`)));
3015	MDNode *Node = MDNode::get(Context&: C, MDs: Elts);
3016
3017	Value *Arg0 = CI->getArgOperand(i: `0`);
3018	Value *Arg1 = CI->getArgOperand(i: `1`);
3019
3020	StoreInst *SI = Builder.CreateAlignedStore(
3021	Val: Arg1, Ptr: Arg0,
3022	Align: Align (Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / `8`));
3023	SI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
3024	} else if (Name == "sse2.storel.dq") {
3025	Value *Arg0 = CI->getArgOperand(i: `0`);
3026	Value *Arg1 = CI->getArgOperand(i: `1`);
3027
3028	auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: `2`);
3029	Value *BC0 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
3030	Value *Elt = Builder.CreateExtractElement(Vec: BC0, Idx: (uint64_t)`0`);
3031	Builder.CreateAlignedStore(Val: Elt, Ptr: Arg0, Align: Align (`1`));
3032	} else if (Name.starts_with(Prefix: "sse.storeu.") \|\|
3033	Name.starts_with(Prefix: "sse2.storeu.") \|\|
3034	Name.starts_with(Prefix: "avx.storeu.")) {
3035	Value *Arg0 = CI->getArgOperand(i: `0`);
3036	Value *Arg1 = CI->getArgOperand(i: `1`);
3037	Builder.CreateAlignedStore(Val: Arg1, Ptr: Arg0, Align: Align (`1`));
3038	} else if (Name == "avx512.mask.store.ss") {
3039	Value *Mask = Builder.CreateAnd(LHS: CI->getArgOperand(i: `2`), RHS: Builder.getInt8(C: `1`));
3040	upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: `0`), Data: CI->getArgOperand(i: `1`),
3041	Mask, Aligned: false);
3042	} else if (Name.starts_with(Prefix: "avx512.mask.store")) {
3043	// "avx512.mask.storeu." or "avx512.mask.store."
3044	bool Aligned = Name [`17`] != `'u'`; // "avx512.mask.storeu".
3045	upgradeMaskedStore(Builder, Ptr: CI->getArgOperand(i: `0`), Data: CI->getArgOperand(i: `1`),
3046	Mask: CI->getArgOperand(i: `2`), Aligned);
3047	} else if (Name.starts_with(Prefix: "sse2.pcmp") \|\| Name.starts_with(Prefix: "avx2.pcmp")) {
3048	// Upgrade packed integer vector compare intrinsics to compare instructions.
3049	// "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3050	bool CmpEq = Name [`9`] == `'e'`;
3051	Rep = Builder.CreateICmp(P: CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3052	LHS: CI->getArgOperand(i: `0`), RHS: CI->getArgOperand(i: `1`));
3053	Rep = Builder.CreateSExt(V: Rep, DestTy: CI->getType(), Name: "");
3054	} else if (Name.starts_with(Prefix: "avx512.broadcastm")) {
3055	Type *ExtTy = Type::getInt32Ty(C);
3056	if (CI->getOperand(i_nocapture: `0`)->getType()->isIntegerTy(BitWidth: `8`))
3057	ExtTy = Type::getInt64Ty(C);
3058	unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3059	ExtTy->getPrimitiveSizeInBits();
3060	Rep = Builder.CreateZExt(V: CI->getArgOperand(i: `0`), DestTy: ExtTy);
3061	Rep = Builder.CreateVectorSplat(NumElts, V: Rep);
3062	} else if (Name == "sse.sqrt.ss" \|\| Name == "sse2.sqrt.sd") {
3063	Value *Vec = CI->getArgOperand(i: `0`);
3064	Value *Elt0 = Builder.CreateExtractElement(Vec, Idx: (uint64_t)`0`);
3065	Elt0 = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: Elt0->getType(), Args: Elt0);
3066	Rep = Builder.CreateInsertElement(Vec, NewElt: Elt0, Idx: (uint64_t)`0`);
3067	} else if (Name.starts_with(Prefix: "avx.sqrt.p") \|\|
3068	Name.starts_with(Prefix: "sse2.sqrt.p") \|\|
3069	Name.starts_with(Prefix: "sse.sqrt.p")) {
3070	Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3071	Args: {CI->getArgOperand(i: `0`)});
3072	} else if (Name.starts_with(Prefix: "avx512.mask.sqrt.p")) {
3073	if (CI->arg_size() == `4` &&
3074	(!isa<ConstantInt>(Val: CI->getArgOperand(i: `3`)) \|\|
3075	cast<ConstantInt>(Val: CI->getArgOperand(i: `3`))->getZExtValue() != `4`)) {
3076	Intrinsic::ID IID = Name [`18`] == `'s'` ? Intrinsic::x86_avx512_sqrt_ps_512
3077	: Intrinsic::x86_avx512_sqrt_pd_512;
3078
3079	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `3`)};
3080	Rep = Builder.CreateIntrinsic(ID: IID, Args);
3081	} else {
3082	Rep = Builder.CreateIntrinsic(ID: Intrinsic::sqrt, OverloadTypes: CI->getType(),
3083	Args: {CI->getArgOperand(i: `0`)});
3084	}
3085	Rep =
3086	emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep, Op1: CI->getArgOperand(i: `1`));
3087	} else if (Name.starts_with(Prefix: "avx512.ptestm") \|\|
3088	Name.starts_with(Prefix: "avx512.ptestnm")) {
3089	Value *Op0 = CI->getArgOperand(i: `0`);
3090	Value *Op1 = CI->getArgOperand(i: `1`);
3091	Value *Mask = CI->getArgOperand(i: `2`);
3092	Rep = Builder.CreateAnd(LHS: Op0, RHS: Op1);
3093	llvm::Type *Ty = Op0->getType();
3094	Value *Zero = llvm::Constant::getNullValue(Ty);
3095	ICmpInst::Predicate Pred = Name.starts_with(Prefix: "avx512.ptestm")
3096	? ICmpInst::ICMP_NE
3097	: ICmpInst::ICMP_EQ;
3098	Rep = Builder.CreateICmp(P: Pred, LHS: Rep, RHS: Zero);
3099	Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask);
3100	} else if (Name.starts_with(Prefix: "avx512.mask.pbroadcast")) {
3101	unsigned NumElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: `1`)->getType())
3102	->getNumElements();
3103	Rep = Builder.CreateVectorSplat(NumElts, V: CI->getArgOperand(i: `0`));
3104	Rep =
3105	emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep, Op1: CI->getArgOperand(i: `1`));
3106	} else if (Name.starts_with(Prefix: "avx512.kunpck")) {
3107	unsigned NumElts = CI->getType()->getScalarSizeInBits();
3108	Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts);
3109	Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `1`), NumElts);
3110	int Indices[`64`];
3111	for (unsigned i = `0`; i != NumElts; ++i)
3112	Indices[i] = i;
3113
3114	// First extract half of each vector. This gives better codegen than
3115	// doing it in a single shuffle.
3116	LHS = Builder.CreateShuffleVector(V1: LHS, V2: LHS, Mask: ArrayRef(Indices, NumElts / `2`));
3117	RHS = Builder.CreateShuffleVector(V1: RHS, V2: RHS, Mask: ArrayRef(Indices, NumElts / `2`));
3118	// Concat the vectors.
3119	// NOTE: Operands have to be swapped to match intrinsic definition.
3120	Rep = Builder.CreateShuffleVector(V1: RHS, V2: LHS, Mask: ArrayRef(Indices, NumElts));
3121	Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3122	} else if (Name == "avx512.kand.w") {
3123	Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts: `16`);
3124	Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `1`), NumElts: `16`);
3125	Rep = Builder.CreateAnd(LHS, RHS);
3126	Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3127	} else if (Name == "avx512.kandn.w") {
3128	Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts: `16`);
3129	Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `1`), NumElts: `16`);
3130	LHS = Builder.CreateNot(V: LHS);
3131	Rep = Builder.CreateAnd(LHS, RHS);
3132	Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3133	} else if (Name == "avx512.kor.w") {
3134	Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts: `16`);
3135	Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `1`), NumElts: `16`);
3136	Rep = Builder.CreateOr(LHS, RHS);
3137	Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3138	} else if (Name == "avx512.kxor.w") {
3139	Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts: `16`);
3140	Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `1`), NumElts: `16`);
3141	Rep = Builder.CreateXor(LHS, RHS);
3142	Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3143	} else if (Name == "avx512.kxnor.w") {
3144	Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts: `16`);
3145	Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `1`), NumElts: `16`);
3146	LHS = Builder.CreateNot(V: LHS);
3147	Rep = Builder.CreateXor(LHS, RHS);
3148	Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3149	} else if (Name == "avx512.knot.w") {
3150	Rep = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts: `16`);
3151	Rep = Builder.CreateNot(V: Rep);
3152	Rep = Builder.CreateBitCast(V: Rep, DestTy: CI->getType());
3153	} else if (Name == "avx512.kortestz.w" \|\| Name == "avx512.kortestc.w") {
3154	Value *LHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `0`), NumElts: `16`);
3155	Value *RHS = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `1`), NumElts: `16`);
3156	Rep = Builder.CreateOr(LHS, RHS);
3157	Rep = Builder.CreateBitCast(V: Rep, DestTy: Builder.getInt16Ty());
3158	Value *C;
3159	if (Name [`14`] == `'c'`)
3160	C = ConstantInt::getAllOnesValue(Ty: Builder.getInt16Ty());
3161	else
3162	C = ConstantInt::getNullValue(Ty: Builder.getInt16Ty());
3163	Rep = Builder.CreateICmpEQ(LHS: Rep, RHS: C);
3164	Rep = Builder.CreateZExt(V: Rep, DestTy: Builder.getInt32Ty());
3165	} else if (Name == "sse.add.ss" \|\| Name == "sse2.add.sd" \|\|
3166	Name == "sse.sub.ss" \|\| Name == "sse2.sub.sd" \|\|
3167	Name == "sse.mul.ss" \|\| Name == "sse2.mul.sd" \|\|
3168	Name == "sse.div.ss" \|\| Name == "sse2.div.sd") {
3169	Type *I32Ty = Type::getInt32Ty(C);
3170	Value *Elt0 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: `0`),
3171	Idx: ConstantInt::get(Ty: I32Ty, V: `0`));
3172	Value *Elt1 = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: `1`),
3173	Idx: ConstantInt::get(Ty: I32Ty, V: `0`));
3174	Value *EltOp;
3175	if (Name.contains(Other: ".add."))
3176	EltOp = Builder.CreateFAdd(L: Elt0, R: Elt1);
3177	else if (Name.contains(Other: ".sub."))
3178	EltOp = Builder.CreateFSub(L: Elt0, R: Elt1);
3179	else if (Name.contains(Other: ".mul."))
3180	EltOp = Builder.CreateFMul(L: Elt0, R: Elt1);
3181	else
3182	EltOp = Builder.CreateFDiv(L: Elt0, R: Elt1);
3183	Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: `0`), NewElt: EltOp,
3184	Idx: ConstantInt::get(Ty: I32Ty, V: `0`));
3185	} else if (Name.starts_with(Prefix: "avx512.mask.pcmp")) {
3186	// "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3187	bool CmpEq = Name [`16`] == `'e'`;
3188	Rep = upgradeMaskedCompare(Builder, CI&: CI, CC: CmpEq ? `0` : `6`, Signed: true*);
3189	} else if (Name.starts_with(Prefix: "avx512.mask.vpshufbitqmb.")) {
3190	Type *OpTy = CI->getArgOperand(i: `0`)->getType();
3191	unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3192	Intrinsic::ID IID;
3193	switch (VecWidth) {
3194	default:
3195	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3196	break;
3197	case `128`:
3198	IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3199	break;
3200	case `256`:
3201	IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3202	break;
3203	case `512`:
3204	IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3205	break;
3206	}
3207
3208	Rep =
3209	Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: `0`), CI->getArgOperand(i: `1`)});
3210	Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: `2`));
3211	} else if (Name.starts_with(Prefix: "avx512.mask.fpclass.p")) {
3212	Type *OpTy = CI->getArgOperand(i: `0`)->getType();
3213	unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3214	unsigned EltWidth = OpTy->getScalarSizeInBits();
3215	Intrinsic::ID IID;
3216	if (VecWidth == `128` && EltWidth == `32`)
3217	IID = Intrinsic::x86_avx512_fpclass_ps_128;
3218	else if (VecWidth == `256` && EltWidth == `32`)
3219	IID = Intrinsic::x86_avx512_fpclass_ps_256;
3220	else if (VecWidth == `512` && EltWidth == `32`)
3221	IID = Intrinsic::x86_avx512_fpclass_ps_512;
3222	else if (VecWidth == `128` && EltWidth == `64`)
3223	IID = Intrinsic::x86_avx512_fpclass_pd_128;
3224	else if (VecWidth == `256` && EltWidth == `64`)
3225	IID = Intrinsic::x86_avx512_fpclass_pd_256;
3226	else if (VecWidth == `512` && EltWidth == `64`)
3227	IID = Intrinsic::x86_avx512_fpclass_pd_512;
3228	else
3229	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3230
3231	Rep =
3232	Builder.CreateIntrinsic(ID: IID, Args: {CI->getOperand(i_nocapture: `0`), CI->getArgOperand(i: `1`)});
3233	Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: CI->getArgOperand(i: `2`));
3234	} else if (Name.starts_with(Prefix: "avx512.cmp.p")) {
3235	SmallVector<Value *, `4`> Args(CI->args());
3236	Type *OpTy = Args [`0`]->getType();
3237	unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3238	unsigned EltWidth = OpTy->getScalarSizeInBits();
3239	Intrinsic::ID IID;
3240	if (VecWidth == `128` && EltWidth == `32`)
3241	IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3242	else if (VecWidth == `256` && EltWidth == `32`)
3243	IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3244	else if (VecWidth == `512` && EltWidth == `32`)
3245	IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3246	else if (VecWidth == `128` && EltWidth == `64`)
3247	IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3248	else if (VecWidth == `256` && EltWidth == `64`)
3249	IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3250	else if (VecWidth == `512` && EltWidth == `64`)
3251	IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3252	else
3253	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
3254
3255	Value *Mask = Constant::getAllOnesValue(Ty: CI->getType());
3256	if (VecWidth == `512`)
3257	std::swap(a&: Mask, b&: Args.back());
3258	Args.push_back(Elt: Mask);
3259
3260	Rep = Builder.CreateIntrinsic(ID: IID, Args);
3261	} else if (Name.starts_with(Prefix: "avx512.mask.cmp.")) {
3262	// Integer compare intrinsics.
3263	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3264	Rep = upgradeMaskedCompare(Builder, CI&: CI, CC: Imm, Signed: true*);
3265	} else if (Name.starts_with(Prefix: "avx512.mask.ucmp.")) {
3266	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3267	Rep = upgradeMaskedCompare(Builder, CI&: CI, CC: Imm, Signed: false*);
3268	} else if (Name.starts_with(Prefix: "avx512.cvtb2mask.") \|\|
3269	Name.starts_with(Prefix: "avx512.cvtw2mask.") \|\|
3270	Name.starts_with(Prefix: "avx512.cvtd2mask.") \|\|
3271	Name.starts_with(Prefix: "avx512.cvtq2mask.")) {
3272	Value *Op = CI->getArgOperand(i: `0`);
3273	Value *Zero = llvm::Constant::getNullValue(Ty: Op->getType());
3274	Rep = Builder.CreateICmp(P: ICmpInst::ICMP_SLT, LHS: Op, RHS: Zero);
3275	Rep = applyX86MaskOn1BitsVec(Builder, Vec: Rep, Mask: nullptr);
3276	} else if (Name == "ssse3.pabs.b.128" \|\| Name == "ssse3.pabs.w.128" \|\|
3277	Name == "ssse3.pabs.d.128" \|\| Name.starts_with(Prefix: "avx2.pabs") \|\|
3278	Name.starts_with(Prefix: "avx512.mask.pabs")) {
3279	Rep = upgradeAbs(Builder, CI&: *CI);
3280	} else if (Name == "sse41.pmaxsb" \|\| Name == "sse2.pmaxs.w" \|\|
3281	Name == "sse41.pmaxsd" \|\| Name.starts_with(Prefix: "avx2.pmaxs") \|\|
3282	Name.starts_with(Prefix: "avx512.mask.pmaxs")) {
3283	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smax);
3284	} else if (Name == "sse2.pmaxu.b" \|\| Name == "sse41.pmaxuw" \|\|
3285	Name == "sse41.pmaxud" \|\| Name.starts_with(Prefix: "avx2.pmaxu") \|\|
3286	Name.starts_with(Prefix: "avx512.mask.pmaxu")) {
3287	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umax);
3288	} else if (Name == "sse41.pminsb" \|\| Name == "sse2.pmins.w" \|\|
3289	Name == "sse41.pminsd" \|\| Name.starts_with(Prefix: "avx2.pmins") \|\|
3290	Name.starts_with(Prefix: "avx512.mask.pmins")) {
3291	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::smin);
3292	} else if (Name == "sse2.pminu.b" \|\| Name == "sse41.pminuw" \|\|
3293	Name == "sse41.pminud" \|\| Name.starts_with(Prefix: "avx2.pminu") \|\|
3294	Name.starts_with(Prefix: "avx512.mask.pminu")) {
3295	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::umin);
3296	} else if (Name == "sse2.pmulu.dq" \|\| Name == "avx2.pmulu.dq" \|\|
3297	Name == "avx512.pmulu.dq.512" \|\|
3298	Name.starts_with(Prefix: "avx512.mask.pmulu.dq.")) {
3299	Rep = upgradePMULDQ(Builder, CI&: CI, /Signed/* IsSigned: false);
3300	} else if (Name == "sse41.pmuldq" \|\| Name == "avx2.pmul.dq" \|\|
3301	Name == "avx512.pmul.dq.512" \|\|
3302	Name.starts_with(Prefix: "avx512.mask.pmul.dq.")) {
3303	Rep = upgradePMULDQ(Builder, CI&: CI, /Signed/* IsSigned: true);
3304	} else if (Name == "sse.cvtsi2ss" \|\| Name == "sse2.cvtsi2sd" \|\|
3305	Name == "sse.cvtsi642ss" \|\| Name == "sse2.cvtsi642sd") {
3306	Rep =
3307	Builder.CreateSIToFP(V: CI->getArgOperand(i: `1`),
3308	DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3309	Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: `0`), NewElt: Rep, Idx: (uint64_t)`0`);
3310	} else if (Name == "avx512.cvtusi2sd") {
3311	Rep =
3312	Builder.CreateUIToFP(V: CI->getArgOperand(i: `1`),
3313	DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3314	Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: `0`), NewElt: Rep, Idx: (uint64_t)`0`);
3315	} else if (Name == "sse2.cvtss2sd") {
3316	Rep = Builder.CreateExtractElement(Vec: CI->getArgOperand(i: `1`), Idx: (uint64_t)`0`);
3317	Rep = Builder.CreateFPExt(
3318	V: Rep, DestTy: cast<VectorType>(Val: CI->getType())->getElementType());
3319	Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: `0`), NewElt: Rep, Idx: (uint64_t)`0`);
3320	} else if (Name == "sse2.cvtdq2pd" \|\| Name == "sse2.cvtdq2ps" \|\|
3321	Name == "avx.cvtdq2.pd.256" \|\| Name == "avx.cvtdq2.ps.256" \|\|
3322	Name.starts_with(Prefix: "avx512.mask.cvtdq2pd.") \|\|
3323	Name.starts_with(Prefix: "avx512.mask.cvtudq2pd.") \|\|
3324	Name.starts_with(Prefix: "avx512.mask.cvtdq2ps.") \|\|
3325	Name.starts_with(Prefix: "avx512.mask.cvtudq2ps.") \|\|
3326	Name.starts_with(Prefix: "avx512.mask.cvtqq2pd.") \|\|
3327	Name.starts_with(Prefix: "avx512.mask.cvtuqq2pd.") \|\|
3328	Name == "avx512.mask.cvtqq2ps.256" \|\|
3329	Name == "avx512.mask.cvtqq2ps.512" \|\|
3330	Name == "avx512.mask.cvtuqq2ps.256" \|\|
3331	Name == "avx512.mask.cvtuqq2ps.512" \|\| Name == "sse2.cvtps2pd" \|\|
3332	Name == "avx.cvt.ps2.pd.256" \|\|
3333	Name == "avx512.mask.cvtps2pd.128" \|\|
3334	Name == "avx512.mask.cvtps2pd.256") {
3335	auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3336	Rep = CI->getArgOperand(i: `0`);
3337	auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3338
3339	unsigned NumDstElts = DstTy->getNumElements();
3340	if (NumDstElts < SrcTy->getNumElements()) {
3341	assert(NumDstElts == `2` && "Unexpected vector size");
3342	Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{`0`, `1`});
3343	}
3344
3345	bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3346	bool IsUnsigned = Name.contains(Other: "cvtu");
3347	if (IsPS2PD)
3348	Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtps2pd");
3349	else if (CI->arg_size() == `4` &&
3350	(!isa<ConstantInt>(Val: CI->getArgOperand(i: `3`)) \|\|
3351	cast<ConstantInt>(Val: CI->getArgOperand(i: `3`))->getZExtValue() != `4`)) {
3352	Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3353	: Intrinsic::x86_avx512_sitofp_round;
3354	Rep = Builder.CreateIntrinsic(ID: IID, OverloadTypes: {DstTy, SrcTy},
3355	Args: {Rep, CI->getArgOperand(i: `3`)});
3356	} else {
3357	Rep = IsUnsigned ? Builder.CreateUIToFP(V: Rep, DestTy: DstTy, Name: "cvt")
3358	: Builder.CreateSIToFP(V: Rep, DestTy: DstTy, Name: "cvt");
3359	}
3360
3361	if (CI->arg_size() >= `3`)
3362	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep,
3363	Op1: CI->getArgOperand(i: `1`));
3364	} else if (Name.starts_with(Prefix: "avx512.mask.vcvtph2ps.") \|\|
3365	Name.starts_with(Prefix: "vcvtph2ps.")) {
3366	auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3367	Rep = CI->getArgOperand(i: `0`);
3368	auto *SrcTy = cast<FixedVectorType>(Val: Rep->getType());
3369	unsigned NumDstElts = DstTy->getNumElements();
3370	if (NumDstElts != SrcTy->getNumElements()) {
3371	assert(NumDstElts == `4` && "Unexpected vector size");
3372	Rep = Builder.CreateShuffleVector(V1: Rep, V2: Rep, Mask: ArrayRef<int>{`0`, `1`, `2`, `3`});
3373	}
3374	Rep = Builder.CreateBitCast(
3375	V: Rep, DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C), NumElts: NumDstElts));
3376	Rep = Builder.CreateFPExt(V: Rep, DestTy: DstTy, Name: "cvtph2ps");
3377	if (CI->arg_size() >= `3`)
3378	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep,
3379	Op1: CI->getArgOperand(i: `1`));
3380	} else if (Name.starts_with(Prefix: "avx512.mask.load")) {
3381	// "avx512.mask.loadu." or "avx512.mask.load."
3382	bool Aligned = Name [`16`] != `'u'`; // "avx512.mask.loadu".
3383	Rep = upgradeMaskedLoad(Builder, Ptr: CI->getArgOperand(i: `0`), Passthru: CI->getArgOperand(i: `1`),
3384	Mask: CI->getArgOperand(i: `2`), Aligned);
3385	} else if (Name.starts_with(Prefix: "avx512.mask.expand.load.")) {
3386	auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3387	auto *PtrTy = CI->getOperand(i_nocapture: `0`)->getType();
3388	Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `2`),
3389	NumElts: ResultTy->getNumElements());
3390	Rep = Builder.CreateIntrinsic(
3391	ID: Intrinsic::masked_expandload, OverloadTypes: {ResultTy, PtrTy},
3392	Args: {CI->getOperand(i_nocapture: `0`), MaskVec, CI->getOperand(i_nocapture: `1`)});
3393	} else if (Name.starts_with(Prefix: "avx512.mask.compress.store.")) {
3394	auto *ResultTy = cast<VectorType>(Val: CI->getArgOperand(i: `1`)->getType());
3395	auto *PtrTy = CI->getArgOperand(i: `0`)->getType();
3396	Value *MaskVec =
3397	getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `2`),
3398	NumElts: cast<FixedVectorType>(Val: ResultTy)->getNumElements());
3399	Rep = Builder.CreateIntrinsic(
3400	ID: Intrinsic::masked_compressstore, OverloadTypes: {ResultTy, PtrTy},
3401	Args: {CI->getArgOperand(i: `1`), CI->getArgOperand(i: `0`), MaskVec});
3402	} else if (Name.starts_with(Prefix: "avx512.mask.compress.") \|\|
3403	Name.starts_with(Prefix: "avx512.mask.expand.")) {
3404	auto *ResultTy = cast<FixedVectorType>(Val: CI->getType());
3405
3406	Value *MaskVec = getX86MaskVec(Builder, Mask: CI->getArgOperand(i: `2`),
3407	NumElts: ResultTy->getNumElements());
3408
3409	bool IsCompress = Name [`12`] == `'c'`;
3410	Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3411	: Intrinsic::x86_avx512_mask_expand;
3412	Rep = Builder.CreateIntrinsic(
3413	ID: IID, OverloadTypes: ResultTy, Args: {CI->getOperand(i_nocapture: `0`), CI->getOperand(i_nocapture: `1`), MaskVec});
3414	} else if (Name.starts_with(Prefix: "xop.vpcom")) {
3415	bool IsSigned;
3416	if (Name.ends_with(Suffix: "ub") \|\| Name.ends_with(Suffix: "uw") \|\| Name.ends_with(Suffix: "ud") \|\|
3417	Name.ends_with(Suffix: "uq"))
3418	IsSigned = false;
3419	else if (Name.ends_with(Suffix: "b") \|\| Name.ends_with(Suffix: "w") \|\|
3420	Name.ends_with(Suffix: "d") \|\| Name.ends_with(Suffix: "q"))
3421	IsSigned = true;
3422	else
3423	reportFatalUsageErrorWithCI(reason: "Intrinsic has unknown suffix", CI);
3424
3425	unsigned Imm;
3426	if (CI->arg_size() == `3`) {
3427	Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3428	} else {
3429	Name = Name.substr(Start: `9`); // strip off "xop.vpcom"
3430	if (Name.starts_with(Prefix: "lt"))
3431	Imm = `0`;
3432	else if (Name.starts_with(Prefix: "le"))
3433	Imm = `1`;
3434	else if (Name.starts_with(Prefix: "gt"))
3435	Imm = `2`;
3436	else if (Name.starts_with(Prefix: "ge"))
3437	Imm = `3`;
3438	else if (Name.starts_with(Prefix: "eq"))
3439	Imm = `4`;
3440	else if (Name.starts_with(Prefix: "ne"))
3441	Imm = `5`;
3442	else if (Name.starts_with(Prefix: "false"))
3443	Imm = `6`;
3444	else if (Name.starts_with(Prefix: "true"))
3445	Imm = `7`;
3446	else
3447	llvm_unreachable("Unknown condition");
3448	}
3449
3450	Rep = upgradeX86vpcom(Builder, CI&: *CI, Imm, IsSigned);
3451	} else if (Name.starts_with(Prefix: "xop.vpcmov")) {
3452	Value *Sel = CI->getArgOperand(i: `2`);
3453	Value *NotSel = Builder.CreateNot(V: Sel);
3454	Value *Sel0 = Builder.CreateAnd(LHS: CI->getArgOperand(i: `0`), RHS: Sel);
3455	Value *Sel1 = Builder.CreateAnd(LHS: CI->getArgOperand(i: `1`), RHS: NotSel);
3456	Rep = Builder.CreateOr(LHS: Sel0, RHS: Sel1);
3457	} else if (Name.starts_with(Prefix: "xop.vprot") \|\| Name.starts_with(Prefix: "avx512.prol") \|\|
3458	Name.starts_with(Prefix: "avx512.mask.prol")) {
3459	Rep = upgradeX86Rotate(Builder, CI&: CI, IsRotateRight: false*);
3460	} else if (Name.starts_with(Prefix: "avx512.pror") \|\|
3461	Name.starts_with(Prefix: "avx512.mask.pror")) {
3462	Rep = upgradeX86Rotate(Builder, CI&: CI, IsRotateRight: true*);
3463	} else if (Name.starts_with(Prefix: "avx512.vpshld.") \|\|
3464	Name.starts_with(Prefix: "avx512.mask.vpshld") \|\|
3465	Name.starts_with(Prefix: "avx512.maskz.vpshld")) {
3466	bool ZeroMask = Name [`11`] == `'z'`;
3467	Rep = upgradeX86ConcatShift(Builder, CI&: CI, IsShiftRight: false*, ZeroMask);
3468	} else if (Name.starts_with(Prefix: "avx512.vpshrd.") \|\|
3469	Name.starts_with(Prefix: "avx512.mask.vpshrd") \|\|
3470	Name.starts_with(Prefix: "avx512.maskz.vpshrd")) {
3471	bool ZeroMask = Name [`11`] == `'z'`;
3472	Rep = upgradeX86ConcatShift(Builder, CI&: CI, IsShiftRight: true*, ZeroMask);
3473	} else if (Name == "sse42.crc32.64.8") {
3474	Value *Trunc0 =
3475	Builder.CreateTrunc(V: CI->getArgOperand(i: `0`), DestTy: Type::getInt32Ty(C));
3476	Rep = Builder.CreateIntrinsic(ID: Intrinsic::x86_sse42_crc32_32_8,
3477	Args: {Trunc0, CI->getArgOperand(i: `1`)});
3478	Rep = Builder.CreateZExt(V: Rep, DestTy: CI->getType(), Name: "");
3479	} else if (Name.starts_with(Prefix: "avx.vbroadcast.s") \|\|
3480	Name.starts_with(Prefix: "avx512.vbroadcast.s")) {
3481	// Replace broadcasts with a series of insertelements.
3482	auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3483	Type *EltTy = VecTy->getElementType();
3484	unsigned EltNum = VecTy->getNumElements();
3485	Value *Load = Builder.CreateLoad(Ty: EltTy, Ptr: CI->getArgOperand(i: `0`));
3486	Type *I32Ty = Type::getInt32Ty(C);
3487	Rep = PoisonValue::get(T: VecTy);
3488	for (unsigned I = `0`; I < EltNum; ++I)
3489	Rep = Builder.CreateInsertElement(Vec: Rep, NewElt: Load, Idx: ConstantInt::get(Ty: I32Ty, V: I));
3490	} else if (Name.starts_with(Prefix: "sse41.pmovsx") \|\|
3491	Name.starts_with(Prefix: "sse41.pmovzx") \|\|
3492	Name.starts_with(Prefix: "avx2.pmovsx") \|\|
3493	Name.starts_with(Prefix: "avx2.pmovzx") \|\|
3494	Name.starts_with(Prefix: "avx512.mask.pmovsx") \|\|
3495	Name.starts_with(Prefix: "avx512.mask.pmovzx")) {
3496	auto *DstTy = cast<FixedVectorType>(Val: CI->getType());
3497	unsigned NumDstElts = DstTy->getNumElements();
3498
3499	// Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3500	SmallVector<int, `8`> ShuffleMask(NumDstElts);
3501	for (unsigned i = `0`; i != NumDstElts; ++i)
3502	ShuffleMask [i] = i;
3503
3504	Value *SV = Builder.CreateShuffleVector(V: CI->getArgOperand(i: `0`), Mask: ShuffleMask);
3505
3506	bool DoSext = Name.contains(Other: "pmovsx");
3507	Rep =
3508	DoSext ? Builder.CreateSExt(V: SV, DestTy: DstTy) : Builder.CreateZExt(V: SV, DestTy: DstTy);
3509	// If there are 3 arguments, it's a masked intrinsic so we need a select.
3510	if (CI->arg_size() == `3`)
3511	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep,
3512	Op1: CI->getArgOperand(i: `1`));
3513	} else if (Name == "avx512.mask.pmov.qd.256" \|\|
3514	Name == "avx512.mask.pmov.qd.512" \|\|
3515	Name == "avx512.mask.pmov.wb.256" \|\|
3516	Name == "avx512.mask.pmov.wb.512") {
3517	Type *Ty = CI->getArgOperand(i: `1`)->getType();
3518	Rep = Builder.CreateTrunc(V: CI->getArgOperand(i: `0`), DestTy: Ty);
3519	Rep =
3520	emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep, Op1: CI->getArgOperand(i: `1`));
3521	} else if (Name.starts_with(Prefix: "avx.vbroadcastf128") \|\|
3522	Name == "avx2.vbroadcasti128") {
3523	// Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3524	Type *EltTy = cast<VectorType>(Val: CI->getType())->getElementType();
3525	unsigned NumSrcElts = `128` / EltTy->getPrimitiveSizeInBits();
3526	auto *VT = FixedVectorType::get(ElementType: EltTy, NumElts: NumSrcElts);
3527	Value *Load = Builder.CreateAlignedLoad(Ty: VT, Ptr: CI->getArgOperand(i: `0`), Align: Align (`1`));
3528	if (NumSrcElts == `2`)
3529	Rep = Builder.CreateShuffleVector(V: Load, Mask: ArrayRef<int>{`0`, `1`, `0`, `1`});
3530	else
3531	Rep = Builder.CreateShuffleVector(V: Load,
3532	Mask: ArrayRef<int>{`0`, `1`, `2`, `3`, `0`, `1`, `2`, `3`});
3533	} else if (Name.starts_with(Prefix: "avx512.mask.shuf.i") \|\|
3534	Name.starts_with(Prefix: "avx512.mask.shuf.f")) {
3535	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3536	Type *VT = CI->getType();
3537	unsigned NumLanes = VT->getPrimitiveSizeInBits() / `128`;
3538	unsigned NumElementsInLane = `128` / VT->getScalarSizeInBits();
3539	unsigned ControlBitsMask = NumLanes - `1`;
3540	unsigned NumControlBits = NumLanes / `2`;
3541	SmallVector<int, `8`> ShuffleMask(`0`);
3542
3543	for (unsigned l = `0`; l != NumLanes; ++l) {
3544	unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3545	// We actually need the other source.
3546	if (l >= NumLanes / `2`)
3547	LaneMask += NumLanes;
3548	for (unsigned i = `0`; i != NumElementsInLane; ++i)
3549	ShuffleMask.push_back(Elt: LaneMask * NumElementsInLane + i);
3550	}
3551	Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: `0`),
3552	V2: CI->getArgOperand(i: `1`), Mask: ShuffleMask);
3553	Rep =
3554	emitX86Select(Builder, Mask: CI->getArgOperand(i: `4`), Op0: Rep, Op1: CI->getArgOperand(i: `3`));
3555	} else if (Name.starts_with(Prefix: "avx512.mask.broadcastf") \|\|
3556	Name.starts_with(Prefix: "avx512.mask.broadcasti")) {
3557	unsigned NumSrcElts = cast<FixedVectorType>(Val: CI->getArgOperand(i: `0`)->getType())
3558	->getNumElements();
3559	unsigned NumDstElts =
3560	cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3561
3562	SmallVector<int, `8`> ShuffleMask(NumDstElts);
3563	for (unsigned i = `0`; i != NumDstElts; ++i)
3564	ShuffleMask [i] = i % NumSrcElts;
3565
3566	Rep = Builder.CreateShuffleVector(V1: CI->getArgOperand(i: `0`),
3567	V2: CI->getArgOperand(i: `0`), Mask: ShuffleMask);
3568	Rep =
3569	emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep, Op1: CI->getArgOperand(i: `1`));
3570	} else if (Name.starts_with(Prefix: "avx2.pbroadcast") \|\|
3571	Name.starts_with(Prefix: "avx2.vbroadcast") \|\|
3572	Name.starts_with(Prefix: "avx512.pbroadcast") \|\|
3573	Name.starts_with(Prefix: "avx512.mask.broadcast.s")) {
3574	// Replace vp?broadcasts with a vector shuffle.
3575	Value *Op = CI->getArgOperand(i: `0`);
3576	ElementCount EC = cast<VectorType>(Val: CI->getType())->getElementCount();
3577	Type *MaskTy = VectorType::get(ElementType: Type::getInt32Ty(C), EC);
3578	SmallVector<int, `8`> M;
3579	ShuffleVectorInst::getShuffleMask(Mask: Constant::getNullValue(Ty: MaskTy), Result&: M);
3580	Rep = Builder.CreateShuffleVector(V: Op, Mask: M);
3581
3582	if (CI->arg_size() == `3`)
3583	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep,
3584	Op1: CI->getArgOperand(i: `1`));
3585	} else if (Name.starts_with(Prefix: "sse2.padds.") \|\|
3586	Name.starts_with(Prefix: "avx2.padds.") \|\|
3587	Name.starts_with(Prefix: "avx512.padds.") \|\|
3588	Name.starts_with(Prefix: "avx512.mask.padds.")) {
3589	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::sadd_sat);
3590	} else if (Name.starts_with(Prefix: "sse2.psubs.") \|\|
3591	Name.starts_with(Prefix: "avx2.psubs.") \|\|
3592	Name.starts_with(Prefix: "avx512.psubs.") \|\|
3593	Name.starts_with(Prefix: "avx512.mask.psubs.")) {
3594	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::ssub_sat);
3595	} else if (Name.starts_with(Prefix: "sse2.paddus.") \|\|
3596	Name.starts_with(Prefix: "avx2.paddus.") \|\|
3597	Name.starts_with(Prefix: "avx512.mask.paddus.")) {
3598	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::uadd_sat);
3599	} else if (Name.starts_with(Prefix: "sse2.psubus.") \|\|
3600	Name.starts_with(Prefix: "avx2.psubus.") \|\|
3601	Name.starts_with(Prefix: "avx512.mask.psubus.")) {
3602	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::usub_sat);
3603	} else if (Name.starts_with(Prefix: "avx512.mask.palignr.")) {
3604	Rep = upgradeX86ALIGNIntrinsics(Builder, Op0: CI->getArgOperand(i: `0`),
3605	Op1: CI->getArgOperand(i: `1`), Shift: CI->getArgOperand(i: `2`),
3606	Passthru: CI->getArgOperand(i: `3`), Mask: CI->getArgOperand(i: `4`),
3607	IsVALIGN: false);
3608	} else if (Name.starts_with(Prefix: "avx512.mask.valign.")) {
3609	Rep = upgradeX86ALIGNIntrinsics(
3610	Builder, Op0: CI->getArgOperand(i: `0`), Op1: CI->getArgOperand(i: `1`),
3611	Shift: CI->getArgOperand(i: `2`), Passthru: CI->getArgOperand(i: `3`), Mask: CI->getArgOperand(i: `4`), IsVALIGN: true);
3612	} else if (Name == "sse2.psll.dq" \|\| Name == "avx2.psll.dq") {
3613	// 128/256-bit shift left specified in bits.
3614	unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3615	Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: `0`),
3616	Shift: Shift / `8`); // Shift is in bits.
3617	} else if (Name == "sse2.psrl.dq" \|\| Name == "avx2.psrl.dq") {
3618	// 128/256-bit shift right specified in bits.
3619	unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3620	Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: `0`),
3621	Shift: Shift / `8`); // Shift is in bits.
3622	} else if (Name == "sse2.psll.dq.bs" \|\| Name == "avx2.psll.dq.bs" \|\|
3623	Name == "avx512.psll.dq.512") {
3624	// 128/256/512-bit shift left specified in bytes.
3625	unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3626	Rep = upgradeX86PSLLDQIntrinsics(Builder, Op: CI->getArgOperand(i: `0`), Shift);
3627	} else if (Name == "sse2.psrl.dq.bs" \|\| Name == "avx2.psrl.dq.bs" \|\|
3628	Name == "avx512.psrl.dq.512") {
3629	// 128/256/512-bit shift right specified in bytes.
3630	unsigned Shift = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3631	Rep = upgradeX86PSRLDQIntrinsics(Builder, Op: CI->getArgOperand(i: `0`), Shift);
3632	} else if (Name == "sse41.pblendw" \|\| Name.starts_with(Prefix: "sse41.blendp") \|\|
3633	Name.starts_with(Prefix: "avx.blend.p") \|\| Name == "avx2.pblendw" \|\|
3634	Name.starts_with(Prefix: "avx2.pblendd.")) {
3635	Value *Op0 = CI->getArgOperand(i: `0`);
3636	Value *Op1 = CI->getArgOperand(i: `1`);
3637	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3638	auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3639	unsigned NumElts = VecTy->getNumElements();
3640
3641	SmallVector<int, `16`> Idxs(NumElts);
3642	for (unsigned i = `0`; i != NumElts; ++i)
3643	Idxs [i] = ((Imm >> (i % `8`)) & `1`) ? i + NumElts : i;
3644
3645	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
3646	} else if (Name.starts_with(Prefix: "avx.vinsertf128.") \|\|
3647	Name == "avx2.vinserti128" \|\|
3648	Name.starts_with(Prefix: "avx512.mask.insert")) {
3649	Value *Op0 = CI->getArgOperand(i: `0`);
3650	Value *Op1 = CI->getArgOperand(i: `1`);
3651	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3652	unsigned DstNumElts =
3653	cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3654	unsigned SrcNumElts =
3655	cast<FixedVectorType>(Val: Op1->getType())->getNumElements();
3656	unsigned Scale = DstNumElts / SrcNumElts;
3657
3658	// Mask off the high bits of the immediate value; hardware ignores those.
3659	Imm = Imm % Scale;
3660
3661	// Extend the second operand into a vector the size of the destination.
3662	SmallVector<int, `8`> Idxs(DstNumElts);
3663	for (unsigned i = `0`; i != SrcNumElts; ++i)
3664	Idxs [i] = i;
3665	for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3666	Idxs [i] = SrcNumElts;
3667	Rep = Builder.CreateShuffleVector(V: Op1, Mask: Idxs);
3668
3669	// Insert the second operand into the first operand.
3670
3671	// Note that there is no guarantee that instruction lowering will actually
3672	// produce a vinsertf128 instruction for the created shuffles. In
3673	// particular, the 0 immediate case involves no lane changes, so it can
3674	// be handled as a blend.
3675
3676	// Example of shuffle mask for 32-bit elements:
3677	// Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3678	// Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3679
3680	// First fill with identify mask.
3681	for (unsigned i = `0`; i != DstNumElts; ++i)
3682	Idxs [i] = i;
3683	// Then replace the elements where we need to insert.
3684	for (unsigned i = `0`; i != SrcNumElts; ++i)
3685	Idxs [i + Imm * SrcNumElts] = i + DstNumElts;
3686	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Rep, Mask: Idxs);
3687
3688	// If the intrinsic has a mask operand, handle that.
3689	if (CI->arg_size() == `5`)
3690	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `4`), Op0: Rep,
3691	Op1: CI->getArgOperand(i: `3`));
3692	} else if (Name.starts_with(Prefix: "avx.vextractf128.") \|\|
3693	Name == "avx2.vextracti128" \|\|
3694	Name.starts_with(Prefix: "avx512.mask.vextract")) {
3695	Value *Op0 = CI->getArgOperand(i: `0`);
3696	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3697	unsigned DstNumElts =
3698	cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3699	unsigned SrcNumElts =
3700	cast<FixedVectorType>(Val: Op0->getType())->getNumElements();
3701	unsigned Scale = SrcNumElts / DstNumElts;
3702
3703	// Mask off the high bits of the immediate value; hardware ignores those.
3704	Imm = Imm % Scale;
3705
3706	// Get indexes for the subvector of the input vector.
3707	SmallVector<int, `8`> Idxs(DstNumElts);
3708	for (unsigned i = `0`; i != DstNumElts; ++i) {
3709	Idxs [i] = i + (Imm * DstNumElts);
3710	}
3711	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3712
3713	// If the intrinsic has a mask operand, handle that.
3714	if (CI->arg_size() == `4`)
3715	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep,
3716	Op1: CI->getArgOperand(i: `2`));
3717	} else if (Name.starts_with(Prefix: "avx512.mask.perm.df.") \|\|
3718	Name.starts_with(Prefix: "avx512.mask.perm.di.")) {
3719	Value *Op0 = CI->getArgOperand(i: `0`);
3720	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3721	auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3722	unsigned NumElts = VecTy->getNumElements();
3723
3724	SmallVector<int, `8`> Idxs(NumElts);
3725	for (unsigned i = `0`; i != NumElts; ++i)
3726	Idxs [i] = (i & ~`0x3`) + ((Imm >> (`2` * (i & `0x3`))) & `3`);
3727
3728	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3729
3730	if (CI->arg_size() == `4`)
3731	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep,
3732	Op1: CI->getArgOperand(i: `2`));
3733	} else if (Name.starts_with(Prefix: "avx.vperm2f128.") \|\| Name == "avx2.vperm2i128") {
3734	// The immediate permute control byte looks like this:
3735	// [1:0] - select 128 bits from sources for low half of destination
3736	// [2] - ignore
3737	// [3] - zero low half of destination
3738	// [5:4] - select 128 bits from sources for high half of destination
3739	// [6] - ignore
3740	// [7] - zero high half of destination
3741
3742	uint8_t Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3743
3744	unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3745	unsigned HalfSize = NumElts / `2`;
3746	SmallVector<int, `8`> ShuffleMask(NumElts);
3747
3748	// Determine which operand(s) are actually in use for this instruction.
3749	Value *V0 = (Imm & `0x02`) ? CI->getArgOperand(i: `1`) : CI->getArgOperand(i: `0`);
3750	Value *V1 = (Imm & `0x20`) ? CI->getArgOperand(i: `1`) : CI->getArgOperand(i: `0`);
3751
3752	// If needed, replace operands based on zero mask.
3753	V0 = (Imm & `0x08`) ? ConstantAggregateZero::get(Ty: CI->getType()) : V0;
3754	V1 = (Imm & `0x80`) ? ConstantAggregateZero::get(Ty: CI->getType()) : V1;
3755
3756	// Permute low half of result.
3757	unsigned StartIndex = (Imm & `0x01`) ? HalfSize : `0`;
3758	for (unsigned i = `0`; i < HalfSize; ++i)
3759	ShuffleMask [i] = StartIndex + i;
3760
3761	// Permute high half of result.
3762	StartIndex = (Imm & `0x10`) ? HalfSize : `0`;
3763	for (unsigned i = `0`; i < HalfSize; ++i)
3764	ShuffleMask [i + HalfSize] = NumElts + StartIndex + i;
3765
3766	Rep = Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
3767
3768	} else if (Name.starts_with(Prefix: "avx.vpermil.") \|\| Name == "sse2.pshuf.d" \|\|
3769	Name.starts_with(Prefix: "avx512.mask.vpermil.p") \|\|
3770	Name.starts_with(Prefix: "avx512.mask.pshuf.d.")) {
3771	Value *Op0 = CI->getArgOperand(i: `0`);
3772	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3773	auto *VecTy = cast<FixedVectorType>(Val: CI->getType());
3774	unsigned NumElts = VecTy->getNumElements();
3775	// Calculate the size of each index in the immediate.
3776	unsigned IdxSize = `64` / VecTy->getScalarSizeInBits();
3777	unsigned IdxMask = ((`1` << IdxSize) - `1`);
3778
3779	SmallVector<int, `8`> Idxs(NumElts);
3780	// Lookup the bits for this element, wrapping around the immediate every
3781	// 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
3782	// to offset by the first index of each group.
3783	for (unsigned i = `0`; i != NumElts; ++i)
3784	Idxs [i] = ((Imm >> ((i * IdxSize) % `8`)) & IdxMask) \| (i & ~IdxMask);
3785
3786	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3787
3788	if (CI->arg_size() == `4`)
3789	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep,
3790	Op1: CI->getArgOperand(i: `2`));
3791	} else if (Name == "sse2.pshufl.w" \|\|
3792	Name.starts_with(Prefix: "avx512.mask.pshufl.w.")) {
3793	Value *Op0 = CI->getArgOperand(i: `0`);
3794	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3795	unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3796
3797	if (Name == "sse2.pshufl.w" && NumElts % `8` != `0`)
3798	reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
3799
3800	SmallVector<int, `16`> Idxs(NumElts);
3801	for (unsigned l = `0`; l != NumElts; l += `8`) {
3802	for (unsigned i = `0`; i != `4`; ++i)
3803	Idxs [i + l] = ((Imm >> (`2` * i)) & `0x3`) + l;
3804	for (unsigned i = `4`; i != `8`; ++i)
3805	Idxs [i + l] = i + l;
3806	}
3807
3808	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3809
3810	if (CI->arg_size() == `4`)
3811	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep,
3812	Op1: CI->getArgOperand(i: `2`));
3813	} else if (Name == "sse2.pshufh.w" \|\|
3814	Name.starts_with(Prefix: "avx512.mask.pshufh.w.")) {
3815	Value *Op0 = CI->getArgOperand(i: `0`);
3816	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
3817	unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3818
3819	if (Name == "sse2.pshufh.w" && NumElts % `8` != `0`)
3820	reportFatalUsageErrorWithCI(reason: "Intrinsic has invalid signature", CI);
3821
3822	SmallVector<int, `16`> Idxs(NumElts);
3823	for (unsigned l = `0`; l != NumElts; l += `8`) {
3824	for (unsigned i = `0`; i != `4`; ++i)
3825	Idxs [i + l] = i + l;
3826	for (unsigned i = `0`; i != `4`; ++i)
3827	Idxs [i + l + `4`] = ((Imm >> (`2` * i)) & `0x3`) + `4` + l;
3828	}
3829
3830	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3831
3832	if (CI->arg_size() == `4`)
3833	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep,
3834	Op1: CI->getArgOperand(i: `2`));
3835	} else if (Name.starts_with(Prefix: "avx512.mask.shuf.p")) {
3836	Value *Op0 = CI->getArgOperand(i: `0`);
3837	Value *Op1 = CI->getArgOperand(i: `1`);
3838	unsigned Imm = cast<ConstantInt>(Val: CI->getArgOperand(i: `2`))->getZExtValue();
3839	unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3840
3841	unsigned NumLaneElts = `128` / CI->getType()->getScalarSizeInBits();
3842	unsigned HalfLaneElts = NumLaneElts / `2`;
3843
3844	SmallVector<int, `16`> Idxs(NumElts);
3845	for (unsigned i = `0`; i != NumElts; ++i) {
3846	// Base index is the starting element of the lane.
3847	Idxs [i] = i - (i % NumLaneElts);
3848	// If we are half way through the lane switch to the other source.
3849	if ((i % NumLaneElts) >= HalfLaneElts)
3850	Idxs [i] += NumElts;
3851	// Now select the specific element. By adding HalfLaneElts bits from
3852	// the immediate. Wrapping around the immediate every 8-bits.
3853	Idxs [i] += (Imm >> ((i * HalfLaneElts) % `8`)) & ((`1` << HalfLaneElts) - `1`);
3854	}
3855
3856	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
3857
3858	Rep =
3859	emitX86Select(Builder, Mask: CI->getArgOperand(i: `4`), Op0: Rep, Op1: CI->getArgOperand(i: `3`));
3860	} else if (Name.starts_with(Prefix: "avx512.mask.movddup") \|\|
3861	Name.starts_with(Prefix: "avx512.mask.movshdup") \|\|
3862	Name.starts_with(Prefix: "avx512.mask.movsldup")) {
3863	Value *Op0 = CI->getArgOperand(i: `0`);
3864	unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3865	unsigned NumLaneElts = `128` / CI->getType()->getScalarSizeInBits();
3866
3867	unsigned Offset = `0`;
3868	if (Name.starts_with(Prefix: "avx512.mask.movshdup."))
3869	Offset = `1`;
3870
3871	SmallVector<int, `16`> Idxs(NumElts);
3872	for (unsigned l = `0`; l != NumElts; l += NumLaneElts)
3873	for (unsigned i = `0`; i != NumLaneElts; i += `2`) {
3874	Idxs [i + l + `0`] = i + l + Offset;
3875	Idxs [i + l + `1`] = i + l + Offset;
3876	}
3877
3878	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op0, Mask: Idxs);
3879
3880	Rep =
3881	emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep, Op1: CI->getArgOperand(i: `1`));
3882	} else if (Name.starts_with(Prefix: "avx512.mask.punpckl") \|\|
3883	Name.starts_with(Prefix: "avx512.mask.unpckl.")) {
3884	Value *Op0 = CI->getArgOperand(i: `0`);
3885	Value *Op1 = CI->getArgOperand(i: `1`);
3886	int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3887	int NumLaneElts = `128` / CI->getType()->getScalarSizeInBits();
3888
3889	SmallVector<int, `64`> Idxs(NumElts);
3890	for (int l = `0`; l != NumElts; l += NumLaneElts)
3891	for (int i = `0`; i != NumLaneElts; ++i)
3892	Idxs [i + l] = l + (i / `2`) + NumElts * (i % `2`);
3893
3894	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
3895
3896	Rep =
3897	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3898	} else if (Name.starts_with(Prefix: "avx512.mask.punpckh") \|\|
3899	Name.starts_with(Prefix: "avx512.mask.unpckh.")) {
3900	Value *Op0 = CI->getArgOperand(i: `0`);
3901	Value *Op1 = CI->getArgOperand(i: `1`);
3902	int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
3903	int NumLaneElts = `128` / CI->getType()->getScalarSizeInBits();
3904
3905	SmallVector<int, `64`> Idxs(NumElts);
3906	for (int l = `0`; l != NumElts; l += NumLaneElts)
3907	for (int i = `0`; i != NumLaneElts; ++i)
3908	Idxs [i + l] = (NumLaneElts / `2`) + l + (i / `2`) + NumElts * (i % `2`);
3909
3910	Rep = Builder.CreateShuffleVector(V1: Op0, V2: Op1, Mask: Idxs);
3911
3912	Rep =
3913	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3914	} else if (Name.starts_with(Prefix: "avx512.mask.and.") \|\|
3915	Name.starts_with(Prefix: "avx512.mask.pand.")) {
3916	VectorType *FTy = cast<VectorType>(Val: CI->getType());
3917	VectorType *ITy = VectorType::getInteger(VTy: FTy);
3918	Rep = Builder.CreateAnd(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: ITy),
3919	RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: `1`), DestTy: ITy));
3920	Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
3921	Rep =
3922	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3923	} else if (Name.starts_with(Prefix: "avx512.mask.andn.") \|\|
3924	Name.starts_with(Prefix: "avx512.mask.pandn.")) {
3925	VectorType *FTy = cast<VectorType>(Val: CI->getType());
3926	VectorType *ITy = VectorType::getInteger(VTy: FTy);
3927	Rep = Builder.CreateNot(V: Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: ITy));
3928	Rep = Builder.CreateAnd(LHS: Rep,
3929	RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: `1`), DestTy: ITy));
3930	Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
3931	Rep =
3932	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3933	} else if (Name.starts_with(Prefix: "avx512.mask.or.") \|\|
3934	Name.starts_with(Prefix: "avx512.mask.por.")) {
3935	VectorType *FTy = cast<VectorType>(Val: CI->getType());
3936	VectorType *ITy = VectorType::getInteger(VTy: FTy);
3937	Rep = Builder.CreateOr(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: ITy),
3938	RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: `1`), DestTy: ITy));
3939	Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
3940	Rep =
3941	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3942	} else if (Name.starts_with(Prefix: "avx512.mask.xor.") \|\|
3943	Name.starts_with(Prefix: "avx512.mask.pxor.")) {
3944	VectorType *FTy = cast<VectorType>(Val: CI->getType());
3945	VectorType *ITy = VectorType::getInteger(VTy: FTy);
3946	Rep = Builder.CreateXor(LHS: Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: ITy),
3947	RHS: Builder.CreateBitCast(V: CI->getArgOperand(i: `1`), DestTy: ITy));
3948	Rep = Builder.CreateBitCast(V: Rep, DestTy: FTy);
3949	Rep =
3950	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3951	} else if (Name.starts_with(Prefix: "avx512.mask.padd.")) {
3952	Rep = Builder.CreateAdd(LHS: CI->getArgOperand(i: `0`), RHS: CI->getArgOperand(i: `1`));
3953	Rep =
3954	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3955	} else if (Name.starts_with(Prefix: "avx512.mask.psub.")) {
3956	Rep = Builder.CreateSub(LHS: CI->getArgOperand(i: `0`), RHS: CI->getArgOperand(i: `1`));
3957	Rep =
3958	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3959	} else if (Name.starts_with(Prefix: "avx512.mask.pmull.")) {
3960	Rep = Builder.CreateMul(LHS: CI->getArgOperand(i: `0`), RHS: CI->getArgOperand(i: `1`));
3961	Rep =
3962	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3963	} else if (Name.starts_with(Prefix: "avx512.mask.add.p")) {
3964	if (Name.ends_with(Suffix: ".512")) {
3965	Intrinsic::ID IID;
3966	if (Name [`17`] == `'s'`)
3967	IID = Intrinsic::x86_avx512_add_ps_512;
3968	else
3969	IID = Intrinsic::x86_avx512_add_pd_512;
3970
3971	Rep = Builder.CreateIntrinsic(
3972	ID: IID,
3973	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), CI->getArgOperand(i: `4`)});
3974	} else {
3975	Rep = Builder.CreateFAdd(L: CI->getArgOperand(i: `0`), R: CI->getArgOperand(i: `1`));
3976	}
3977	Rep =
3978	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3979	} else if (Name.starts_with(Prefix: "avx512.mask.div.p")) {
3980	if (Name.ends_with(Suffix: ".512")) {
3981	Intrinsic::ID IID;
3982	if (Name [`17`] == `'s'`)
3983	IID = Intrinsic::x86_avx512_div_ps_512;
3984	else
3985	IID = Intrinsic::x86_avx512_div_pd_512;
3986
3987	Rep = Builder.CreateIntrinsic(
3988	ID: IID,
3989	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), CI->getArgOperand(i: `4`)});
3990	} else {
3991	Rep = Builder.CreateFDiv(L: CI->getArgOperand(i: `0`), R: CI->getArgOperand(i: `1`));
3992	}
3993	Rep =
3994	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
3995	} else if (Name.starts_with(Prefix: "avx512.mask.mul.p")) {
3996	if (Name.ends_with(Suffix: ".512")) {
3997	Intrinsic::ID IID;
3998	if (Name [`17`] == `'s'`)
3999	IID = Intrinsic::x86_avx512_mul_ps_512;
4000	else
4001	IID = Intrinsic::x86_avx512_mul_pd_512;
4002
4003	Rep = Builder.CreateIntrinsic(
4004	ID: IID,
4005	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), CI->getArgOperand(i: `4`)});
4006	} else {
4007	Rep = Builder.CreateFMul(L: CI->getArgOperand(i: `0`), R: CI->getArgOperand(i: `1`));
4008	}
4009	Rep =
4010	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
4011	} else if (Name.starts_with(Prefix: "avx512.mask.sub.p")) {
4012	if (Name.ends_with(Suffix: ".512")) {
4013	Intrinsic::ID IID;
4014	if (Name [`17`] == `'s'`)
4015	IID = Intrinsic::x86_avx512_sub_ps_512;
4016	else
4017	IID = Intrinsic::x86_avx512_sub_pd_512;
4018
4019	Rep = Builder.CreateIntrinsic(
4020	ID: IID,
4021	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), CI->getArgOperand(i: `4`)});
4022	} else {
4023	Rep = Builder.CreateFSub(L: CI->getArgOperand(i: `0`), R: CI->getArgOperand(i: `1`));
4024	}
4025	Rep =
4026	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
4027	} else if ((Name.starts_with(Prefix: "avx512.mask.max.p") \|\|
4028	Name.starts_with(Prefix: "avx512.mask.min.p")) &&
4029	Name.drop_front(N: `18`) == ".512") {
4030	bool IsDouble = Name [`17`] == `'d'`;
4031	bool IsMin = Name [`13`] == `'i'`;
4032	static const Intrinsic::ID MinMaxTbl[`2`][`2`] = {
4033	{Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4034	{Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4035	Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4036
4037	Rep = Builder.CreateIntrinsic(
4038	ID: IID,
4039	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), CI->getArgOperand(i: `4`)});
4040	Rep =
4041	emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: CI->getArgOperand(i: `2`));
4042	} else if (Name.starts_with(Prefix: "avx512.mask.lzcnt.")) {
4043	Rep =
4044	Builder.CreateIntrinsic(ID: Intrinsic::ctlz, OverloadTypes: CI->getType(),
4045	Args: {CI->getArgOperand(i: `0`), Builder.getInt1(V: false)});
4046	Rep =
4047	emitX86Select(Builder, Mask: CI->getArgOperand(i: `2`), Op0: Rep, Op1: CI->getArgOperand(i: `1`));
4048	} else if (Name.starts_with(Prefix: "avx512.mask.psll")) {
4049	bool IsImmediate = Name [`16`] == `'i'` \|\| (Name.size() > `18` && Name [`18`] == `'i'`);
4050	bool IsVariable = Name [`16`] == `'v'`;
4051	char Size = Name [`16`] == `'.'` ? Name [`17`]
4052	: Name [`17`] == `'.'` ? Name [`18`]
4053	: Name [`18`] == `'.'` ? Name [`19`]
4054	: Name [`20`];
4055
4056	Intrinsic::ID IID;
4057	if (IsVariable && Name [`17`] != `'.'`) {
4058	if (Size == `'d'` && Name [`17`] == `'2'`) // avx512.mask.psllv2.di
4059	IID = Intrinsic::x86_avx2_psllv_q;
4060	else if (Size == `'d'` && Name [`17`] == `'4'`) // avx512.mask.psllv4.di
4061	IID = Intrinsic::x86_avx2_psllv_q_256;
4062	else if (Size == `'s'` && Name [`17`] == `'4'`) // avx512.mask.psllv4.si
4063	IID = Intrinsic::x86_avx2_psllv_d;
4064	else if (Size == `'s'` && Name [`17`] == `'8'`) // avx512.mask.psllv8.si
4065	IID = Intrinsic::x86_avx2_psllv_d_256;
4066	else if (Size == `'h'` && Name [`17`] == `'8'`) // avx512.mask.psllv8.hi
4067	IID = Intrinsic::x86_avx512_psllv_w_128;
4068	else if (Size == `'h'` && Name [`17`] == `'1'`) // avx512.mask.psllv16.hi
4069	IID = Intrinsic::x86_avx512_psllv_w_256;
4070	else if (Name [`17`] == `'3'` && Name [`18`] == `'2'`) // avx512.mask.psllv32hi
4071	IID = Intrinsic::x86_avx512_psllv_w_512;
4072	else
4073	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4074	} else if (Name.ends_with(Suffix: ".128")) {
4075	if (Size == `'d'`) // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4076	IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4077	: Intrinsic::x86_sse2_psll_d;
4078	else if (Size == `'q'`) // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4079	IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4080	: Intrinsic::x86_sse2_psll_q;
4081	else if (Size == `'w'`) // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4082	IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4083	: Intrinsic::x86_sse2_psll_w;
4084	else
4085	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4086	} else if (Name.ends_with(Suffix: ".256")) {
4087	if (Size == `'d'`) // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4088	IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4089	: Intrinsic::x86_avx2_psll_d;
4090	else if (Size == `'q'`) // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4091	IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4092	: Intrinsic::x86_avx2_psll_q;
4093	else if (Size == `'w'`) // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4094	IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4095	: Intrinsic::x86_avx2_psll_w;
4096	else
4097	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4098	} else {
4099	if (Size == `'d'`) // psll.di.512, pslli.d, psll.d, psllv.d.512
4100	IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4101	: IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4102	: Intrinsic::x86_avx512_psll_d_512;
4103	else if (Size == `'q'`) // psll.qi.512, pslli.q, psll.q, psllv.q.512
4104	IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4105	: IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4106	: Intrinsic::x86_avx512_psll_q_512;
4107	else if (Size == `'w'`) // psll.wi.512, pslli.w, psll.w
4108	IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4109	: Intrinsic::x86_avx512_psll_w_512;
4110	else
4111	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4112	}
4113
4114	Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4115	} else if (Name.starts_with(Prefix: "avx512.mask.psrl")) {
4116	bool IsImmediate = Name [`16`] == `'i'` \|\| (Name.size() > `18` && Name [`18`] == `'i'`);
4117	bool IsVariable = Name [`16`] == `'v'`;
4118	char Size = Name [`16`] == `'.'` ? Name [`17`]
4119	: Name [`17`] == `'.'` ? Name [`18`]
4120	: Name [`18`] == `'.'` ? Name [`19`]
4121	: Name [`20`];
4122
4123	Intrinsic::ID IID;
4124	if (IsVariable && Name [`17`] != `'.'`) {
4125	if (Size == `'d'` && Name [`17`] == `'2'`) // avx512.mask.psrlv2.di
4126	IID = Intrinsic::x86_avx2_psrlv_q;
4127	else if (Size == `'d'` && Name [`17`] == `'4'`) // avx512.mask.psrlv4.di
4128	IID = Intrinsic::x86_avx2_psrlv_q_256;
4129	else if (Size == `'s'` && Name [`17`] == `'4'`) // avx512.mask.psrlv4.si
4130	IID = Intrinsic::x86_avx2_psrlv_d;
4131	else if (Size == `'s'` && Name [`17`] == `'8'`) // avx512.mask.psrlv8.si
4132	IID = Intrinsic::x86_avx2_psrlv_d_256;
4133	else if (Size == `'h'` && Name [`17`] == `'8'`) // avx512.mask.psrlv8.hi
4134	IID = Intrinsic::x86_avx512_psrlv_w_128;
4135	else if (Size == `'h'` && Name [`17`] == `'1'`) // avx512.mask.psrlv16.hi
4136	IID = Intrinsic::x86_avx512_psrlv_w_256;
4137	else if (Name [`17`] == `'3'` && Name [`18`] == `'2'`) // avx512.mask.psrlv32hi
4138	IID = Intrinsic::x86_avx512_psrlv_w_512;
4139	else
4140	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4141	} else if (Name.ends_with(Suffix: ".128")) {
4142	if (Size == `'d'`) // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4143	IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4144	: Intrinsic::x86_sse2_psrl_d;
4145	else if (Size == `'q'`) // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4146	IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4147	: Intrinsic::x86_sse2_psrl_q;
4148	else if (Size == `'w'`) // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4149	IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4150	: Intrinsic::x86_sse2_psrl_w;
4151	else
4152	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4153	} else if (Name.ends_with(Suffix: ".256")) {
4154	if (Size == `'d'`) // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4155	IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4156	: Intrinsic::x86_avx2_psrl_d;
4157	else if (Size == `'q'`) // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4158	IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4159	: Intrinsic::x86_avx2_psrl_q;
4160	else if (Size == `'w'`) // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4161	IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4162	: Intrinsic::x86_avx2_psrl_w;
4163	else
4164	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4165	} else {
4166	if (Size == `'d'`) // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4167	IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4168	: IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4169	: Intrinsic::x86_avx512_psrl_d_512;
4170	else if (Size == `'q'`) // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4171	IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4172	: IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4173	: Intrinsic::x86_avx512_psrl_q_512;
4174	else if (Size == `'w'`) // psrl.wi.512, psrli.w, psrl.w)
4175	IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4176	: Intrinsic::x86_avx512_psrl_w_512;
4177	else
4178	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4179	}
4180
4181	Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4182	} else if (Name.starts_with(Prefix: "avx512.mask.psra")) {
4183	bool IsImmediate = Name [`16`] == `'i'` \|\| (Name.size() > `18` && Name [`18`] == `'i'`);
4184	bool IsVariable = Name [`16`] == `'v'`;
4185	char Size = Name [`16`] == `'.'` ? Name [`17`]
4186	: Name [`17`] == `'.'` ? Name [`18`]
4187	: Name [`18`] == `'.'` ? Name [`19`]
4188	: Name [`20`];
4189
4190	Intrinsic::ID IID;
4191	if (IsVariable && Name [`17`] != `'.'`) {
4192	if (Size == `'s'` && Name [`17`] == `'4'`) // avx512.mask.psrav4.si
4193	IID = Intrinsic::x86_avx2_psrav_d;
4194	else if (Size == `'s'` && Name [`17`] == `'8'`) // avx512.mask.psrav8.si
4195	IID = Intrinsic::x86_avx2_psrav_d_256;
4196	else if (Size == `'h'` && Name [`17`] == `'8'`) // avx512.mask.psrav8.hi
4197	IID = Intrinsic::x86_avx512_psrav_w_128;
4198	else if (Size == `'h'` && Name [`17`] == `'1'`) // avx512.mask.psrav16.hi
4199	IID = Intrinsic::x86_avx512_psrav_w_256;
4200	else if (Name [`17`] == `'3'` && Name [`18`] == `'2'`) // avx512.mask.psrav32hi
4201	IID = Intrinsic::x86_avx512_psrav_w_512;
4202	else
4203	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4204	} else if (Name.ends_with(Suffix: ".128")) {
4205	if (Size == `'d'`) // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4206	IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4207	: Intrinsic::x86_sse2_psra_d;
4208	else if (Size == `'q'`) // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4209	IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4210	: IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4211	: Intrinsic::x86_avx512_psra_q_128;
4212	else if (Size == `'w'`) // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4213	IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4214	: Intrinsic::x86_sse2_psra_w;
4215	else
4216	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4217	} else if (Name.ends_with(Suffix: ".256")) {
4218	if (Size == `'d'`) // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4219	IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4220	: Intrinsic::x86_avx2_psra_d;
4221	else if (Size == `'q'`) // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4222	IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4223	: IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4224	: Intrinsic::x86_avx512_psra_q_256;
4225	else if (Size == `'w'`) // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4226	IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4227	: Intrinsic::x86_avx2_psra_w;
4228	else
4229	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4230	} else {
4231	if (Size == `'d'`) // psra.di.512, psrai.d, psra.d, psrav.d.512
4232	IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4233	: IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4234	: Intrinsic::x86_avx512_psra_d_512;
4235	else if (Size == `'q'`) // psra.qi.512, psrai.q, psra.q
4236	IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4237	: IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4238	: Intrinsic::x86_avx512_psra_q_512;
4239	else if (Size == `'w'`) // psra.wi.512, psrai.w, psra.w
4240	IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4241	: Intrinsic::x86_avx512_psra_w_512;
4242	else
4243	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected size", CI);
4244	}
4245
4246	Rep = upgradeX86MaskedShift(Builder, CI&: *CI, IID);
4247	} else if (Name.starts_with(Prefix: "avx512.mask.move.s")) {
4248	Rep = upgradeMaskedMove(Builder, CI&: *CI);
4249	} else if (Name.starts_with(Prefix: "avx512.cvtmask2")) {
4250	Rep = upgradeMaskToInt(Builder, CI&: *CI);
4251	} else if (Name.ends_with(Suffix: ".movntdqa")) {
4252	MDNode *Node = MDNode::get(
4253	Context&: C, MDs: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C), V: `1`)));
4254
4255	LoadInst *LI = Builder.CreateAlignedLoad(
4256	Ty: CI->getType(), Ptr: CI->getArgOperand(i: `0`),
4257	Align: Align (CI->getType()->getPrimitiveSizeInBits().getFixedValue() / `8`));
4258	LI->setMetadata(KindID: LLVMContext::MD_nontemporal, Node);
4259	Rep = LI;
4260	} else if (Name.starts_with(Prefix: "fma.vfmadd.") \|\|
4261	Name.starts_with(Prefix: "fma.vfmsub.") \|\|
4262	Name.starts_with(Prefix: "fma.vfnmadd.") \|\|
4263	Name.starts_with(Prefix: "fma.vfnmsub.")) {
4264	bool NegMul = Name [`6`] == `'n'`;
4265	bool NegAcc = NegMul ? Name [`8`] == `'s'` : Name [`7`] == `'s'`;
4266	bool IsScalar = NegMul ? Name [`12`] == `'s'` : Name [`11`] == `'s'`;
4267
4268	Value *Ops[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4269	CI->getArgOperand(i: `2`)};
4270
4271	if (IsScalar) {
4272	Ops[`0`] = Builder.CreateExtractElement(Vec: Ops[`0`], Idx: (uint64_t)`0`);
4273	Ops[`1`] = Builder.CreateExtractElement(Vec: Ops[`1`], Idx: (uint64_t)`0`);
4274	Ops[`2`] = Builder.CreateExtractElement(Vec: Ops[`2`], Idx: (uint64_t)`0`);
4275	}
4276
4277	if (NegMul && !IsScalar)
4278	Ops[`0`] = Builder.CreateFNeg(V: Ops[`0`]);
4279	if (NegMul && IsScalar)
4280	Ops[`1`] = Builder.CreateFNeg(V: Ops[`1`]);
4281	if (NegAcc)
4282	Ops[`2`] = Builder.CreateFNeg(V: Ops[`2`]);
4283
4284	Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[`0`]->getType(), Args: Ops);
4285
4286	if (IsScalar)
4287	Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: `0`), NewElt: Rep, Idx: (uint64_t)`0`);
4288	} else if (Name.starts_with(Prefix: "fma4.vfmadd.s")) {
4289	Value *Ops[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4290	CI->getArgOperand(i: `2`)};
4291
4292	Ops[`0`] = Builder.CreateExtractElement(Vec: Ops[`0`], Idx: (uint64_t)`0`);
4293	Ops[`1`] = Builder.CreateExtractElement(Vec: Ops[`1`], Idx: (uint64_t)`0`);
4294	Ops[`2`] = Builder.CreateExtractElement(Vec: Ops[`2`], Idx: (uint64_t)`0`);
4295
4296	Rep = Builder.CreateIntrinsic(ID: Intrinsic::fma, OverloadTypes: Ops[`0`]->getType(), Args: Ops);
4297
4298	Rep = Builder.CreateInsertElement(Vec: Constant::getNullValue(Ty: CI->getType()),
4299	NewElt: Rep, Idx: (uint64_t)`0`);
4300	} else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.s") \|\|
4301	Name.starts_with(Prefix: "avx512.maskz.vfmadd.s") \|\|
4302	Name.starts_with(Prefix: "avx512.mask3.vfmadd.s") \|\|
4303	Name.starts_with(Prefix: "avx512.mask3.vfmsub.s") \|\|
4304	Name.starts_with(Prefix: "avx512.mask3.vfnmsub.s")) {
4305	bool IsMask3 = Name [`11`] == `'3'`;
4306	bool IsMaskZ = Name [`11`] == `'z'`;
4307	// Drop the "avx512.mask." to make it easier.
4308	Name = Name.drop_front(N: IsMask3 \|\| IsMaskZ ? `13` : `12`);
4309	bool NegMul = Name [`2`] == `'n'`;
4310	bool NegAcc = NegMul ? Name [`4`] == `'s'` : Name [`3`] == `'s'`;
4311
4312	Value *A = CI->getArgOperand(i: `0`);
4313	Value *B = CI->getArgOperand(i: `1`);
4314	Value *C = CI->getArgOperand(i: `2`);
4315
4316	if (NegMul && (IsMask3 \|\| IsMaskZ))
4317	A = Builder.CreateFNeg(V: A);
4318	if (NegMul && !(IsMask3 \|\| IsMaskZ))
4319	B = Builder.CreateFNeg(V: B);
4320	if (NegAcc)
4321	C = Builder.CreateFNeg(V: C);
4322
4323	A = Builder.CreateExtractElement(Vec: A, Idx: (uint64_t)`0`);
4324	B = Builder.CreateExtractElement(Vec: B, Idx: (uint64_t)`0`);
4325	C = Builder.CreateExtractElement(Vec: C, Idx: (uint64_t)`0`);
4326
4327	if (!isa<ConstantInt>(Val: CI->getArgOperand(i: `4`)) \|\|
4328	cast<ConstantInt>(Val: CI->getArgOperand(i: `4`))->getZExtValue() != `4`) {
4329	Value *Ops[] = {A, B, C, CI->getArgOperand(i: `4`)};
4330
4331	Intrinsic::ID IID;
4332	if (Name.back() == `'d'`)
4333	IID = Intrinsic::x86_avx512_vfmadd_f64;
4334	else
4335	IID = Intrinsic::x86_avx512_vfmadd_f32;
4336	Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4337	} else {
4338	Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4339	}
4340
4341	Value *PassThru = IsMaskZ ? Constant::getNullValue(Ty: Rep->getType())
4342	: IsMask3 ? C
4343	: A;
4344
4345	// For Mask3 with NegAcc, we need to create a new extractelement that
4346	// avoids the negation above.
4347	if (NegAcc && IsMask3)
4348	PassThru =
4349	Builder.CreateExtractElement(Vec: CI->getArgOperand(i: `2`), Idx: (uint64_t)`0`);
4350
4351	Rep = emitX86ScalarSelect(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: PassThru);
4352	Rep = Builder.CreateInsertElement(Vec: CI->getArgOperand(i: IsMask3 ? `2` : `0`), NewElt: Rep,
4353	Idx: (uint64_t)`0`);
4354	} else if (Name.starts_with(Prefix: "avx512.mask.vfmadd.p") \|\|
4355	Name.starts_with(Prefix: "avx512.mask.vfnmadd.p") \|\|
4356	Name.starts_with(Prefix: "avx512.mask.vfnmsub.p") \|\|
4357	Name.starts_with(Prefix: "avx512.mask3.vfmadd.p") \|\|
4358	Name.starts_with(Prefix: "avx512.mask3.vfmsub.p") \|\|
4359	Name.starts_with(Prefix: "avx512.mask3.vfnmsub.p") \|\|
4360	Name.starts_with(Prefix: "avx512.maskz.vfmadd.p")) {
4361	bool IsMask3 = Name [`11`] == `'3'`;
4362	bool IsMaskZ = Name [`11`] == `'z'`;
4363	// Drop the "avx512.mask." to make it easier.
4364	Name = Name.drop_front(N: IsMask3 \|\| IsMaskZ ? `13` : `12`);
4365	bool NegMul = Name [`2`] == `'n'`;
4366	bool NegAcc = NegMul ? Name [`4`] == `'s'` : Name [`3`] == `'s'`;
4367
4368	Value *A = CI->getArgOperand(i: `0`);
4369	Value *B = CI->getArgOperand(i: `1`);
4370	Value *C = CI->getArgOperand(i: `2`);
4371
4372	if (NegMul && (IsMask3 \|\| IsMaskZ))
4373	A = Builder.CreateFNeg(V: A);
4374	if (NegMul && !(IsMask3 \|\| IsMaskZ))
4375	B = Builder.CreateFNeg(V: B);
4376	if (NegAcc)
4377	C = Builder.CreateFNeg(V: C);
4378
4379	if (CI->arg_size() == `5` &&
4380	(!isa<ConstantInt>(Val: CI->getArgOperand(i: `4`)) \|\|
4381	cast<ConstantInt>(Val: CI->getArgOperand(i: `4`))->getZExtValue() != `4`)) {
4382	Intrinsic::ID IID;
4383	// Check the character before ".512" in string.
4384	if (Name [Name.size() - `5`] == `'s'`)
4385	IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4386	else
4387	IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4388
4389	Rep = Builder.CreateIntrinsic(ID: IID, Args: {A, B, C, CI->getArgOperand(i: `4`)});
4390	} else {
4391	Rep = Builder.CreateFMA(Factor1: A, Factor2: B, Summand: C);
4392	}
4393
4394	Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4395	: IsMask3 ? CI->getArgOperand(i: `2`)
4396	: CI->getArgOperand(i: `0`);
4397
4398	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: PassThru);
4399	} else if (Name.starts_with(Prefix: "fma.vfmsubadd.p")) {
4400	unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4401	unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4402	Intrinsic::ID IID;
4403	if (VecWidth == `128` && EltWidth == `32`)
4404	IID = Intrinsic::x86_fma_vfmaddsub_ps;
4405	else if (VecWidth == `256` && EltWidth == `32`)
4406	IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4407	else if (VecWidth == `128` && EltWidth == `64`)
4408	IID = Intrinsic::x86_fma_vfmaddsub_pd;
4409	else if (VecWidth == `256` && EltWidth == `64`)
4410	IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4411	else
4412	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4413
4414	Value *Ops[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4415	CI->getArgOperand(i: `2`)};
4416	Ops[`2`] = Builder.CreateFNeg(V: Ops[`2`]);
4417	Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4418	} else if (Name.starts_with(Prefix: "avx512.mask.vfmaddsub.p") \|\|
4419	Name.starts_with(Prefix: "avx512.mask3.vfmaddsub.p") \|\|
4420	Name.starts_with(Prefix: "avx512.maskz.vfmaddsub.p") \|\|
4421	Name.starts_with(Prefix: "avx512.mask3.vfmsubadd.p")) {
4422	bool IsMask3 = Name [`11`] == `'3'`;
4423	bool IsMaskZ = Name [`11`] == `'z'`;
4424	// Drop the "avx512.mask." to make it easier.
4425	Name = Name.drop_front(N: IsMask3 \|\| IsMaskZ ? `13` : `12`);
4426	bool IsSubAdd = Name [`3`] == `'s'`;
4427	if (CI->arg_size() == `5`) {
4428	Intrinsic::ID IID;
4429	// Check the character before ".512" in string.
4430	if (Name [Name.size() - `5`] == `'s'`)
4431	IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4432	else
4433	IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4434
4435	Value *Ops[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4436	CI->getArgOperand(i: `2`), CI->getArgOperand(i: `4`)};
4437	if (IsSubAdd)
4438	Ops[`2`] = Builder.CreateFNeg(V: Ops[`2`]);
4439
4440	Rep = Builder.CreateIntrinsic(ID: IID, Args: Ops);
4441	} else {
4442	int NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
4443
4444	Value *Ops[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4445	CI->getArgOperand(i: `2`)};
4446
4447	Function *FMA = Intrinsic::getOrInsertDeclaration(
4448	M: CI->getModule(), id: Intrinsic::fma, OverloadTys: Ops[`0`]->getType());
4449	Value *Odd = Builder.CreateCall(Callee: FMA, Args: Ops);
4450	Ops[`2`] = Builder.CreateFNeg(V: Ops[`2`]);
4451	Value *Even = Builder.CreateCall(Callee: FMA, Args: Ops);
4452
4453	if (IsSubAdd)
4454	std::swap(a&: Even, b&: Odd);
4455
4456	SmallVector<int, `32`> Idxs(NumElts);
4457	for (int i = `0`; i != NumElts; ++i)
4458	Idxs [i] = i + (i % `2`) * NumElts;
4459
4460	Rep = Builder.CreateShuffleVector(V1: Even, V2: Odd, Mask: Idxs);
4461	}
4462
4463	Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(Ty: CI->getType())
4464	: IsMask3 ? CI->getArgOperand(i: `2`)
4465	: CI->getArgOperand(i: `0`);
4466
4467	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: PassThru);
4468	} else if (Name.starts_with(Prefix: "avx512.mask.pternlog.") \|\|
4469	Name.starts_with(Prefix: "avx512.maskz.pternlog.")) {
4470	bool ZeroMask = Name [`11`] == `'z'`;
4471	unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4472	unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4473	Intrinsic::ID IID;
4474	if (VecWidth == `128` && EltWidth == `32`)
4475	IID = Intrinsic::x86_avx512_pternlog_d_128;
4476	else if (VecWidth == `256` && EltWidth == `32`)
4477	IID = Intrinsic::x86_avx512_pternlog_d_256;
4478	else if (VecWidth == `512` && EltWidth == `32`)
4479	IID = Intrinsic::x86_avx512_pternlog_d_512;
4480	else if (VecWidth == `128` && EltWidth == `64`)
4481	IID = Intrinsic::x86_avx512_pternlog_q_128;
4482	else if (VecWidth == `256` && EltWidth == `64`)
4483	IID = Intrinsic::x86_avx512_pternlog_q_256;
4484	else if (VecWidth == `512` && EltWidth == `64`)
4485	IID = Intrinsic::x86_avx512_pternlog_q_512;
4486	else
4487	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4488
4489	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4490	CI->getArgOperand(i: `2`), CI->getArgOperand(i: `3`)};
4491	Rep = Builder.CreateIntrinsic(ID: IID, Args);
4492	Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4493	: CI->getArgOperand(i: `0`);
4494	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `4`), Op0: Rep, Op1: PassThru);
4495	} else if (Name.starts_with(Prefix: "avx512.mask.vpmadd52") \|\|
4496	Name.starts_with(Prefix: "avx512.maskz.vpmadd52")) {
4497	bool ZeroMask = Name [`11`] == `'z'`;
4498	bool High = Name [`20`] == `'h'` \|\| Name [`21`] == `'h'`;
4499	unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4500	Intrinsic::ID IID;
4501	if (VecWidth == `128` && !High)
4502	IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4503	else if (VecWidth == `256` && !High)
4504	IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4505	else if (VecWidth == `512` && !High)
4506	IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4507	else if (VecWidth == `128` && High)
4508	IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4509	else if (VecWidth == `256` && High)
4510	IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4511	else if (VecWidth == `512` && High)
4512	IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4513	else
4514	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4515
4516	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4517	CI->getArgOperand(i: `2`)};
4518	Rep = Builder.CreateIntrinsic(ID: IID, Args);
4519	Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4520	: CI->getArgOperand(i: `0`);
4521	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: PassThru);
4522	} else if (Name.starts_with(Prefix: "avx512.mask.vpermi2var.") \|\|
4523	Name.starts_with(Prefix: "avx512.mask.vpermt2var.") \|\|
4524	Name.starts_with(Prefix: "avx512.maskz.vpermt2var.")) {
4525	bool ZeroMask = Name [`11`] == `'z'`;
4526	bool IndexForm = Name [`17`] == `'i'`;
4527	Rep = upgradeX86VPERMT2Intrinsics(Builder, CI&: *CI, ZeroMask, IndexForm);
4528	} else if (Name.starts_with(Prefix: "avx512.mask.vpdpbusd.") \|\|
4529	Name.starts_with(Prefix: "avx512.maskz.vpdpbusd.") \|\|
4530	Name.starts_with(Prefix: "avx512.mask.vpdpbusds.") \|\|
4531	Name.starts_with(Prefix: "avx512.maskz.vpdpbusds.")) {
4532	bool ZeroMask = Name [`11`] == `'z'`;
4533	bool IsSaturating = Name [ZeroMask ? `21` : `20`] == `'s'`;
4534	unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4535	Intrinsic::ID IID;
4536	if (VecWidth == `128` && !IsSaturating)
4537	IID = Intrinsic::x86_avx512_vpdpbusd_128;
4538	else if (VecWidth == `256` && !IsSaturating)
4539	IID = Intrinsic::x86_avx512_vpdpbusd_256;
4540	else if (VecWidth == `512` && !IsSaturating)
4541	IID = Intrinsic::x86_avx512_vpdpbusd_512;
4542	else if (VecWidth == `128` && IsSaturating)
4543	IID = Intrinsic::x86_avx512_vpdpbusds_128;
4544	else if (VecWidth == `256` && IsSaturating)
4545	IID = Intrinsic::x86_avx512_vpdpbusds_256;
4546	else if (VecWidth == `512` && IsSaturating)
4547	IID = Intrinsic::x86_avx512_vpdpbusds_512;
4548	else
4549	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4550
4551	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4552	CI->getArgOperand(i: `2`)};
4553
4554	// Input arguments types were incorrectly set to vectors of i32 before but
4555	// they should be vectors of i8. Insert bit cast when encountering the old
4556	// types
4557	if (Args[`1`]->getType()->isVectorTy() &&
4558	cast<VectorType>(Val: Args[`1`]->getType())
4559	->getElementType()
4560	->isIntegerTy(BitWidth: `32`) &&
4561	Args[`2`]->getType()->isVectorTy() &&
4562	cast<VectorType>(Val: Args[`2`]->getType())
4563	->getElementType()
4564	->isIntegerTy(BitWidth: `32`)) {
4565	Type NewArgType = nullptr*;
4566	if (VecWidth == `128`)
4567	NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: `16`, Scalable: false);
4568	else if (VecWidth == `256`)
4569	NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: `32`, Scalable: false);
4570	else if (VecWidth == `512`)
4571	NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: `64`, Scalable: false);
4572	else
4573	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
4574	CI);
4575
4576	Args[`1`] = Builder.CreateBitCast(V: Args[`1`], DestTy: NewArgType);
4577	Args[`2`] = Builder.CreateBitCast(V: Args[`2`], DestTy: NewArgType);
4578	}
4579
4580	Rep = Builder.CreateIntrinsic(ID: IID, Args);
4581	Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4582	: CI->getArgOperand(i: `0`);
4583	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: PassThru);
4584	} else if (Name.starts_with(Prefix: "avx512.mask.vpdpwssd.") \|\|
4585	Name.starts_with(Prefix: "avx512.maskz.vpdpwssd.") \|\|
4586	Name.starts_with(Prefix: "avx512.mask.vpdpwssds.") \|\|
4587	Name.starts_with(Prefix: "avx512.maskz.vpdpwssds.")) {
4588	bool ZeroMask = Name [`11`] == `'z'`;
4589	bool IsSaturating = Name [ZeroMask ? `21` : `20`] == `'s'`;
4590	unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4591	Intrinsic::ID IID;
4592	if (VecWidth == `128` && !IsSaturating)
4593	IID = Intrinsic::x86_avx512_vpdpwssd_128;
4594	else if (VecWidth == `256` && !IsSaturating)
4595	IID = Intrinsic::x86_avx512_vpdpwssd_256;
4596	else if (VecWidth == `512` && !IsSaturating)
4597	IID = Intrinsic::x86_avx512_vpdpwssd_512;
4598	else if (VecWidth == `128` && IsSaturating)
4599	IID = Intrinsic::x86_avx512_vpdpwssds_128;
4600	else if (VecWidth == `256` && IsSaturating)
4601	IID = Intrinsic::x86_avx512_vpdpwssds_256;
4602	else if (VecWidth == `512` && IsSaturating)
4603	IID = Intrinsic::x86_avx512_vpdpwssds_512;
4604	else
4605	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4606
4607	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4608	CI->getArgOperand(i: `2`)};
4609
4610	// Input arguments types were incorrectly set to vectors of i32 before but
4611	// they should be vectors of i16. Insert bit cast when encountering the old
4612	// types
4613	if (Args[`1`]->getType()->isVectorTy() &&
4614	cast<VectorType>(Val: Args[`1`]->getType())
4615	->getElementType()
4616	->isIntegerTy(BitWidth: `32`) &&
4617	Args[`2`]->getType()->isVectorTy() &&
4618	cast<VectorType>(Val: Args[`2`]->getType())
4619	->getElementType()
4620	->isIntegerTy(BitWidth: `32`)) {
4621	Type NewArgType = nullptr*;
4622	if (VecWidth == `128`)
4623	NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: `8`, Scalable: false);
4624	else if (VecWidth == `256`)
4625	NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: `16`, Scalable: false);
4626	else if (VecWidth == `512`)
4627	NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: `32`, Scalable: false);
4628	else
4629	reportFatalUsageErrorWithCI(reason: "Intrinsic has unexpected vector bit width",
4630	CI);
4631
4632	Args[`1`] = Builder.CreateBitCast(V: Args[`1`], DestTy: NewArgType);
4633	Args[`2`] = Builder.CreateBitCast(V: Args[`2`], DestTy: NewArgType);
4634	}
4635
4636	Rep = Builder.CreateIntrinsic(ID: IID, Args);
4637	Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty: CI->getType())
4638	: CI->getArgOperand(i: `0`);
4639	Rep = emitX86Select(Builder, Mask: CI->getArgOperand(i: `3`), Op0: Rep, Op1: PassThru);
4640	} else if (Name == "addcarryx.u32" \|\| Name == "addcarryx.u64" \|\|
4641	Name == "addcarry.u32" \|\| Name == "addcarry.u64" \|\|
4642	Name == "subborrow.u32" \|\| Name == "subborrow.u64") {
4643	Intrinsic::ID IID;
4644	if (Name [`0`] == `'a'` && Name.back() == `'2'`)
4645	IID = Intrinsic::x86_addcarry_32;
4646	else if (Name [`0`] == `'a'` && Name.back() == `'4'`)
4647	IID = Intrinsic::x86_addcarry_64;
4648	else if (Name [`0`] == `'s'` && Name.back() == `'2'`)
4649	IID = Intrinsic::x86_subborrow_32;
4650	else if (Name [`0`] == `'s'` && Name.back() == `'4'`)
4651	IID = Intrinsic::x86_subborrow_64;
4652	else
4653	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4654
4655	// Make a call with 3 operands.
4656	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
4657	CI->getArgOperand(i: `2`)};
4658	Value *NewCall = Builder.CreateIntrinsic(ID: IID, Args);
4659
4660	// Extract the second result and store it.
4661	Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: `1`);
4662	Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: `3`), Align: Align (`1`));
4663	// Replace the original call result with the first result of the new call.
4664	Value *CF = Builder.CreateExtractValue(Agg: NewCall, Idxs: `0`);
4665
4666	CI->replaceAllUsesWith(V: CF);
4667	Rep = nullptr;
4668	} else if (Name.starts_with(Prefix: "avx512.mask.") &&
4669	upgradeAVX512MaskToSelect(Name, Builder, CI&: *CI, Rep)) {
4670	// Rep will be updated by the call in the condition.
4671	} else if (Name.starts_with(Prefix: "bmi.pdep.")) {
4672	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pdep);
4673	} else if (Name.starts_with(Prefix: "bmi.pext.")) {
4674	Rep = upgradeX86BinaryIntrinsics(Builder, CI&: *CI, IID: Intrinsic::pext);
4675	} else
4676	reportFatalUsageErrorWithCI(reason: "Unexpected intrinsic", CI);
4677
4678	return Rep;
4679	}
4680
4681	static Value upgradeAArch64IntrinsicCall(StringRef Name, CallBase CI,
4682	Function *F, IRBuilder<> &Builder) {
4683	if (Name.starts_with(Prefix: "neon.bfcvt")) {
4684	if (Name.starts_with(Prefix: "neon.bfcvtn2")) {
4685	SmallVector<int, `32`> LoMask(`4`);
4686	std::iota(first: LoMask.begin(), last: LoMask.end(), value: `0`);
4687	SmallVector<int, `32`> ConcatMask(`8`);
4688	std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: `0`);
4689	Value *Inactive = Builder.CreateShuffleVector(V: CI->getOperand(i_nocapture: `0`), Mask: LoMask);
4690	Value *Trunc =
4691	Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: `1`), DestTy: Inactive->getType());
4692	return Builder.CreateShuffleVector(V1: Inactive, V2: Trunc, Mask: ConcatMask);
4693	} else if (Name.starts_with(Prefix: "neon.bfcvtn")) {
4694	SmallVector<int, `32`> ConcatMask(`8`);
4695	std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: `0`);
4696	Type *V4BF16 =
4697	FixedVectorType::get(ElementType: Type::getBFloatTy(C&: F->getContext()), NumElts: `4`);
4698	Value *Trunc = Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: `0`), DestTy: V4BF16);
4699	dbgs() << "Trunc: " << *Trunc << "\n";
4700	return Builder.CreateShuffleVector(
4701	V1: Trunc, V2: ConstantAggregateZero::get(Ty: V4BF16), Mask: ConcatMask);
4702	} else {
4703	return Builder.CreateFPTrunc(V: CI->getOperand(i_nocapture: `0`),
4704	DestTy: Type::getBFloatTy(C&: F->getContext()));
4705	}
4706	} else if (Name.starts_with(Prefix: "sve.fcvt")) {
4707	Intrinsic::ID NewID =
4708	StringSwitch<Intrinsic::ID>(Name)
4709	.Case(S: "sve.fcvt.bf16f32", Value: Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4710	.Case(S: "sve.fcvtnt.bf16f32",
4711	Value: Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4712	.Default(Value: Intrinsic::not_intrinsic);
4713	if (NewID == Intrinsic::not_intrinsic)
4714	llvm_unreachable("Unhandled Intrinsic!");
4715
4716	SmallVector<Value *, `3`> Args(CI->args());
4717
4718	// The original intrinsics incorrectly used a predicate based on the
4719	// smallest element type rather than the largest.
4720	Type *BadPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: `8`);
4721	Type *GoodPredTy = ScalableVectorType::get(ElementType: Builder.getInt1Ty(), MinNumElts: `4`);
4722
4723	if (Args [`1`]->getType() != BadPredTy)
4724	llvm_unreachable("Unexpected predicate type!");
4725
4726	Args [`1`] = Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
4727	OverloadTypes: BadPredTy, Args: Args [`1`]);
4728	Args [`1`] = Builder.CreateIntrinsic(
4729	ID: Intrinsic::aarch64_sve_convert_from_svbool, OverloadTypes: GoodPredTy, Args: Args [`1`]);
4730
4731	return Builder.CreateIntrinsic(ID: NewID, Args, /FMFSource=/nullptr,
4732	Name: CI->getName());
4733	}
4734
4735	if (Name == "neon.vcvtfp2hf")
4736	return Builder.CreateBitCast(
4737	V: Builder.CreateFPTrunc(
4738	V: CI->getOperand(i_nocapture: `0`),
4739	DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: `4`)),
4740	DestTy: FixedVectorType::get(ElementType: Type::getInt16Ty(C&: F->getContext()), NumElts: `4`));
4741	if (Name == "neon.vcvthf2fp")
4742	return Builder.CreateFPExt(
4743	V: Builder.CreateBitCast(
4744	V: CI->getOperand(i_nocapture: `0`),
4745	DestTy: FixedVectorType::get(ElementType: Type::getHalfTy(C&: F->getContext()), NumElts: `4`)),
4746	DestTy: FixedVectorType::get(ElementType: Type::getFloatTy(C&: F->getContext()), NumElts: `4`));
4747
4748	llvm_unreachable("Unhandled Intrinsic!");
4749	}
4750
4751	static Value upgradeARMIntrinsicCall(StringRef Name, CallBase CI, Function *F,
4752	IRBuilder<> &Builder) {
4753	if (Name == "mve.vctp64.old") {
4754	// Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
4755	// correct type.
4756	Value *VCTP = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vctp64, OverloadTypes: {},
4757	Args: CI->getArgOperand(i: `0`),
4758	/FMFSource=/nullptr, Name: CI->getName());
4759	Value *C1 = Builder.CreateIntrinsic(
4760	ID: Intrinsic::arm_mve_pred_v2i,
4761	OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: `2`, Scalable: false)}, Args: VCTP);
4762	return Builder.CreateIntrinsic(
4763	ID: Intrinsic::arm_mve_pred_i2v,
4764	OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: `4`, Scalable: false)}, Args: C1);
4765	} else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" \|\|
4766	Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" \|\|
4767	Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" \|\|
4768	Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" \|\|
4769	Name ==
4770	"mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" \|\|
4771	Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" \|\|
4772	Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" \|\|
4773	Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" \|\|
4774	Name ==
4775	"mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" \|\|
4776	Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" \|\|
4777	Name == "cde.vcx1q.predicated.v2i64.v4i1" \|\|
4778	Name == "cde.vcx1qa.predicated.v2i64.v4i1" \|\|
4779	Name == "cde.vcx2q.predicated.v2i64.v4i1" \|\|
4780	Name == "cde.vcx2qa.predicated.v2i64.v4i1" \|\|
4781	Name == "cde.vcx3q.predicated.v2i64.v4i1" \|\|
4782	Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
4783	std::vector<Type *> Tys;
4784	unsigned ID = CI->getIntrinsicID();
4785	Type *V2I1Ty = FixedVectorType::get(ElementType: Builder.getInt1Ty(), NumElts: `2`);
4786	switch (ID) {
4787	case Intrinsic::arm_mve_mull_int_predicated:
4788	case Intrinsic::arm_mve_vqdmull_predicated:
4789	case Intrinsic::arm_mve_vldr_gather_base_predicated:
4790	Tys = {CI->getType(), CI->getOperand(i_nocapture: `0`)->getType(), V2I1Ty};
4791	break;
4792	case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
4793	case Intrinsic::arm_mve_vstr_scatter_base_predicated:
4794	case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
4795	Tys = {CI->getOperand(i_nocapture: `0`)->getType(), CI->getOperand(i_nocapture: `0`)->getType(),
4796	V2I1Ty};
4797	break;
4798	case Intrinsic::arm_mve_vldr_gather_offset_predicated:
4799	Tys = {CI->getType(), CI->getOperand(i_nocapture: `0`)->getType(),
4800	CI->getOperand(i_nocapture: `1`)->getType(), V2I1Ty};
4801	break;
4802	case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
4803	Tys = {CI->getOperand(i_nocapture: `0`)->getType(), CI->getOperand(i_nocapture: `1`)->getType(),
4804	CI->getOperand(i_nocapture: `2`)->getType(), V2I1Ty};
4805	break;
4806	case Intrinsic::arm_cde_vcx1q_predicated:
4807	case Intrinsic::arm_cde_vcx1qa_predicated:
4808	case Intrinsic::arm_cde_vcx2q_predicated:
4809	case Intrinsic::arm_cde_vcx2qa_predicated:
4810	case Intrinsic::arm_cde_vcx3q_predicated:
4811	case Intrinsic::arm_cde_vcx3qa_predicated:
4812	Tys = {CI->getOperand(i_nocapture: `1`)->getType(), V2I1Ty};
4813	break;
4814	default:
4815	llvm_unreachable("Unhandled Intrinsic!");
4816	}
4817
4818	std::vector<Value *> Ops;
4819	for (Value *Op : CI->args()) {
4820	Type *Ty = Op->getType();
4821	if (Ty->getScalarSizeInBits() == `1`) {
4822	Value *C1 = Builder.CreateIntrinsic(
4823	ID: Intrinsic::arm_mve_pred_v2i,
4824	OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), NumElements: `4`, Scalable: false)}, Args: Op);
4825	Op = Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_pred_i2v, OverloadTypes: {V2I1Ty}, Args: C1);
4826	}
4827	Ops.push_back(x: Op);
4828	}
4829
4830	return Builder.CreateIntrinsic(ID, OverloadTypes: Tys, Args: Ops, /FMFSource=/nullptr,
4831	Name: CI->getName());
4832	}
4833	llvm_unreachable("Unknown function for ARM CallBase upgrade.");
4834	}
4835
4836	// These are expected to have the arguments:
4837	// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
4838	//
4839	// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
4840	//
4841	static Value upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase CI,
4842	Function *F, IRBuilder<> &Builder) {
4843	// Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
4844	// for compatibility.
4845	auto UpgradeLegacyWMMAIUIntrinsicCall =
4846	[](Function F, CallBase CI, IRBuilder<> &Builder,
4847	ArrayRef<Type > OverloadTys) -> Value {
4848	// Prepare arguments, append clamp=0 for compatibility
4849	SmallVector<Value *, `10`> Args(CI->args().begin(), CI->args().end());
4850	Args.push_back(Elt: Builder.getFalse());
4851
4852	// Insert the declaration for the right overload types
4853	Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4854	M: F->getParent(), id: F->getIntrinsicID(), OverloadTys);
4855
4856	// Copy operand bundles if any
4857	SmallVector<OperandBundleDef, `1`> Bundles;
4858	CI->getOperandBundlesAsDefs(Defs&: Bundles);
4859
4860	// Create the new call and copy calling properties
4861	auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
4862	NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
4863	NewCall->setCallingConv(CI->getCallingConv());
4864	NewCall->setAttributes(CI->getAttributes());
4865	NewCall->setDebugLoc(CI->getDebugLoc());
4866	NewCall->copyMetadata(SrcInst: *CI);
4867	return NewCall;
4868	};
4869
4870	if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
4871	assert(CI->arg_size() == `7` && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
4872	"intrinsic should have 7 arguments");
4873	Type *T1 = CI->getArgOperand(i: `4`)->getType();
4874	Type *T2 = CI->getArgOperand(i: `1`)->getType();
4875	return UpgradeLegacyWMMAIUIntrinsicCall (F, CI, Builder, {T1, T2});
4876	}
4877	if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
4878	assert(CI->arg_size() == `8` && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
4879	"intrinsic should have 8 arguments");
4880	Type *T1 = CI->getArgOperand(i: `4`)->getType();
4881	Type *T2 = CI->getArgOperand(i: `1`)->getType();
4882	Type *T3 = CI->getArgOperand(i: `3`)->getType();
4883	Type *T4 = CI->getArgOperand(i: `5`)->getType();
4884	return UpgradeLegacyWMMAIUIntrinsicCall (F, CI, Builder, {T1, T2, T3, T4});
4885	}
4886
4887	switch (F->getIntrinsicID()) {
4888	default:
4889	break;
4890	case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4891	case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4892	case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4893	case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4894	case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4895	case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
4896	// Drop src0 and src1 modifiers.
4897	const Value *Op0 = CI->getArgOperand(i: `0`);
4898	const Value *Op2 = CI->getArgOperand(i: `2`);
4899	assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
4900	const ConstantInt *ModA = dyn_cast<ConstantInt>(Val: Op0);
4901	const ConstantInt *ModB = dyn_cast<ConstantInt>(Val: Op2);
4902	if (!ModA->isZero() \|\| !ModB->isZero())
4903	reportFatalUsageError(reason: Name + " matrix A and B modifiers shall be zero");
4904
4905	SmallVector<Value *, `8`> Args{CI->getArgOperand(i: `1`), CI->getArgOperand(i: `3`)};
4906	for (int I = `4`, E = CI->arg_size(); I < E; ++I)
4907	Args.push_back(Elt: CI->getArgOperand(i: I));
4908
4909	SmallVector<Type *, `3`> Overloads{F->getReturnType(), Args [`0`]->getType()};
4910	if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
4911	Overloads.push_back(Elt: Args [`3`]->getType());
4912	Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4913	M: F->getParent(), id: F->getIntrinsicID(), OverloadTys: Overloads);
4914
4915	SmallVector<OperandBundleDef, `1`> Bundles;
4916	CI->getOperandBundlesAsDefs(Defs&: Bundles);
4917
4918	auto *NewCall = cast<CallInst>(Val: Builder.CreateCall(Callee: NewDecl, Args, OpBundles: Bundles));
4919	NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
4920	NewCall->setCallingConv(CI->getCallingConv());
4921	NewCall->setAttributes(CI->getAttributes());
4922	NewCall->setDebugLoc(CI->getDebugLoc());
4923	NewCall->copyMetadata(SrcInst: *CI);
4924	NewCall->takeName(V: CI);
4925	return NewCall;
4926	}
4927	}
4928
4929	AtomicRMWInst::BinOp RMWOp =
4930	StringSwitch<AtomicRMWInst::BinOp>(Name)
4931	.StartsWith(S: "ds.fadd", Value: AtomicRMWInst::FAdd)
4932	.StartsWith(S: "ds.fmin", Value: AtomicRMWInst::FMin)
4933	.StartsWith(S: "ds.fmax", Value: AtomicRMWInst::FMax)
4934	.StartsWith(S: "atomic.inc.", Value: AtomicRMWInst::UIncWrap)
4935	.StartsWith(S: "atomic.dec.", Value: AtomicRMWInst::UDecWrap)
4936	.StartsWith(S: "global.atomic.fadd", Value: AtomicRMWInst::FAdd)
4937	.StartsWith(S: "flat.atomic.fadd", Value: AtomicRMWInst::FAdd)
4938	.StartsWith(S: "global.atomic.fmin", Value: AtomicRMWInst::FMin)
4939	.StartsWith(S: "flat.atomic.fmin", Value: AtomicRMWInst::FMin)
4940	.StartsWith(S: "global.atomic.fmax", Value: AtomicRMWInst::FMax)
4941	.StartsWith(S: "flat.atomic.fmax", Value: AtomicRMWInst::FMax)
4942	.StartsWith(S: "atomic.cond.sub", Value: AtomicRMWInst::USubCond)
4943	.StartsWith(S: "atomic.csub", Value: AtomicRMWInst::USubSat);
4944
4945	unsigned NumOperands = CI->getNumOperands();
4946	if (NumOperands < `3`) // Malformed bitcode.
4947	return nullptr;
4948
4949	Value *Ptr = CI->getArgOperand(i: `0`);
4950	PointerType *PtrTy = dyn_cast<PointerType>(Val: Ptr->getType());
4951	if (!PtrTy) // Malformed.
4952	return nullptr;
4953
4954	Value *Val = CI->getArgOperand(i: `1`);
4955	if (Val->getType() != CI->getType()) // Malformed.
4956	return nullptr;
4957
4958	ConstantInt OrderArg = nullptr*;
4959	bool IsVolatile = false;
4960
4961	// These should have 5 arguments (plus the callee). A separate version of the
4962	// ds_fadd intrinsic was defined for bf16 which was missing arguments.
4963	if (NumOperands > `3`)
4964	OrderArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: `2`));
4965
4966	// Ignore scope argument at 3
4967
4968	if (NumOperands > `5`) {
4969	ConstantInt *VolatileArg = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: `4`));
4970	IsVolatile = !VolatileArg \|\| !VolatileArg->isZero();
4971	}
4972
4973	AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
4974	if (OrderArg && isValidAtomicOrdering(I: OrderArg->getZExtValue()))
4975	Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
4976	if (Order == AtomicOrdering::NotAtomic \|\| Order == AtomicOrdering::Unordered)
4977	Order = AtomicOrdering::SequentiallyConsistent;
4978
4979	LLVMContext &Ctx = F->getContext();
4980
4981	// Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
4982	Type *RetTy = CI->getType();
4983	if (VectorType *VT = dyn_cast<VectorType>(Val: RetTy)) {
4984	if (VT->getElementType()->isIntegerTy(BitWidth: `16`)) {
4985	VectorType *AsBF16 =
4986	VectorType::get(ElementType: Type::getBFloatTy(C&: Ctx), EC: VT->getElementCount());
4987	Val = Builder.CreateBitCast(V: Val, DestTy: AsBF16);
4988	}
4989	}
4990
4991	// The scope argument never really worked correctly. Use agent as the most
4992	// conservative option which should still always produce the instruction.
4993	SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID(SSN: "agent");
4994	AtomicRMWInst *RMW =
4995	Builder.CreateAtomicRMW(Op: RMWOp, Ptr, Val, Align: std::nullopt, Ordering: Order, SSID);
4996
4997	unsigned AddrSpace = PtrTy->getAddressSpace();
4998	if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
4999	MDNode *EmptyMD = MDNode::get(Context&: F->getContext(), MDs: {});
5000	RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: EmptyMD);
5001	if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5002	RMW->setMetadata(Kind: "amdgpu.ignore.denormal.mode", Node: EmptyMD);
5003	}
5004
5005	if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5006	MDBuilder MDB(F->getContext());
5007	MDNode *RangeNotPrivate =
5008	MDB.createRange(Lo: APInt (`32`, AMDGPUAS::PRIVATE_ADDRESS),
5009	Hi: APInt (`32`, AMDGPUAS::PRIVATE_ADDRESS + `1`));
5010	RMW->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: RangeNotPrivate);
5011	}
5012
5013	if (IsVolatile)
5014	RMW->setVolatile(true);
5015
5016	return Builder.CreateBitCast(V: RMW, DestTy: RetTy);
5017	}
5018
5019	/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5020	/// plain MDNode, as it's the verifier's job to check these are the correct
5021	/// types later.
5022	static MDNode unwrapMAVOp(CallBase CI, unsigned Op) {
5023	if (Op < CI->arg_size()) {
5024	if (MetadataAsValue *MAV =
5025	dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op))) {
5026	Metadata *MD = MAV->getMetadata();
5027	return dyn_cast_if_present<MDNode>(Val: MD);
5028	}
5029	}
5030	return nullptr;
5031	}
5032
5033	/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5034	static Metadata unwrapMAVMetadataOp(CallBase CI, unsigned Op) {
5035	if (Op < CI->arg_size())
5036	if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(Val: CI->getArgOperand(i: Op)))
5037	return MAV->getMetadata();
5038	return nullptr;
5039	}
5040
5041	/// Convert debug intrinsic calls to non-instruction debug records.
5042	/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5043	/// \p CI - The debug intrinsic call.
5044	static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5045	DbgRecord DR = nullptr*;
5046	if (Name == "label") {
5047	DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(Label: unwrapMAVOp(CI, Op: `0`));
5048	} else if (Name == "assign") {
5049	DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5050	Type: DbgVariableRecord::LocationType::Assign, Val: unwrapMAVMetadataOp(CI, Op: `0`),
5051	Variable: unwrapMAVOp(CI, Op: `1`), Expression: unwrapMAVOp(CI, Op: `2`), AssignID: unwrapMAVOp(CI, Op: `3`),
5052	Address: unwrapMAVMetadataOp(CI, Op: `4`),
5053	/The address is a Value ref, it will be stored as a Metadata /
5054	AddressExpression: unwrapMAVOp(CI, Op: `5`));
5055	} else if (Name == "declare") {
5056	DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5057	Type: DbgVariableRecord::LocationType::Declare, Val: unwrapMAVMetadataOp(CI, Op: `0`),
5058	Variable: unwrapMAVOp(CI, Op: `1`), Expression: unwrapMAVOp(CI, Op: `2`), AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5059	} else if (Name == "addr") {
5060	// Upgrade dbg.addr to dbg.value with DW_OP_deref.
5061	MDNode *ExprNode = unwrapMAVOp(CI, Op: `2`);
5062	// Don't try to add something to the expression if it's not an expression.
5063	// Instead, allow the verifier to fail later.
5064	if (DIExpression *Expr = dyn_cast<DIExpression>(Val: ExprNode)) {
5065	ExprNode = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
5066	}
5067	DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5068	Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: `0`),
5069	Variable: unwrapMAVOp(CI, Op: `1`), Expression: ExprNode, AssignID: nullptr, Address: nullptr, AddressExpression: nullptr);
5070	} else if (Name == "value") {
5071	// An old version of dbg.value had an extra offset argument.
5072	unsigned VarOp = `1`;
5073	unsigned ExprOp = `2`;
5074	if (CI->arg_size() == `4`) {
5075	auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: `1`));
5076	// Nonzero offset dbg.values get dropped without a replacement.
5077	if (!Offset \|\| !Offset->isNullValue())
5078	return;
5079	VarOp = `2`;
5080	ExprOp = `3`;
5081	}
5082	DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5083	Type: DbgVariableRecord::LocationType::Value, Val: unwrapMAVMetadataOp(CI, Op: `0`),
5084	Variable: unwrapMAVOp(CI, Op: VarOp), Expression: unwrapMAVOp(CI, Op: ExprOp), AssignID: nullptr, Address: nullptr,
5085	AddressExpression: nullptr);
5086	}
5087	DR->setDebugLoc(CI->getDebugLoc());
5088	assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5089	CI->getParent()->insertDbgRecordBefore(DR, Here: CI->getIterator());
5090	}
5091
5092	static Value upgradeVectorSplice(CallBase CI, IRBuilder<> &Builder) {
5093	auto *Offset = dyn_cast<ConstantInt>(Val: CI->getArgOperand(i: `2`));
5094	if (!Offset)
5095	reportFatalUsageError(reason: "Invalid llvm.vector.splice offset argument");
5096	int64_t OffsetVal = Offset->getSExtValue();
5097	return Builder.CreateIntrinsic(ID: OffsetVal >= `0`
5098	? Intrinsic::vector_splice_left
5099	: Intrinsic::vector_splice_right,
5100	OverloadTypes: CI->getType(),
5101	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
5102	Builder.getInt32(C: std::abs(i: OffsetVal))});
5103	}
5104
5105	static Value upgradeConvertIntrinsicCall(StringRef Name, CallBase CI,
5106	Function *F, IRBuilder<> &Builder) {
5107	if (Name.starts_with(Prefix: "to.fp16")) {
5108	Value *Cast =
5109	Builder.CreateFPTrunc(V: CI->getArgOperand(i: `0`), DestTy: Builder.getHalfTy());
5110	return Builder.CreateBitCast(V: Cast, DestTy: CI->getType());
5111	}
5112
5113	if (Name.starts_with(Prefix: "from.fp16")) {
5114	Value *Cast =
5115	Builder.CreateBitCast(V: CI->getArgOperand(i: `0`), DestTy: Builder.getHalfTy());
5116	return Builder.CreateFPExt(V: Cast, DestTy: CI->getType());
5117	}
5118
5119	return nullptr;
5120	}
5121
5122	static bool upgradeIntrinsicCallWithDefaultArgs(CallBase CI, Function NewFn,
5123	IRBuilder<> &Builder) {
5124	Intrinsic::ID IID = NewFn->getIntrinsicID();
5125
5126	auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5127	if (Defaults.empty())
5128	return false;
5129
5130	unsigned OldArgCount = CI->arg_size();
5131	unsigned NewArgCount = NewFn->arg_size();
5132
5133	// If the caller already supplied all arguments (or more), nothing to do.
5134	// This mirrors C++ semantics: an explicitly-passed value is never overridden.
5135	if (OldArgCount >= NewArgCount)
5136	return false;
5137
5138	// Start with the existing arguments from the old call.
5139	SmallVector<Value *, `8`> NewArgs(CI->args());
5140
5141	// Defaults are a contiguous trailing block, so checking the first missing
5142	// argument is enough.
5143	if (OldArgCount < FirstDefault)
5144	return false;
5145
5146	// Fill in each missing trailing argument from the table.
5147	FunctionType *NewFT = NewFn->getFunctionType();
5148	for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5149	assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5150	"missing argument outside the default range");
5151	Type *ParamTy = NewFT->getParamType(i: Idx);
5152
5153	// Only integer types are supported (i1, i8, i16, i32, i64).
5154	if (!ParamTy->isIntegerTy())
5155	return false;
5156	NewArgs.push_back(Elt: ConstantInt::get(Ty: ParamTy, V: Defaults [Idx - FirstDefault]));
5157	}
5158
5159	// Preserve operand bundles by creating the call with them.
5160	SmallVector<OperandBundleDef, `1`> OpBundles;
5161	CI->getOperandBundlesAsDefs(Defs&: OpBundles);
5162	CallInst *NewCall = Builder.CreateCall(Callee: NewFn, Args: NewArgs, OpBundles);
5163
5164	NewCall->takeName(V: CI);
5165	NewCall->setCallingConv(CI->getCallingConv());
5166	NewCall->copyMetadata(SrcInst: *CI);
5167	if (auto *OldCI = dyn_cast<CallInst>(Val: CI))
5168	NewCall->setTailCallKind(OldCI->getTailCallKind());
5169
5170	CI->replaceAllUsesWith(V: NewCall);
5171	CI->eraseFromParent();
5172	return true;
5173	}
5174
5175	/// Upgrade a call to an old intrinsic. All argument and return casting must be
5176	/// provided to seamlessly integrate with existing context.
5177	void llvm::UpgradeIntrinsicCall(CallBase CI, Function NewFn) {
5178	// Note dyn_cast to Function is not quite the same as getCalledFunction, which
5179	// checks the callee's function type matches. It's likely we need to handle
5180	// type changes here.
5181	Function *F = dyn_cast<Function>(Val: CI->getCalledOperand());
5182	if (!F)
5183	return;
5184
5185	LLVMContext &C = CI->getContext();
5186	IRBuilder<> Builder(C);
5187	if (isa<FPMathOperator>(Val: CI))
5188	Builder.setFastMathFlags(CI->getFastMathFlags());
5189	Builder.SetInsertPoint(TheBB: CI->getParent(), IP: CI->getIterator());
5190
5191	if (!NewFn) {
5192	// Get the Function's name.
5193	StringRef Name = F->getName();
5194	if (!Name.consume_front(Prefix: "llvm."))
5195	llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5196
5197	bool IsX86 = Name.consume_front(Prefix: "x86.");
5198	bool IsNVVM = Name.consume_front(Prefix: "nvvm.");
5199	bool IsAArch64 = Name.consume_front(Prefix: "aarch64.");
5200	bool IsARM = Name.consume_front(Prefix: "arm.");
5201	bool IsAMDGCN = Name.consume_front(Prefix: "amdgcn.");
5202	bool IsDbg = Name.consume_front(Prefix: "dbg.");
5203	bool IsOldSplice =
5204	(Name.consume_front(Prefix: "experimental.vector.splice") \|\|
5205	Name.consume_front(Prefix: "vector.splice")) &&
5206	!(Name.starts_with(Prefix: ".left") \|\| Name.starts_with(Prefix: ".right"));
5207	Value Rep = nullptr*;
5208
5209	if (!IsX86 && Name == "stackprotectorcheck") {
5210	Rep = nullptr;
5211	} else if (IsNVVM) {
5212	Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5213	} else if (IsX86) {
5214	Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5215	} else if (IsAArch64) {
5216	Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5217	} else if (IsARM) {
5218	Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5219	} else if (IsAMDGCN) {
5220	Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5221	} else if (IsDbg) {
5222	upgradeDbgIntrinsicToDbgRecord(Name, CI);
5223	} else if (IsOldSplice) {
5224	Rep = upgradeVectorSplice(CI, Builder);
5225	} else if (Name.consume_front(Prefix: "convert.")) {
5226	Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5227	} else if (Name == "lifetime.start.i64" \|\| Name == "lifetime.end.i64") {
5228	// Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5229	Rep = nullptr;
5230	} else {
5231	llvm_unreachable("Unknown function for CallBase upgrade.");
5232	}
5233
5234	if (Rep)
5235	CI->replaceAllUsesWith(V: Rep);
5236	CI->eraseFromParent();
5237	return;
5238	}
5239
5240	const auto &DefaultCase = [&]() -> void {
5241	if (F == NewFn)
5242	return;
5243
5244	if (CI->getFunctionType() == NewFn->getFunctionType()) {
5245	// Handle generic mangling change.
5246	assert(
5247	(CI->getCalledFunction()->getName() != NewFn->getName()) &&
5248	"Unknown function for CallBase upgrade and isn't just a name change");
5249	CI->setCalledFunction(NewFn);
5250	return;
5251	}
5252
5253	// This must be an upgrade from a named to a literal struct.
5254	if (auto *OldST = dyn_cast<StructType>(Val: CI->getType())) {
5255	assert(OldST != NewFn->getReturnType() &&
5256	"Return type must have changed");
5257	assert(OldST->getNumElements() ==
5258	cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5259	"Must have same number of elements");
5260
5261	SmallVector<Value *> Args(CI->args());
5262	CallInst *NewCI = Builder.CreateCall(Callee: NewFn, Args);
5263	NewCI->setAttributes(CI->getAttributes());
5264	Value *Res = PoisonValue::get(T: OldST);
5265	for (unsigned Idx = `0`; Idx < OldST->getNumElements(); ++Idx) {
5266	Value *Elem = Builder.CreateExtractValue(Agg: NewCI, Idxs: Idx);
5267	Res = Builder.CreateInsertValue(Agg: Res, Val: Elem, Idxs: Idx);
5268	}
5269	CI->replaceAllUsesWith(V: Res);
5270	CI->eraseFromParent();
5271	return;
5272	}
5273
5274	// We're probably about to produce something invalid. Let the verifier catch
5275	// it instead of dying here.
5276	CI->setCalledOperand(
5277	ConstantExpr::getPointerCast(C: NewFn, Ty: CI->getCalledOperand()->getType()));
5278	return;
5279	};
5280	CallInst NewCall = nullptr*;
5281	switch (NewFn->getIntrinsicID()) {
5282	default: {
5283	// Last resort: try the data-driven default-arg upgrade.
5284	// Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5285	// in its .td definition, without needing a dedicated case.
5286	if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5287	return;
5288	DefaultCase ();
5289	return;
5290	}
5291	case Intrinsic::arm_neon_vst1:
5292	case Intrinsic::arm_neon_vst2:
5293	case Intrinsic::arm_neon_vst3:
5294	case Intrinsic::arm_neon_vst4:
5295	case Intrinsic::arm_neon_vst2lane:
5296	case Intrinsic::arm_neon_vst3lane:
5297	case Intrinsic::arm_neon_vst4lane: {
5298	SmallVector<Value *, `4`> Args(CI->args());
5299	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5300	break;
5301	}
5302	case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5303	case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5304	case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5305	LLVMContext &Ctx = F->getParent()->getContext();
5306	SmallVector<Value *, `4`> Args(CI->args());
5307	Args [`3`] = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx),
5308	V: cast<ConstantInt>(Val: Args [`3`])->getZExtValue());
5309	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5310	break;
5311	}
5312	case Intrinsic::aarch64_sve_ld3_sret:
5313	case Intrinsic::aarch64_sve_ld4_sret:
5314	case Intrinsic::aarch64_sve_ld2_sret: {
5315	// Is this a trivial remangle of the name to support ptr address spaces?
5316	if (isa<StructType>(Val: F->getReturnType())) {
5317	DefaultCase ();
5318	return;
5319	}
5320
5321	StringRef Name = F->getName();
5322	Name = Name.substr(Start: `5`);
5323	unsigned N = StringSwitch<unsigned>(Name)
5324	.StartsWith(S: "aarch64.sve.ld2", Value: `2`)
5325	.StartsWith(S: "aarch64.sve.ld3", Value: `3`)
5326	.StartsWith(S: "aarch64.sve.ld4", Value: `4`)
5327	.Default(Value: `0`);
5328	auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5329	unsigned MinElts = RetTy->getMinNumElements() / N;
5330	SmallVector<Value *, `2`> Args(CI->args());
5331	Value *NewLdCall = Builder.CreateCall(Callee: NewFn, Args);
5332	Value *Ret = llvm::PoisonValue::get(T: RetTy);
5333	for (unsigned I = `0`; I < N; I++) {
5334	Value *SRet = Builder.CreateExtractValue(Agg: NewLdCall, Idxs: I);
5335	Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: SRet, Idx: I * MinElts);
5336	}
5337	NewCall = dyn_cast<CallInst>(Val: Ret);
5338	break;
5339	}
5340
5341	case Intrinsic::coro_end: {
5342	SmallVector<Value *, `3`> Args(CI->args());
5343	Args.push_back(Elt: ConstantTokenNone::get(Context&: CI->getContext()));
5344	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5345	break;
5346	}
5347
5348	case Intrinsic::vector_extract: {
5349	StringRef Name = F->getName();
5350	Name = Name.substr(Start: `5`); // Strip llvm
5351	if (!Name.starts_with(Prefix: "aarch64.sve.tuple.get")) {
5352	DefaultCase ();
5353	return;
5354	}
5355	auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5356	unsigned MinElts = RetTy->getMinNumElements();
5357	unsigned I = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
5358	Value NewIdx = ConstantInt::get(Ty: Type::getInt64Ty(C), V: I MinElts);
5359	NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: `0`), NewIdx});
5360	break;
5361	}
5362
5363	case Intrinsic::vector_insert: {
5364	StringRef Name = F->getName();
5365	Name = Name.substr(Start: `5`);
5366	if (!Name.starts_with(Prefix: "aarch64.sve.tuple")) {
5367	DefaultCase ();
5368	return;
5369	}
5370	if (Name.starts_with(Prefix: "aarch64.sve.tuple.set")) {
5371	unsigned I = cast<ConstantInt>(Val: CI->getArgOperand(i: `1`))->getZExtValue();
5372	auto *Ty = cast<ScalableVectorType>(Val: CI->getArgOperand(i: `2`)->getType());
5373	Value *NewIdx =
5374	ConstantInt::get(Ty: Type::getInt64Ty(C), V: I * Ty->getMinNumElements());
5375	NewCall = Builder.CreateCall(
5376	Callee: NewFn, Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `2`), NewIdx});
5377	break;
5378	}
5379	if (Name.starts_with(Prefix: "aarch64.sve.tuple.create")) {
5380	unsigned N = StringSwitch<unsigned>(Name)
5381	.StartsWith(S: "aarch64.sve.tuple.create2", Value: `2`)
5382	.StartsWith(S: "aarch64.sve.tuple.create3", Value: `3`)
5383	.StartsWith(S: "aarch64.sve.tuple.create4", Value: `4`)
5384	.Default(Value: `0`);
5385	assert(N > `1` && "Create is expected to be between 2-4");
5386	auto *RetTy = cast<ScalableVectorType>(Val: F->getReturnType());
5387	Value *Ret = llvm::PoisonValue::get(T: RetTy);
5388	unsigned MinElts = RetTy->getMinNumElements() / N;
5389	for (unsigned I = `0`; I < N; I++) {
5390	Value *V = CI->getArgOperand(i: I);
5391	Ret = Builder.CreateInsertVector(DstType: RetTy, SrcVec: Ret, SubVec: V, Idx: I * MinElts);
5392	}
5393	NewCall = dyn_cast<CallInst>(Val: Ret);
5394	}
5395	break;
5396	}
5397
5398	case Intrinsic::arm_neon_bfdot:
5399	case Intrinsic::arm_neon_bfmmla:
5400	case Intrinsic::arm_neon_bfmlalb:
5401	case Intrinsic::arm_neon_bfmlalt:
5402	case Intrinsic::aarch64_neon_bfdot:
5403	case Intrinsic::aarch64_neon_bfmmla:
5404	case Intrinsic::aarch64_neon_bfmlalb:
5405	case Intrinsic::aarch64_neon_bfmlalt: {
5406	SmallVector<Value *, `3`> Args;
5407	assert(CI->arg_size() == `3` &&
5408	"Mismatch between function args and call args");
5409	size_t OperandWidth =
5410	CI->getArgOperand(i: `1`)->getType()->getPrimitiveSizeInBits();
5411	assert((OperandWidth == `64` \|\| OperandWidth == `128`) &&
5412	"Unexpected operand width");
5413	Type *NewTy = FixedVectorType::get(ElementType: Type::getBFloatTy(C), NumElts: OperandWidth / `16`);
5414	auto Iter = CI->args().begin();
5415	Args.push_back(Elt: *Iter++);
5416	Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
5417	Args.push_back(Elt: Builder.CreateBitCast(V: *Iter++, DestTy: NewTy));
5418	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5419	break;
5420	}
5421
5422	case Intrinsic::bitreverse:
5423	NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: `0`)});
5424	break;
5425
5426	case Intrinsic::ctlz:
5427	case Intrinsic::cttz: {
5428	if (CI->arg_size() != `1`) {
5429	DefaultCase ();
5430	return;
5431	}
5432
5433	NewCall =
5434	Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: `0`), Builder.getFalse()});
5435	break;
5436	}
5437
5438	case Intrinsic::objectsize: {
5439	Value *NullIsUnknownSize =
5440	CI->arg_size() == `2` ? Builder.getFalse() : CI->getArgOperand(i: `2`);
5441	Value *Dynamic =
5442	CI->arg_size() < `4` ? Builder.getFalse() : CI->getArgOperand(i: `3`);
5443	NewCall = Builder.CreateCall(
5444	Callee: NewFn, Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), NullIsUnknownSize, Dynamic});
5445	break;
5446	}
5447
5448	case Intrinsic::ctpop:
5449	NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: `0`)});
5450	break;
5451	case Intrinsic::dbg_value: {
5452	StringRef Name = F->getName();
5453	Name = Name.substr(Start: `5`); // Strip llvm.
5454	// Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5455	if (Name.starts_with(Prefix: "dbg.addr")) {
5456	DIExpression *Expr = cast<DIExpression>(
5457	Val: cast<MetadataAsValue>(Val: CI->getArgOperand(i: `2`))->getMetadata());
5458	Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
5459	NewCall =
5460	Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
5461	MetadataAsValue::get(Context&: C, MD: Expr)});
5462	break;
5463	}
5464
5465	// Upgrade from the old version that had an extra offset argument.
5466	assert(CI->arg_size() == `4`);
5467	// Drop nonzero offsets instead of attempting to upgrade them.
5468	if (auto *Offset = dyn_cast_or_null<Constant>(Val: CI->getArgOperand(i: `1`)))
5469	if (Offset->isNullValue()) {
5470	NewCall = Builder.CreateCall(
5471	Callee: NewFn,
5472	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `2`), CI->getArgOperand(i: `3`)});
5473	break;
5474	}
5475	CI->eraseFromParent();
5476	return;
5477	}
5478
5479	case Intrinsic::ptr_annotation:
5480	// Upgrade from versions that lacked the annotation attribute argument.
5481	if (CI->arg_size() != `4`) {
5482	DefaultCase ();
5483	return;
5484	}
5485
5486	// Create a new call with an added null annotation attribute argument.
5487	NewCall = Builder.CreateCall(
5488	Callee: NewFn,
5489	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), CI->getArgOperand(i: `2`),
5490	CI->getArgOperand(i: `3`), ConstantPointerNull::get(T: Builder.getPtrTy())});
5491	NewCall->takeName(V: CI);
5492	CI->replaceAllUsesWith(V: NewCall);
5493	CI->eraseFromParent();
5494	return;
5495
5496	case Intrinsic::var_annotation:
5497	// Upgrade from versions that lacked the annotation attribute argument.
5498	if (CI->arg_size() != `4`) {
5499	DefaultCase ();
5500	return;
5501	}
5502	// Create a new call with an added null annotation attribute argument.
5503	NewCall = Builder.CreateCall(
5504	Callee: NewFn,
5505	Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`), CI->getArgOperand(i: `2`),
5506	CI->getArgOperand(i: `3`), ConstantPointerNull::get(T: Builder.getPtrTy())});
5507	NewCall->takeName(V: CI);
5508	CI->replaceAllUsesWith(V: NewCall);
5509	CI->eraseFromParent();
5510	return;
5511
5512	case Intrinsic::riscv_aes32dsi:
5513	case Intrinsic::riscv_aes32dsmi:
5514	case Intrinsic::riscv_aes32esi:
5515	case Intrinsic::riscv_aes32esmi:
5516	case Intrinsic::riscv_sm4ks:
5517	case Intrinsic::riscv_sm4ed: {
5518	// The last argument to these intrinsics used to be i8 and changed to i32.
5519	// The type overload for sm4ks and sm4ed was removed.
5520	Value *Arg2 = CI->getArgOperand(i: `2`);
5521	if (Arg2->getType()->isIntegerTy(BitWidth: `32`) && !CI->getType()->isIntegerTy(BitWidth: `64`))
5522	return;
5523
5524	Value *Arg0 = CI->getArgOperand(i: `0`);
5525	Value *Arg1 = CI->getArgOperand(i: `1`);
5526	if (CI->getType()->isIntegerTy(BitWidth: `64`)) {
5527	Arg0 = Builder.CreateTrunc(V: Arg0, DestTy: Builder.getInt32Ty());
5528	Arg1 = Builder.CreateTrunc(V: Arg1, DestTy: Builder.getInt32Ty());
5529	}
5530
5531	Arg2 = ConstantInt::get(Ty: Type::getInt32Ty(C),
5532	V: cast<ConstantInt>(Val: Arg2)->getZExtValue());
5533
5534	NewCall = Builder.CreateCall(Callee: NewFn, Args: {Arg0, Arg1, Arg2});
5535	Value *Res = NewCall;
5536	if (Res->getType() != CI->getType())
5537	Res = Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /isSigned/ true);
5538	NewCall->takeName(V: CI);
5539	CI->replaceAllUsesWith(V: Res);
5540	CI->eraseFromParent();
5541	return;
5542	}
5543	case Intrinsic::nvvm_mapa_shared_cluster: {
5544	// Create a new call with the correct address space.
5545	NewCall =
5546	Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`)});
5547	Value *Res = NewCall;
5548	Res = Builder.CreateAddrSpaceCast(
5549	V: Res, DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED));
5550	NewCall->takeName(V: CI);
5551	CI->replaceAllUsesWith(V: Res);
5552	CI->eraseFromParent();
5553	return;
5554	}
5555	case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5556	case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5557	// Create a new call with the correct address space.
5558	SmallVector<Value *, `4`> Args(CI->args());
5559	Args [`0`] = Builder.CreateAddrSpaceCast(
5560	V: Args [`0`], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5561
5562	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5563	NewCall->takeName(V: CI);
5564	CI->replaceAllUsesWith(V: NewCall);
5565	CI->eraseFromParent();
5566	return;
5567	}
5568	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5569	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5570	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5571	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5572	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5573	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5574	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5575	case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5576	SmallVector<Value *, `16`> Args(CI->args());
5577
5578	// Create AddrSpaceCast to shared_cluster if needed.
5579	// This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5580	unsigned AS = CI->getArgOperand(i: `0`)->getType()->getPointerAddressSpace();
5581	if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5582	Args [`0`] = Builder.CreateAddrSpaceCast(
5583	V: Args [`0`], DestTy: Builder.getPtrTy(AddrSpace: NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5584
5585	// Attach the flag argument for cta_group, with a
5586	// default value of 0. This handles case (2) in
5587	// shouldUpgradeNVPTXTMAG2SIntrinsics().
5588	size_t NumArgs = CI->arg_size();
5589	Value *FlagArg = CI->getArgOperand(i: NumArgs - `3`);
5590	if (!FlagArg->getType()->isIntegerTy(BitWidth: `1`))
5591	Args.push_back(Elt: ConstantInt::get(Ty: Builder.getInt32Ty(), V: `0`));
5592
5593	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5594	NewCall->takeName(V: CI);
5595	CI->replaceAllUsesWith(V: NewCall);
5596	CI->eraseFromParent();
5597	return;
5598	}
5599	case Intrinsic::riscv_sha256sig0:
5600	case Intrinsic::riscv_sha256sig1:
5601	case Intrinsic::riscv_sha256sum0:
5602	case Intrinsic::riscv_sha256sum1:
5603	case Intrinsic::riscv_sm3p0:
5604	case Intrinsic::riscv_sm3p1: {
5605	// The last argument to these intrinsics used to be i8 and changed to i32.
5606	// The type overload for sm4ks and sm4ed was removed.
5607	if (!CI->getType()->isIntegerTy(BitWidth: `64`))
5608	return;
5609
5610	Value *Arg =
5611	Builder.CreateTrunc(V: CI->getArgOperand(i: `0`), DestTy: Builder.getInt32Ty());
5612
5613	NewCall = Builder.CreateCall(Callee: NewFn, Args: Arg);
5614	Value *Res =
5615	Builder.CreateIntCast(V: NewCall, DestTy: CI->getType(), /isSigned/ true);
5616	NewCall->takeName(V: CI);
5617	CI->replaceAllUsesWith(V: Res);
5618	CI->eraseFromParent();
5619	return;
5620	}
5621
5622	case Intrinsic::x86_xop_vfrcz_ss:
5623	case Intrinsic::x86_xop_vfrcz_sd:
5624	NewCall = Builder.CreateCall(Callee: NewFn, Args: {CI->getArgOperand(i: `1`)});
5625	break;
5626
5627	case Intrinsic::x86_xop_vpermil2pd:
5628	case Intrinsic::x86_xop_vpermil2ps:
5629	case Intrinsic::x86_xop_vpermil2pd_256:
5630	case Intrinsic::x86_xop_vpermil2ps_256: {
5631	SmallVector<Value *, `4`> Args(CI->args());
5632	VectorType *FltIdxTy = cast<VectorType>(Val: Args [`2`]->getType());
5633	VectorType *IntIdxTy = VectorType::getInteger(VTy: FltIdxTy);
5634	Args [`2`] = Builder.CreateBitCast(V: Args [`2`], DestTy: IntIdxTy);
5635	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5636	break;
5637	}
5638
5639	case Intrinsic::x86_sse41_ptestc:
5640	case Intrinsic::x86_sse41_ptestz:
5641	case Intrinsic::x86_sse41_ptestnzc: {
5642	// The arguments for these intrinsics used to be v4f32, and changed
5643	// to v2i64. This is purely a nop, since those are bitwise intrinsics.
5644	// So, the only thing required is a bitcast for both arguments.
5645	// First, check the arguments have the old type.
5646	Value *Arg0 = CI->getArgOperand(i: `0`);
5647	if (Arg0->getType() != FixedVectorType::get(ElementType: Type::getFloatTy(C), NumElts: `4`))
5648	return;
5649
5650	// Old intrinsic, add bitcasts
5651	Value *Arg1 = CI->getArgOperand(i: `1`);
5652
5653	auto *NewVecTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C), NumElts: `2`);
5654
5655	Value *BC0 = Builder.CreateBitCast(V: Arg0, DestTy: NewVecTy, Name: "cast");
5656	Value *BC1 = Builder.CreateBitCast(V: Arg1, DestTy: NewVecTy, Name: "cast");
5657
5658	NewCall = Builder.CreateCall(Callee: NewFn, Args: {BC0, BC1});
5659	break;
5660	}
5661
5662	case Intrinsic::x86_rdtscp: {
5663	// This used to take 1 arguments. If we have no arguments, it is already
5664	// upgraded.
5665	if (CI->getNumOperands() == `0`)
5666	return;
5667
5668	NewCall = Builder.CreateCall(Callee: NewFn);
5669	// Extract the second result and store it.
5670	Value *Data = Builder.CreateExtractValue(Agg: NewCall, Idxs: `1`);
5671	Builder.CreateAlignedStore(Val: Data, Ptr: CI->getArgOperand(i: `0`), Align: Align (`1`));
5672	// Replace the original call result with the first result of the new call.
5673	Value *TSC = Builder.CreateExtractValue(Agg: NewCall, Idxs: `0`);
5674
5675	NewCall->takeName(V: CI);
5676	CI->replaceAllUsesWith(V: TSC);
5677	CI->eraseFromParent();
5678	return;
5679	}
5680
5681	case Intrinsic::x86_sse41_insertps:
5682	case Intrinsic::x86_sse41_dppd:
5683	case Intrinsic::x86_sse41_dpps:
5684	case Intrinsic::x86_sse41_mpsadbw:
5685	case Intrinsic::x86_avx_dp_ps_256:
5686	case Intrinsic::x86_avx2_mpsadbw: {
5687	// Need to truncate the last argument from i32 to i8 -- this argument models
5688	// an inherently 8-bit immediate operand to these x86 instructions.
5689	SmallVector<Value *, `4`> Args(CI->args());
5690
5691	// Replace the last argument with a trunc.
5692	Args.back() = Builder.CreateTrunc(V: Args.back(), DestTy: Type::getInt8Ty(C), Name: "trunc");
5693	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5694	break;
5695	}
5696
5697	case Intrinsic::x86_avx512_mask_cmp_pd_128:
5698	case Intrinsic::x86_avx512_mask_cmp_pd_256:
5699	case Intrinsic::x86_avx512_mask_cmp_pd_512:
5700	case Intrinsic::x86_avx512_mask_cmp_ps_128:
5701	case Intrinsic::x86_avx512_mask_cmp_ps_256:
5702	case Intrinsic::x86_avx512_mask_cmp_ps_512: {
5703	SmallVector<Value *, `4`> Args(CI->args());
5704	unsigned NumElts =
5705	cast<FixedVectorType>(Val: Args [`0`]->getType())->getNumElements();
5706	Args [`3`] = getX86MaskVec(Builder, Mask: Args [`3`], NumElts);
5707
5708	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5709	Value Res = applyX86MaskOn1BitsVec(Builder, Vec: NewCall, Mask: nullptr*);
5710
5711	NewCall->takeName(V: CI);
5712	CI->replaceAllUsesWith(V: Res);
5713	CI->eraseFromParent();
5714	return;
5715	}
5716
5717	case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
5718	case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
5719	case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
5720	case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
5721	case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
5722	case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
5723	SmallVector<Value *, `4`> Args(CI->args());
5724	unsigned NumElts = cast<FixedVectorType>(Val: CI->getType())->getNumElements();
5725	if (NewFn->getIntrinsicID() ==
5726	Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
5727	Args [`1`] = Builder.CreateBitCast(
5728	V: Args [`1`], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
5729
5730	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5731	Value *Res = Builder.CreateBitCast(
5732	V: NewCall, DestTy: FixedVectorType::get(ElementType: Builder.getInt16Ty(), NumElts));
5733
5734	NewCall->takeName(V: CI);
5735	CI->replaceAllUsesWith(V: Res);
5736	CI->eraseFromParent();
5737	return;
5738	}
5739	case Intrinsic::x86_avx512bf16_dpbf16ps_128:
5740	case Intrinsic::x86_avx512bf16_dpbf16ps_256:
5741	case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
5742	SmallVector<Value *, `4`> Args(CI->args());
5743	unsigned NumElts =
5744	cast<FixedVectorType>(Val: CI->getType())->getNumElements() * `2`;
5745	Args [`1`] = Builder.CreateBitCast(
5746	V: Args [`1`], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
5747	Args [`2`] = Builder.CreateBitCast(
5748	V: Args [`2`], DestTy: FixedVectorType::get(ElementType: Builder.getBFloatTy(), NumElts));
5749
5750	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5751	break;
5752	}
5753
5754	case Intrinsic::thread_pointer: {
5755	NewCall = Builder.CreateCall(Callee: NewFn, Args: {});
5756	break;
5757	}
5758
5759	case Intrinsic::memcpy:
5760	case Intrinsic::memmove:
5761	case Intrinsic::memset: {
5762	// We have to make sure that the call signature is what we're expecting.
5763	// We only want to change the old signatures by removing the alignment arg:
5764	// @llvm.mem[cpy\|move]...(i8, i8, i[32\|i64], i32, i1)
5765	// -> @llvm.mem[cpy\|move]...(i8, i8, i[32\|i64], i1)
5766	// @llvm.memset...(i8, i8, i[32\|64], i32, i1)*
5767	// -> @llvm.memset...(i8, i8, i[32\|64], i1)*
5768	// Note: i8's in the above can be any pointer type*
5769	if (CI->arg_size() != `5`) {
5770	DefaultCase ();
5771	return;
5772	}
5773	// Remove alignment argument (3), and add alignment attributes to the
5774	// dest/src pointers.
5775	Value *Args[`4`] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
5776	CI->getArgOperand(i: `2`), CI->getArgOperand(i: `4`)};
5777	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5778	AttributeList OldAttrs = CI->getAttributes();
5779	AttributeList NewAttrs = AttributeList::get(
5780	C, FnAttrs: OldAttrs.getFnAttrs(), RetAttrs: OldAttrs.getRetAttrs(),
5781	ArgAttrs: {OldAttrs.getParamAttrs(ArgNo: `0`), OldAttrs.getParamAttrs(ArgNo: `1`),
5782	OldAttrs.getParamAttrs(ArgNo: `2`), OldAttrs.getParamAttrs(ArgNo: `4`)});
5783	NewCall->setAttributes(NewAttrs);
5784	auto *MemCI = cast<MemIntrinsic>(Val: NewCall);
5785	// All mem intrinsics support dest alignment.
5786	const ConstantInt *Align = cast<ConstantInt>(Val: CI->getArgOperand(i: `3`));
5787	MemCI->setDestAlignment(Align->getMaybeAlignValue());
5788	// Memcpy/Memmove also support source alignment.
5789	if (auto *MTI = dyn_cast<MemTransferInst>(Val: MemCI))
5790	MTI->setSourceAlignment(Align->getMaybeAlignValue());
5791	break;
5792	}
5793
5794	case Intrinsic::masked_load:
5795	case Intrinsic::masked_gather:
5796	case Intrinsic::masked_store:
5797	case Intrinsic::masked_scatter: {
5798	if (CI->arg_size() != `4`) {
5799	DefaultCase ();
5800	return;
5801	}
5802
5803	auto GetMaybeAlign = [](Value *Op) {
5804	if (auto *CI = dyn_cast<ConstantInt>(Val: Op)) {
5805	uint64_t Val = CI->getZExtValue();
5806	if (Val == `0`)
5807	return MaybeAlign ();
5808	if (isPowerOf2_64(Value: Val))
5809	return MaybeAlign (Val);
5810	}
5811	reportFatalUsageError(reason: "Invalid alignment argument");
5812	};
5813	auto GetAlign = [&](Value *Op) {
5814	MaybeAlign Align = GetMaybeAlign (Op);
5815	if (Align)
5816	return *Align;
5817	reportFatalUsageError(reason: "Invalid zero alignment argument");
5818	};
5819
5820	const DataLayout &DL = CI->getDataLayout();
5821	switch (NewFn->getIntrinsicID()) {
5822	case Intrinsic::masked_load:
5823	NewCall = Builder.CreateMaskedLoad(
5824	Ty: CI->getType(), Ptr: CI->getArgOperand(i: `0`), Alignment: GetAlign (CI->getArgOperand(i: `1`)),
5825	Mask: CI->getArgOperand(i: `2`), PassThru: CI->getArgOperand(i: `3`));
5826	break;
5827	case Intrinsic::masked_gather:
5828	NewCall = Builder.CreateMaskedGather(
5829	Ty: CI->getType(), Ptrs: CI->getArgOperand(i: `0`),
5830	Alignment: DL.getValueOrABITypeAlignment(Alignment: GetMaybeAlign (CI->getArgOperand(i: `1`)),
5831	Ty: CI->getType()->getScalarType()),
5832	Mask: CI->getArgOperand(i: `2`), PassThru: CI->getArgOperand(i: `3`));
5833	break;
5834	case Intrinsic::masked_store:
5835	NewCall = Builder.CreateMaskedStore(
5836	Val: CI->getArgOperand(i: `0`), Ptr: CI->getArgOperand(i: `1`),
5837	Alignment: GetAlign (CI->getArgOperand(i: `2`)), Mask: CI->getArgOperand(i: `3`));
5838	break;
5839	case Intrinsic::masked_scatter:
5840	NewCall = Builder.CreateMaskedScatter(
5841	Val: CI->getArgOperand(i: `0`), Ptrs: CI->getArgOperand(i: `1`),
5842	Alignment: DL.getValueOrABITypeAlignment(
5843	Alignment: GetMaybeAlign (CI->getArgOperand(i: `2`)),
5844	Ty: CI->getArgOperand(i: `0`)->getType()->getScalarType()),
5845	Mask: CI->getArgOperand(i: `3`));
5846	break;
5847	default:
5848	llvm_unreachable("Unexpected intrinsic ID");
5849	}
5850	// Previous metadata is still valid.
5851	NewCall->copyMetadata(SrcInst: *CI);
5852	NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
5853	break;
5854	}
5855
5856	case Intrinsic::lifetime_start:
5857	case Intrinsic::lifetime_end: {
5858	if (CI->arg_size() != `2`) {
5859	DefaultCase ();
5860	return;
5861	}
5862
5863	Value *Ptr = CI->getArgOperand(i: `1`);
5864	// Try to strip pointer casts, such that the lifetime works on an alloca.
5865	Ptr = Ptr->stripPointerCasts();
5866	if (isa<AllocaInst>(Val: Ptr)) {
5867	// Don't use NewFn, as we might have looked through an addrspacecast.
5868	if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
5869	NewCall = Builder.CreateLifetimeStart(Ptr);
5870	else
5871	NewCall = Builder.CreateLifetimeEnd(Ptr);
5872	break;
5873	}
5874
5875	// Otherwise remove the lifetime marker.
5876	CI->eraseFromParent();
5877	return;
5878	}
5879
5880	case Intrinsic::x86_avx512_vpdpbusd_128:
5881	case Intrinsic::x86_avx512_vpdpbusd_256:
5882	case Intrinsic::x86_avx512_vpdpbusd_512:
5883	case Intrinsic::x86_avx512_vpdpbusds_128:
5884	case Intrinsic::x86_avx512_vpdpbusds_256:
5885	case Intrinsic::x86_avx512_vpdpbusds_512:
5886	case Intrinsic::x86_avx2_vpdpbssd_128:
5887	case Intrinsic::x86_avx2_vpdpbssd_256:
5888	case Intrinsic::x86_avx10_vpdpbssd_512:
5889	case Intrinsic::x86_avx2_vpdpbssds_128:
5890	case Intrinsic::x86_avx2_vpdpbssds_256:
5891	case Intrinsic::x86_avx10_vpdpbssds_512:
5892	case Intrinsic::x86_avx2_vpdpbsud_128:
5893	case Intrinsic::x86_avx2_vpdpbsud_256:
5894	case Intrinsic::x86_avx10_vpdpbsud_512:
5895	case Intrinsic::x86_avx2_vpdpbsuds_128:
5896	case Intrinsic::x86_avx2_vpdpbsuds_256:
5897	case Intrinsic::x86_avx10_vpdpbsuds_512:
5898	case Intrinsic::x86_avx2_vpdpbuud_128:
5899	case Intrinsic::x86_avx2_vpdpbuud_256:
5900	case Intrinsic::x86_avx10_vpdpbuud_512:
5901	case Intrinsic::x86_avx2_vpdpbuuds_128:
5902	case Intrinsic::x86_avx2_vpdpbuuds_256:
5903	case Intrinsic::x86_avx10_vpdpbuuds_512: {
5904	unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / `8`;
5905	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
5906	CI->getArgOperand(i: `2`)};
5907	Type NewArgType = VectorType::get(ElementType: Builder.getInt8Ty(), NumElements: NumElts, Scalable: false*);
5908	Args[`1`] = Builder.CreateBitCast(V: Args[`1`], DestTy: NewArgType);
5909	Args[`2`] = Builder.CreateBitCast(V: Args[`2`], DestTy: NewArgType);
5910
5911	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5912	break;
5913	}
5914	case Intrinsic::x86_avx512_vpdpwssd_128:
5915	case Intrinsic::x86_avx512_vpdpwssd_256:
5916	case Intrinsic::x86_avx512_vpdpwssd_512:
5917	case Intrinsic::x86_avx512_vpdpwssds_128:
5918	case Intrinsic::x86_avx512_vpdpwssds_256:
5919	case Intrinsic::x86_avx512_vpdpwssds_512:
5920	case Intrinsic::x86_avx2_vpdpwsud_128:
5921	case Intrinsic::x86_avx2_vpdpwsud_256:
5922	case Intrinsic::x86_avx10_vpdpwsud_512:
5923	case Intrinsic::x86_avx2_vpdpwsuds_128:
5924	case Intrinsic::x86_avx2_vpdpwsuds_256:
5925	case Intrinsic::x86_avx10_vpdpwsuds_512:
5926	case Intrinsic::x86_avx2_vpdpwusd_128:
5927	case Intrinsic::x86_avx2_vpdpwusd_256:
5928	case Intrinsic::x86_avx10_vpdpwusd_512:
5929	case Intrinsic::x86_avx2_vpdpwusds_128:
5930	case Intrinsic::x86_avx2_vpdpwusds_256:
5931	case Intrinsic::x86_avx10_vpdpwusds_512:
5932	case Intrinsic::x86_avx2_vpdpwuud_128:
5933	case Intrinsic::x86_avx2_vpdpwuud_256:
5934	case Intrinsic::x86_avx10_vpdpwuud_512:
5935	case Intrinsic::x86_avx2_vpdpwuuds_128:
5936	case Intrinsic::x86_avx2_vpdpwuuds_256:
5937	case Intrinsic::x86_avx10_vpdpwuuds_512:
5938	unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / `16`;
5939	Value *Args[] = {CI->getArgOperand(i: `0`), CI->getArgOperand(i: `1`),
5940	CI->getArgOperand(i: `2`)};
5941	Type NewArgType = VectorType::get(ElementType: Builder.getInt16Ty(), NumElements: NumElts, Scalable: false*);
5942	Args[`1`] = Builder.CreateBitCast(V: Args[`1`], DestTy: NewArgType);
5943	Args[`2`] = Builder.CreateBitCast(V: Args[`2`], DestTy: NewArgType);
5944
5945	NewCall = Builder.CreateCall(Callee: NewFn, Args);
5946	break;
5947	}
5948	assert(NewCall && "Should have either set this variable or returned through "
5949	"the default case");
5950	NewCall->takeName(V: CI);
5951	CI->replaceAllUsesWith(V: NewCall);
5952	CI->eraseFromParent();
5953	}
5954
5955	void llvm::UpgradeCallsToIntrinsic(Function *F) {
5956	assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
5957
5958	// Check if this function should be upgraded and get the replacement function
5959	// if there is one.
5960	Function *NewFn;
5961	if (UpgradeIntrinsicFunction(F, NewFn)) {
5962	// Replace all users of the old function with the new function or new
5963	// instructions. This is not a range loop because the call is deleted.
5964	for (User *U : make_early_inc_range(Range: F->users()))
5965	if (CallBase *CB = dyn_cast<CallBase>(Val: U))
5966	UpgradeIntrinsicCall(CI: CB, NewFn);
5967
5968	// Remove old function, no longer used, from the module.
5969	if (F != NewFn)
5970	F->eraseFromParent();
5971	}
5972	}
5973
5974	MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
5975	const unsigned NumOperands = MD.getNumOperands();
5976	if (NumOperands == `0`)
5977	return &MD; // Invalid, punt to a verifier error.
5978
5979	// Check if the tag uses struct-path aware TBAA format.
5980	if (isa<MDNode>(Val: MD.getOperand(I: `0`)) && NumOperands >= `3`)
5981	return &MD;
5982
5983	auto &Context = MD.getContext();
5984	if (NumOperands == `3`) {
5985	Metadata *Elts[] = {MD.getOperand(I: `0`), MD.getOperand(I: `1`)};
5986	MDNode *ScalarType = MDNode::get(Context, MDs: Elts);
5987	// Create a MDNode <ScalarType, ScalarType, offset 0, const>
5988	Metadata *Elts2[] = {ScalarType, ScalarType,
5989	ConstantAsMetadata::get(
5990	C: Constant::getNullValue(Ty: Type::getInt64Ty(C&: Context))),
5991	MD.getOperand(I: `2`)};
5992	return MDNode::get(Context, MDs: Elts2);
5993	}
5994	// Create a MDNode <MD, MD, offset 0>
5995	Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(C: Constant::getNullValue(
5996	Ty: Type::getInt64Ty(C&: Context)))};
5997	return MDNode::get(Context, MDs: Elts);
5998	}
5999
6000	Instruction llvm::UpgradeBitCastInst(unsigned* Opc, Value V, Type DestTy,
6001	Instruction *&Temp) {
6002	if (Opc != Instruction::BitCast)
6003	return nullptr;
6004
6005	Temp = nullptr;
6006	Type *SrcTy = V->getType();
6007	if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6008	SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6009	LLVMContext &Context = V->getContext();
6010
6011	// We have no information about target data layout, so we assume that
6012	// the maximum pointer size is 64bit.
6013	Type *MidTy = Type::getInt64Ty(C&: Context);
6014	Temp = CastInst::Create(Instruction::PtrToInt, S: V, Ty: MidTy);
6015
6016	return CastInst::Create(Instruction::IntToPtr, S: Temp, Ty: DestTy);
6017	}
6018
6019	return nullptr;
6020	}
6021
6022	Constant llvm::UpgradeBitCastExpr(unsigned* Opc, Constant C, Type DestTy) {
6023	if (Opc != Instruction::BitCast)
6024	return nullptr;
6025
6026	Type *SrcTy = C->getType();
6027	if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6028	SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6029	LLVMContext &Context = C->getContext();
6030
6031	// We have no information about target data layout, so we assume that
6032	// the maximum pointer size is 64bit.
6033	Type *MidTy = Type::getInt64Ty(C&: Context);
6034
6035	return ConstantExpr::getIntToPtr(C: ConstantExpr::getPtrToInt(C, Ty: MidTy),
6036	Ty: DestTy);
6037	}
6038
6039	return nullptr;
6040	}
6041
6042	static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6043	if (Flag.getNumOperands() < `3`)
6044	return std::nullopt;
6045	if (MDString *Name = dyn_cast_or_null<MDString>(Val: Flag.getOperand(I: `1`)))
6046	return Name->getString();
6047	return std::nullopt;
6048	}
6049
6050	/// Check the debug info version number, if it is out-dated, drop the debug
6051	/// info. Return true if module is modified.
6052	bool llvm::UpgradeDebugInfo(Module &M) {
6053	if (DisableAutoUpgradeDebugInfo)
6054	return false;
6055
6056	llvm::TimeTraceScope timeScope("Upgrade debug info");
6057	// We need to get metadata before the module is verified (i.e., getModuleFlag
6058	// makes assumptions that we haven't verified yet). Carefully extract the flag
6059	// from the metadata.
6060	unsigned Version = `0`;
6061	if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6062	auto OpIt = find_if(Range: ModFlags->operands(), P: [](const MDNode *Flag) {
6063	if (auto Name = getModuleFlagNameSafely(Flag: *Flag))
6064	return *Name == "Debug Info Version";
6065	return false;
6066	});
6067	if (OpIt != ModFlags->op_end()) {
6068	const MDOperand &ValOp = (*OpIt)->getOperand(I: `2`);
6069	if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(MD: ValOp))
6070	Version = CI->getZExtValue();
6071	}
6072	}
6073
6074	if (Version == DEBUG_METADATA_VERSION) {
6075	bool BrokenDebugInfo = false;
6076	if (verifyModule(M, OS: &llvm::errs(), BrokenDebugInfo: &BrokenDebugInfo))
6077	report_fatal_error(reason: "Broken module found, compilation aborted!");
6078	if (!BrokenDebugInfo)
6079	// Everything is ok.
6080	return false;
6081	else {
6082	// Diagnose malformed debug info.
6083	DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6084	M.getContext().diagnose(DI: Diag);
6085	}
6086	}
6087	bool Modified = StripDebugInfo(M);
6088	if (Modified && Version != DEBUG_METADATA_VERSION) {
6089	// Diagnose a version mismatch.
6090	DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6091	M.getContext().diagnose(DI: DiagVersion);
6092	}
6093	return Modified;
6094	}
6095
6096	static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6097	GlobalValue GV, const* Metadata *V) {
6098	Function *F = cast<Function>(Val: GV);
6099
6100	constexpr StringLiteral DefaultValue = "1";
6101	StringRef Vect3[`3`] = {DefaultValue, DefaultValue, DefaultValue};
6102	unsigned Length = `0`;
6103
6104	if (F->hasFnAttribute(Kind: Attr)) {
6105	// We expect the existing attribute to have the form "x[,y[,z]]". Here we
6106	// parse these elements placing them into Vect3
6107	StringRef S = F->getFnAttribute(Kind: Attr).getValueAsString();
6108	for (; Length < `3` && !S.empty(); Length++) {
6109	auto [Part, Rest] = S.split(Separator: `','`);
6110	Vect3[Length] = Part.trim();
6111	S = Rest;
6112	}
6113	}
6114
6115	const unsigned Dim = DimC - `'x'`;
6116	assert(Dim < `3` && "Unexpected dim char");
6117
6118	const uint64_t VInt = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6119
6120	// local variable required for StringRef in Vect3 to point to.
6121	const std::string VStr = llvm::utostr(X: VInt);
6122	Vect3[Dim] = VStr;
6123	Length = std::max(a: Length, b: Dim + `1`);
6124
6125	const std::string NewAttr = llvm::join(R: ArrayRef(Vect3, Length), Separator: ",");
6126	F->addFnAttr(Kind: Attr, Val: NewAttr);
6127	}
6128
6129	static inline bool isXYZ(StringRef S) {
6130	return S == "x" \|\| S == "y" \|\| S == "z";
6131	}
6132
6133	bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6134	const Metadata *V) {
6135	if (K == "kernel") {
6136	if (!mdconst::extract<ConstantInt>(MD&: V)->isZero())
6137	cast<Function>(Val: GV)->setCallingConv(CallingConv::PTX_Kernel);
6138	return true;
6139	}
6140	if (K == "align") {
6141	// V is a bitfeild specifying two 16-bit values. The alignment value is
6142	// specfied in low 16-bits, The index is specified in the high bits. For the
6143	// index, 0 indicates the return value while higher values correspond to
6144	// each parameter (idx = param + 1).
6145	const uint64_t AlignIdxValuePair =
6146	mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6147	const unsigned Idx = (AlignIdxValuePair >> `16`);
6148	const Align StackAlign = Align (AlignIdxValuePair & `0xFFFF`);
6149	cast<Function>(Val: GV)->addAttributeAtIndex(
6150	i: Idx, Attr: Attribute::getWithStackAlignment(Context&: GV->getContext(), Alignment: StackAlign));
6151	return true;
6152	}
6153	if (K == "maxclusterrank" \|\| K == "cluster_max_blocks") {
6154	const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6155	cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: CV));
6156	return true;
6157	}
6158	if (K == "minctasm") {
6159	const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6160	cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MinCTASm, Val: llvm::utostr(X: CV));
6161	return true;
6162	}
6163	if (K == "maxnreg") {
6164	const auto CV = mdconst::extract<ConstantInt>(MD&: V)->getZExtValue();
6165	cast<Function>(Val: GV)->addFnAttr(Kind: NVVMAttr::MaxNReg, Val: llvm::utostr(X: CV));
6166	return true;
6167	}
6168	if (K.consume_front(Prefix: "maxntid") && isXYZ(S: K)) {
6169	upgradeNVVMFnVectorAttr(Attr: NVVMAttr::MaxNTID, DimC: K [`0`], GV, V);
6170	return true;
6171	}
6172	if (K.consume_front(Prefix: "reqntid") && isXYZ(S: K)) {
6173	upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ReqNTID, DimC: K [`0`], GV, V);
6174	return true;
6175	}
6176	if (K.consume_front(Prefix: "cluster_dim_") && isXYZ(S: K)) {
6177	upgradeNVVMFnVectorAttr(Attr: NVVMAttr::ClusterDim, DimC: K [`0`], GV, V);
6178	return true;
6179	}
6180	if (K == "grid_constant") {
6181	const auto Attr = Attribute::get(Context&: GV->getContext(), Kind: NVVMAttr::GridConstant);
6182	for (const auto &Op : cast<MDNode>(Val: V)->operands()) {
6183	// For some reason, the index is 1-based in the metadata. Good thing we're
6184	// able to auto-upgrade it!
6185	const auto Index = mdconst::extract<ConstantInt>(MD: Op)->getZExtValue() - `1`;
6186	cast<Function>(Val: GV)->addParamAttr(ArgNo: Index, Attr);
6187	}
6188	return true;
6189	}
6190
6191	return false;
6192	}
6193
6194	void llvm::UpgradeNVVMAnnotations(Module &M) {
6195	NamedMDNode *NamedMD = M.getNamedMetadata(Name: "nvvm.annotations");
6196	if (!NamedMD)
6197	return;
6198
6199	SmallVector<MDNode *, `8`> NewNodes;
6200	SmallPtrSet<const MDNode *, `8`> SeenNodes;
6201	for (MDNode *MD : NamedMD->operands()) {
6202	if (!SeenNodes.insert(Ptr: MD).second)
6203	continue;
6204
6205	auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD: MD->getOperand(I: `0`));
6206	if (!GV)
6207	continue;
6208
6209	assert((MD->getNumOperands() % `2`) == `1` && "Invalid number of operands");
6210
6211	SmallVector<Metadata *, `8`> NewOperands{MD->getOperand(I: `0`)};
6212	// Each nvvm.annotations metadata entry will be of the following form:
6213	// !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6214	// start index = 1, to skip the global variable key
6215	// increment = 2, to skip the value for each property-value pairs
6216	for (unsigned j = `1`, je = MD->getNumOperands(); j < je; j += `2`) {
6217	MDString *K = cast<MDString>(Val: MD->getOperand(I: j));
6218	const MDOperand &V = MD->getOperand(I: j + `1`);
6219	bool Upgraded = upgradeSingleNVVMAnnotation(GV, K: K->getString(), V);
6220	if (!Upgraded)
6221	NewOperands.append(IL: {K, V});
6222	}
6223
6224	if (NewOperands.size() > `1`)
6225	NewNodes.push_back(Elt: MDNode::get(Context&: M.getContext(), MDs: NewOperands));
6226	}
6227
6228	NamedMD->clearOperands();
6229	for (MDNode *N : NewNodes)
6230	NamedMD->addOperand(M: N);
6231	}
6232
6233	/// This checks for objc retain release marker which should be upgraded. It
6234	/// returns true if module is modified.
6235	static bool upgradeRetainReleaseMarker(Module &M) {
6236	bool Changed = false;
6237	const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6238	NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(Name: MarkerKey);
6239	if (ModRetainReleaseMarker) {
6240	MDNode *Op = ModRetainReleaseMarker->getOperand(i: `0`);
6241	if (Op) {
6242	MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: `0`));
6243	if (ID) {
6244	SmallVector<StringRef, `4`> ValueComp;
6245	ID->getString().split(A&: ValueComp, Separator: "#");
6246	if (ValueComp.size() == `2`) {
6247	std::string NewValue = ValueComp [`0`].str() + ";" + ValueComp [`1`].str();
6248	ID = MDString::get(Context&: M.getContext(), Str: NewValue);
6249	}
6250	M.addModuleFlag(Behavior: Module::Error, Key: MarkerKey, Val: ID);
6251	M.eraseNamedMetadata(NMD: ModRetainReleaseMarker);
6252	Changed = true;
6253	}
6254	}
6255	}
6256	return Changed;
6257	}
6258
6259	void llvm::UpgradeARCRuntime(Module &M) {
6260	// This lambda converts normal function calls to ARC runtime functions to
6261	// intrinsic calls.
6262	auto UpgradeToIntrinsic = [&](const char *OldFunc,
6263	llvm::Intrinsic::ID IntrinsicFunc) {
6264	Function *Fn = M.getFunction(Name: OldFunc);
6265
6266	if (!Fn)
6267	return;
6268
6269	Function *NewFn =
6270	llvm::Intrinsic::getOrInsertDeclaration(M: &M, id: IntrinsicFunc);
6271
6272	for (User *U : make_early_inc_range(Range: Fn->users())) {
6273	CallInst *CI = dyn_cast<CallInst>(Val: U);
6274	if (!CI \|\| CI->getCalledFunction() != Fn)
6275	continue;
6276
6277	IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6278	FunctionType *NewFuncTy = NewFn->getFunctionType();
6279	SmallVector<Value *, `2`> Args;
6280
6281	// Don't upgrade the intrinsic if it's not valid to bitcast the return
6282	// value to the return type of the old function.
6283	if (NewFuncTy->getReturnType() != CI->getType() &&
6284	!CastInst::castIsValid(op: Instruction::BitCast, S: CI,
6285	DstTy: NewFuncTy->getReturnType()))
6286	continue;
6287
6288	bool InvalidCast = false;
6289
6290	for (unsigned I = `0`, E = CI->arg_size(); I != E; ++I) {
6291	Value *Arg = CI->getArgOperand(i: I);
6292
6293	// Bitcast argument to the parameter type of the new function if it's
6294	// not a variadic argument.
6295	if (I < NewFuncTy->getNumParams()) {
6296	// Don't upgrade the intrinsic if it's not valid to bitcast the argument
6297	// to the parameter type of the new function.
6298	if (!CastInst::castIsValid(op: Instruction::BitCast, S: Arg,
6299	DstTy: NewFuncTy->getParamType(i: I))) {
6300	InvalidCast = true;
6301	break;
6302	}
6303	Arg = Builder.CreateBitCast(V: Arg, DestTy: NewFuncTy->getParamType(i: I));
6304	}
6305	Args.push_back(Elt: Arg);
6306	}
6307
6308	if (InvalidCast)
6309	continue;
6310
6311	// Create a call instruction that calls the new function.
6312	CallInst *NewCall = Builder.CreateCall(FTy: NewFuncTy, Callee: NewFn, Args);
6313	NewCall->setTailCallKind(cast<CallInst>(Val: CI)->getTailCallKind());
6314	NewCall->takeName(V: CI);
6315
6316	// Bitcast the return value back to the type of the old call.
6317	Value *NewRetVal = Builder.CreateBitCast(V: NewCall, DestTy: CI->getType());
6318
6319	if (!CI->use_empty())
6320	CI->replaceAllUsesWith(V: NewRetVal);
6321	CI->eraseFromParent();
6322	}
6323
6324	if (Fn->use_empty())
6325	Fn->eraseFromParent();
6326	};
6327
6328	// Unconditionally convert a call to "clang.arc.use" to a call to
6329	// "llvm.objc.clang.arc.use".
6330	UpgradeToIntrinsic ("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6331
6332	// Upgrade the retain release marker. If there is no need to upgrade
6333	// the marker, that means either the module is already new enough to contain
6334	// new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6335	if (!upgradeRetainReleaseMarker(M))
6336	return;
6337
6338	std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6339	{"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6340	{"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6341	{"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6342	{"objc_autoreleaseReturnValue",
6343	llvm::Intrinsic::objc_autoreleaseReturnValue},
6344	{"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6345	{"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6346	{"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6347	{"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6348	{"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6349	{"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6350	{"objc_release", llvm::Intrinsic::objc_release},
6351	{"objc_retain", llvm::Intrinsic::objc_retain},
6352	{"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6353	{"objc_retainAutoreleaseReturnValue",
6354	llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6355	{"objc_retainAutoreleasedReturnValue",
6356	llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6357	{"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6358	{"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6359	{"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6360	{"objc_unsafeClaimAutoreleasedReturnValue",
6361	llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6362	{"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6363	{"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6364	{"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6365	{"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6366	{"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6367	{"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6368	{"objc_arc_annotation_topdown_bbstart",
6369	llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6370	{"objc_arc_annotation_topdown_bbend",
6371	llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6372	{"objc_arc_annotation_bottomup_bbstart",
6373	llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6374	{"objc_arc_annotation_bottomup_bbend",
6375	llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6376
6377	for (auto &I : RuntimeFuncs)
6378	UpgradeToIntrinsic (I.first, I.second);
6379	}
6380
6381	// Upgrade the way signing of pointers to init/fini functions is described.
6382	//
6383	// Originally, the `@llvm.global_(ctors\|dtors)` arrays contained `ptrauth`
6384	// constants, if signing was requested. After the upgrade, these arrays contain
6385	// plain function pointers and the desired signing schema is described via a
6386	// pair of module flags.
6387	//
6388	// Note that the upgrade is only performed if all elements of both* arrays*
6389	// agree on a common signing schema.
6390	static bool upgradePtrauthInitFiniArrays(Module &M) {
6391	// As we cannot always decide whether the particular module should have
6392	// ptrauth-init-fini flags, we have to treat absent flags as having zero
6393	// values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6394	// returns as soon as it spots any non-signed init/fini pointer: either we
6395	// should request non-signed pointers (safe to omit both flags) or there is
6396	// no common schema (and thus we do not modify anything).
6397	//
6398	// UseAddressDisc's value either represents "not decided yet" state (nullopt)
6399	// or whether we should request address diversity in addition to the basic
6400	// constant diversity. There is no value representing "decided not to sign"
6401	// for the reasons explained above.
6402	std::optional<bool> UseAddressDisc;
6403
6404	// Do not attempt upgrading if the new module flags already exist.
6405	if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6406	for (const MDNode *Flag : ModFlags->operands()) {
6407	std::optional<StringRef> Name = getModuleFlagNameSafely(Flag: *Flag);
6408	if (Name && (*Name == "ptrauth-init-fini" \|\|
6409	*Name == "ptrauth-init-fini-address-discrimination"))
6410	return false;
6411	}
6412	}
6413
6414	auto UpgradeSinglePointer = [&UseAddressDisc](Constant CV) -> Constant {
6415	constexpr unsigned ExpectedConstDisc = `0xD9D4`;
6416	constexpr unsigned ExpectedAddressMarker = `1`;
6417
6418	auto *CPA = dyn_cast<ConstantPtrAuth>(Val: CV);
6419	if (!CPA \|\| !CPA->getDiscriminator()->equalsInt(V: ExpectedConstDisc))
6420	return nullptr; // Nothing to upgrade or unknown pattern found.
6421
6422	bool HasAddressDisc;
6423	if (!CPA->hasAddressDiscriminator())
6424	HasAddressDisc = false;
6425	else if (CPA->hasSpecialAddressDiscriminator(Value: ExpectedAddressMarker))
6426	HasAddressDisc = true;
6427	else
6428	return nullptr; // Unknown pattern.
6429
6430	if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
6431	return nullptr; // Disagreement with the decided mode.
6432
6433	UseAddressDisc = HasAddressDisc;
6434	return CPA->getPointer();
6435	};
6436
6437	// Do not apply any changes until we know the upgrade is non-ambiguous.
6438	using PendingUpgrade = std::pair<GlobalVariable , Constant >;
6439	SmallVector<PendingUpgrade, `2`> GlobalArraysToUpgrade;
6440
6441	for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
6442	auto *GV = dyn_cast_if_present<GlobalVariable>(Val: M.getNamedValue(Name));
6443	if (!GV \|\| !GV->hasInitializer())
6444	continue; // Skip, but it is okay to upgrade the other variable.
6445
6446	auto *OldStructorsArray = dyn_cast<ConstantArray>(Val: GV->getInitializer());
6447	if (!OldStructorsArray \|\| OldStructorsArray->getNumOperands() == `0`)
6448	return false;
6449
6450	std::vector<Constant *> NewStructors;
6451	NewStructors.reserve(n: OldStructorsArray->getNumOperands());
6452
6453	for (Use &U : OldStructorsArray->operands()) {
6454	ConstantStruct *Structor = dyn_cast<ConstantStruct>(Val: U.get());
6455	if (!Structor \|\| Structor->getNumOperands() != `3`)
6456	return false;
6457
6458	Constant *Prio = Structor->getOperand(i_nocapture: `0`);
6459	Constant *Func = Structor->getOperand(i_nocapture: `1`);
6460	Constant *Arg = Structor->getOperand(i_nocapture: `2`);
6461
6462	Func = UpgradeSinglePointer (Func);
6463	if (!Func)
6464	return false;
6465
6466	NewStructors.push_back(
6467	x: ConstantStruct::get(T: Structor->getType(), V: {Prio, Func, Arg}));
6468	}
6469
6470	Constant *NewInit =
6471	ConstantArray::get(T: OldStructorsArray->getType(), V: NewStructors);
6472	GlobalArraysToUpgrade.emplace_back(Args&: GV, Args&: NewInit);
6473	}
6474
6475	if (GlobalArraysToUpgrade.empty())
6476	return false;
6477	assert(UseAddressDisc.has_value());
6478
6479	for (auto [GV, NewInit] : GlobalArraysToUpgrade)
6480	GV->setInitializer(NewInit);
6481
6482	M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini", Val: `1`);
6483	M.addModuleFlag(Behavior: Module::Error, Key: "ptrauth-init-fini-address-discrimination",
6484	Val: *UseAddressDisc);
6485
6486	return true;
6487	}
6488
6489	bool llvm::UpgradeModuleFlags(Module &M) {
6490	bool Changed = false;
6491	Changed \|= upgradePtrauthInitFiniArrays(M);
6492
6493	NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6494	if (!ModFlags)
6495	return Changed;
6496
6497	bool HasObjCFlag = false, HasClassProperties = false;
6498	bool HasSwiftVersionFlag = false;
6499	uint8_t SwiftMajorVersion, SwiftMinorVersion;
6500	uint32_t SwiftABIVersion;
6501	auto Int8Ty = Type::getInt8Ty(C&: M.getContext());
6502	auto Int32Ty = Type::getInt32Ty(C&: M.getContext());
6503
6504	for (unsigned I = `0`, E = ModFlags->getNumOperands(); I != E; ++I) {
6505	MDNode *Op = ModFlags->getOperand(i: I);
6506	if (Op->getNumOperands() != `3`)
6507	continue;
6508	MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: `1`));
6509	if (!ID)
6510	continue;
6511	auto SetBehavior = [&](Module::ModFlagBehavior B) {
6512	Metadata *Ops[`3`] = {ConstantAsMetadata::get(C: ConstantInt::get(
6513	Ty: Type::getInt32Ty(C&: M.getContext()), V: B)),
6514	MDString::get(Context&: M.getContext(), Str: ID->getString()),
6515	Op->getOperand(I: `2`)};
6516	ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
6517	Changed = true;
6518	};
6519
6520	if (ID->getString() == "Objective-C Image Info Version")
6521	HasObjCFlag = true;
6522	if (ID->getString() == "Objective-C Class Properties")
6523	HasClassProperties = true;
6524	// Upgrade PIC from Error/Max to Min.
6525	if (ID->getString() == "PIC Level") {
6526	if (auto *Behavior =
6527	mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: `0`))) {
6528	uint64_t V = Behavior->getLimitedValue();
6529	if (V == Module::Error \|\| V == Module::Max)
6530	SetBehavior (Module::Min);
6531	}
6532	}
6533	// Upgrade "PIE Level" from Error to Max.
6534	if (ID->getString() == "PIE Level")
6535	if (auto *Behavior =
6536	mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: `0`)))
6537	if (Behavior->getLimitedValue() == Module::Error)
6538	SetBehavior (Module::Max);
6539
6540	// Upgrade branch protection and return address signing module flags. The
6541	// module flag behavior for these fields were Error and now they are Min.
6542	if (ID->getString() == "branch-target-enforcement" \|\|
6543	ID->getString().starts_with(Prefix: "sign-return-address")) {
6544	if (auto *Behavior =
6545	mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: `0`))) {
6546	if (Behavior->getLimitedValue() == Module::Error) {
6547	Type *Int32Ty = Type::getInt32Ty(C&: M.getContext());
6548	Metadata *Ops[`3`] = {
6549	ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Module::Min)),
6550	Op->getOperand(I: `1`), Op->getOperand(I: `2`)};
6551	ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
6552	Changed = true;
6553	}
6554	}
6555	}
6556
6557	// Upgrade Objective-C Image Info Section. Removed the whitespce in the
6558	// section name so that llvm-lto will not complain about mismatching
6559	// module flags that is functionally the same.
6560	if (ID->getString() == "Objective-C Image Info Section") {
6561	if (auto *Value = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: `2`))) {
6562	SmallVector<StringRef, `4`> ValueComp;
6563	Value->getString().split(A&: ValueComp, Separator: " ");
6564	if (ValueComp.size() != `1`) {
6565	std::string NewValue;
6566	for (auto &S : ValueComp)
6567	NewValue += S.str();
6568	Metadata *Ops[`3`] = {Op->getOperand(I: `0`), Op->getOperand(I: `1`),
6569	MDString::get(Context&: M.getContext(), Str: NewValue)};
6570	ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
6571	Changed = true;
6572	}
6573	}
6574	}
6575
6576	// IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
6577	// If the higher bits are set, it adds new module flag for swift info.
6578	if (ID->getString() == "Objective-C Garbage Collection") {
6579	auto Md = dyn_cast<ConstantAsMetadata>(Val: Op->getOperand(I: `2`));
6580	if (Md) {
6581	assert(Md->getValue() && "Expected non-empty metadata");
6582	auto Type = Md->getValue()->getType();
6583	if (Type == Int8Ty)
6584	continue;
6585	unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
6586	if ((Val & `0xff`) != Val) {
6587	HasSwiftVersionFlag = true;
6588	SwiftABIVersion = (Val & `0xff00`) >> `8`;
6589	SwiftMajorVersion = (Val & `0xff000000`) >> `24`;
6590	SwiftMinorVersion = (Val & `0xff0000`) >> `16`;
6591	}
6592	Metadata *Ops[`3`] = {
6593	ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty,V: Module::Error)),
6594	Op->getOperand(I: `1`),
6595	ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int8Ty,V: Val & `0xff`))};
6596	ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
6597	Changed = true;
6598	}
6599	}
6600
6601	if (ID->getString() == "amdgpu_code_object_version") {
6602	Metadata *Ops[`3`] = {
6603	Op->getOperand(I: `0`),
6604	MDString::get(Context&: M.getContext(), Str: "amdhsa_code_object_version"),
6605	Op->getOperand(I: `2`)};
6606	ModFlags->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Ops));
6607	Changed = true;
6608	}
6609	}
6610
6611	// "Objective-C Class Properties" is recently added for Objective-C. We
6612	// upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
6613	// flag of value 0, so we can correclty downgrade this flag when trying to
6614	// link an ObjC bitcode without this module flag with an ObjC bitcode with
6615	// this module flag.
6616	if (HasObjCFlag && !HasClassProperties) {
6617	M.addModuleFlag(Behavior: llvm::Module::Override, Key: "Objective-C Class Properties",
6618	Val: (uint32_t)`0`);
6619	Changed = true;
6620	}
6621
6622	if (HasSwiftVersionFlag) {
6623	M.addModuleFlag(Behavior: Module::Error, Key: "Swift ABI Version",
6624	Val: SwiftABIVersion);
6625	M.addModuleFlag(Behavior: Module::Error, Key: "Swift Major Version",
6626	Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMajorVersion));
6627	M.addModuleFlag(Behavior: Module::Error, Key: "Swift Minor Version",
6628	Val: ConstantInt::get(Ty: Int8Ty, V: SwiftMinorVersion));
6629	Changed = true;
6630	}
6631
6632	return Changed;
6633	}
6634
6635	bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
6636	NamedMDNode *CFIConsts = M.getNamedMetadata(Name: "cfi.functions");
6637	// If this metadata has operands, we expect all of them to be either from
6638	// before or from after the format change handled here, so we can bail out
6639	// fast if the first (if any) operands is of the new format.
6640	auto MatchesVersion = [](const MDNode *Op) {
6641	return Op->getNumOperands() >= `3` &&
6642	isa<ConstantAsMetadata>(Val: Op->getOperand(I: `2`)) &&
6643	cast<ConstantAsMetadata>(Val: Op->getOperand(I: `2`))
6644	->getType()
6645	->isIntegerTy(BitWidth: `64`);
6646	};
6647
6648	if (!CFIConsts \|\| !CFIConsts->getNumOperands() \|\|
6649	MatchesVersion (CFIConsts->getOperand(i: `0`)))
6650	return false;
6651
6652	bool Changed = false;
6653	for (unsigned I = `0`, E = CFIConsts->getNumOperands(); I != E; ++I) {
6654	MDNode *Op = CFIConsts->getOperand(i: I);
6655	assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
6656	assert(Op->getNumOperands() >= `2` &&
6657	"Expected at least 2 operands - name and linkage type");
6658	MDString *NameMD = dyn_cast<MDString>(Val: Op->getOperand(I: `0`));
6659	StringRef Name = NameMD->getString();
6660	GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
6661	GlobalName: GlobalValue::dropLLVMManglingEscape(Name));
6662
6663	SmallVector<Metadata *, `4`> Elts;
6664	Elts.push_back(Elt: Op->getOperand(I: `0`));
6665	Elts.push_back(Elt: Op->getOperand(I: `1`));
6666	Elts.push_back(Elt: ConstantAsMetadata::get(
6667	C: ConstantInt::get(Ty: Type::getInt64Ty(C&: M.getContext()), V: GUID)));
6668
6669	for (unsigned J = `2`, EJ = Op->getNumOperands(); J != EJ; ++J)
6670	Elts.push_back(Elt: Op->getOperand(I: J));
6671
6672	CFIConsts->setOperand(I, New: MDNode::get(Context&: M.getContext(), MDs: Elts));
6673	Changed = true;
6674	}
6675
6676	return Changed;
6677	}
6678
6679	void llvm::UpgradeSectionAttributes(Module &M) {
6680	auto TrimSpaces = [](StringRef Section) -> std::string {
6681	SmallVector<StringRef, `5`> Components;
6682	Section.split(A&: Components, Separator: `','`);
6683
6684	SmallString<`32`> Buffer;
6685	raw_svector_ostream OS(Buffer);
6686
6687	for (auto Component : Components)
6688	OS << `','` << Component.trim();
6689
6690	return std::string (OS.str().substr(Start: `1`));
6691	};
6692
6693	for (auto &GV : M.globals()) {
6694	if (!GV.hasSection())
6695	continue;
6696
6697	StringRef Section = GV.getSection();
6698
6699	if (!Section.starts_with(Prefix: "__DATA, __objc_catlist"))
6700	continue;
6701
6702	// __DATA, __objc_catlist, regular, no_dead_strip
6703	// __DATA,__objc_catlist,regular,no_dead_strip
6704	GV.setSection(TrimSpaces (Section));
6705	}
6706	}
6707
6708	namespace {
6709	// Prior to LLVM 10.0, the strictfp attribute could be used on individual
6710	// callsites within a function that did not also have the strictfp attribute.
6711	// Since 10.0, if strict FP semantics are needed within a function, the
6712	// function must have the strictfp attribute and all calls within the function
6713	// must also have the strictfp attribute. This latter restriction is
6714	// necessary to prevent unwanted libcall simplification when a function is
6715	// being cloned (such as for inlining).
6716	//
6717	// The "dangling" strictfp attribute usage was only used to prevent constant
6718	// folding and other libcall simplification. The nobuiltin attribute on the
6719	// callsite has the same effect.
6720	struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
6721	StrictFPUpgradeVisitor() = default;
6722
6723	void visitCallBase(CallBase &Call) {
6724	if (!Call.isStrictFP())
6725	return;
6726	if (isa<ConstrainedFPIntrinsic>(Val: &Call))
6727	return;
6728	// If we get here, the caller doesn't have the strictfp attribute
6729	// but this callsite does. Replace the strictfp attribute with nobuiltin.
6730	Call.removeFnAttr(Kind: Attribute::StrictFP);
6731	Call.addFnAttr(Kind: Attribute::NoBuiltin);
6732	}
6733	};
6734
6735	/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
6736	struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
6737	: public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
6738	AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
6739
6740	void visitAtomicRMWInst(AtomicRMWInst &RMW) {
6741	if (!RMW.isFloatingPointOperation())
6742	return;
6743
6744	MDNode *Empty = MDNode::get(Context&: RMW.getContext(), MDs: {});
6745	RMW.setMetadata(Kind: "amdgpu.no.fine.grained.host.memory", Node: Empty);
6746	RMW.setMetadata(Kind: "amdgpu.no.remote.memory.access", Node: Empty);
6747	RMW.setMetadata(Kind: "amdgpu.ignore.denormal.mode", Node: Empty);
6748	}
6749	};
6750	} // namespace
6751
6752	void llvm::UpgradeFunctionAttributes(Function &F) {
6753	// If a function definition doesn't have the strictfp attribute,
6754	// convert any callsite strictfp attributes to nobuiltin.
6755	if (!F.isDeclaration() && !F.hasFnAttribute(Kind: Attribute::StrictFP)) {
6756	StrictFPUpgradeVisitor SFPV;
6757	SFPV.visit(F);
6758	}
6759
6760	// Remove all incompatibile attributes from function.
6761	F.removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
6762	Ty: F.getReturnType(), AS: F.getAttributes().getRetAttrs()));
6763	for (auto &Arg : F.args())
6764	Arg.removeAttrs(
6765	AM: AttributeFuncs::typeIncompatible(Ty: Arg.getType(), AS: Arg.getAttributes()));
6766
6767	bool AddingAttrs = false, RemovingAttrs = false;
6768	AttrBuilder AttrsToAdd(F.getContext());
6769	AttributeMask AttrsToRemove;
6770
6771	// Older versions of LLVM treated an "implicit-section-name" attribute
6772	// similarly to directly setting the section on a Function.
6773	if (Attribute A = F.getFnAttribute(Kind: "implicit-section-name");
6774	A.isValid() && A.isStringAttribute()) {
6775	F.setSection(A.getValueAsString());
6776	AttrsToRemove.addAttribute(A: "implicit-section-name");
6777	RemovingAttrs = true;
6778	}
6779
6780	if (Attribute A = F.getFnAttribute(Kind: "nooutline");
6781	A.isValid() && A.isStringAttribute()) {
6782	AttrsToRemove.addAttribute(A: "nooutline");
6783	AttrsToAdd.addAttribute(Val: Attribute::NoOutline);
6784	AddingAttrs = RemovingAttrs = true;
6785	}
6786
6787	if (Attribute A = F.getFnAttribute(Kind: "uniform-work-group-size");
6788	A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
6789	AttrsToRemove.addAttribute(A: "uniform-work-group-size");
6790	RemovingAttrs = true;
6791	if (A.getValueAsString() == "true") {
6792	AttrsToAdd.addAttribute(A: "uniform-work-group-size");
6793	AddingAttrs = true;
6794	}
6795	}
6796
6797	if (!F.empty()) {
6798	// For some reason this is called twice, and the first time is before any
6799	// instructions are loaded into the body.
6800
6801	if (Attribute A = F.getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics");
6802	A.isValid()) {
6803
6804	if (A.getValueAsBool()) {
6805	AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
6806	Visitor.visit(F);
6807	}
6808
6809	// We will leave behind dead attribute uses on external declarations, but
6810	// clang never added these to declarations anyway.
6811	AttrsToRemove.addAttribute(A: "amdgpu-unsafe-fp-atomics");
6812	RemovingAttrs = true;
6813	}
6814	}
6815
6816	DenormalMode DenormalFPMath = DenormalMode::getIEEE();
6817	DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
6818
6819	bool HandleDenormalMode = false;
6820
6821	if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math"); Attr.isValid()) {
6822	DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
6823	if (ParsedMode.isValid()) {
6824	DenormalFPMath = ParsedMode;
6825	AttrsToRemove.addAttribute(A: "denormal-fp-math");
6826	AddingAttrs = RemovingAttrs = true;
6827	HandleDenormalMode = true;
6828	}
6829	}
6830
6831	if (Attribute Attr = F.getFnAttribute(Kind: "denormal-fp-math-f32");
6832	Attr.isValid()) {
6833	DenormalMode ParsedMode = parseDenormalFPAttribute(Str: Attr.getValueAsString());
6834	if (ParsedMode.isValid()) {
6835	DenormalFPMathF32 = ParsedMode;
6836	AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
6837	AddingAttrs = RemovingAttrs = true;
6838	HandleDenormalMode = true;
6839	}
6840	}
6841
6842	if (HandleDenormalMode)
6843	AttrsToAdd.addDenormalFPEnvAttr(
6844	Mode: DenormalFPEnv (DenormalFPMath, DenormalFPMathF32));
6845
6846	if (RemovingAttrs)
6847	F.removeFnAttrs(Attrs: AttrsToRemove);
6848
6849	if (AddingAttrs)
6850	F.addFnAttrs(Attrs: AttrsToAdd);
6851	}
6852
6853	// Check if the function attribute is not present and set it.
6854	static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
6855	StringRef Value) {
6856	if (!F.hasFnAttribute(Kind: FnAttrName))
6857	F.addFnAttr(Kind: FnAttrName, Val: Value);
6858	}
6859
6860	// Check if the function attribute is not present and set it if needed.
6861	// If the attribute is "false" then removes it.
6862	// If the attribute is "true" resets it to a valueless attribute.
6863	static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
6864	if (!F.hasFnAttribute(Kind: FnAttrName)) {
6865	if (Set)
6866	F.addFnAttr(Kind: FnAttrName);
6867	} else {
6868	auto A = F.getFnAttribute(Kind: FnAttrName);
6869	if ("false" == A.getValueAsString())
6870	F.removeFnAttr(Kind: FnAttrName);
6871	else if ("true" == A.getValueAsString()) {
6872	F.removeFnAttr(Kind: FnAttrName);
6873	F.addFnAttr(Kind: FnAttrName);
6874	}
6875	}
6876	}
6877
6878	void llvm::copyModuleAttrToFunctions(Module &M) {
6879	Triple T(M.getTargetTriple());
6880	if (!T.isThumb() && !T.isARM() && !T.isAArch64())
6881	return;
6882
6883	uint64_t BTEValue = `0`;
6884	uint64_t BPPLRValue = `0`;
6885	uint64_t GCSValue = `0`;
6886	uint64_t SRAValue = `0`;
6887	uint64_t SRAALLValue = `0`;
6888	uint64_t SRABKeyValue = `0`;
6889
6890	NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6891	if (ModFlags) {
6892	for (unsigned I = `0`, E = ModFlags->getNumOperands(); I != E; ++I) {
6893	MDNode *Op = ModFlags->getOperand(i: I);
6894	if (Op->getNumOperands() != `3`)
6895	continue;
6896
6897	MDString *ID = dyn_cast_or_null<MDString>(Val: Op->getOperand(I: `1`));
6898	auto *CI = mdconst::dyn_extract<ConstantInt>(MD: Op->getOperand(I: `2`));
6899	if (!ID \|\| !CI)
6900	continue;
6901
6902	StringRef IDStr = ID->getString();
6903	uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
6904	: IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
6905	: IDStr == "guarded-control-stack" ? &GCSValue
6906	: IDStr == "sign-return-address" ? &SRAValue
6907	: IDStr == "sign-return-address-all" ? &SRAALLValue
6908	: IDStr == "sign-return-address-with-bkey"
6909	? &SRABKeyValue
6910	: nullptr;
6911	if (!ValPtr)
6912	continue;
6913
6914	*ValPtr = CI->getZExtValue();
6915	if (*ValPtr == `2`)
6916	return;
6917	}
6918	}
6919
6920	bool BTE = BTEValue == `1`;
6921	bool BPPLR = BPPLRValue == `1`;
6922	bool GCS = GCSValue == `1`;
6923	bool SRA = SRAValue == `1`;
6924
6925	StringRef SignTypeValue = "non-leaf";
6926	if (SRA && SRAALLValue == `1`)
6927	SignTypeValue = "all";
6928
6929	StringRef SignKeyValue = "a_key";
6930	if (SRA && SRABKeyValue == `1`)
6931	SignKeyValue = "b_key";
6932
6933	for (Function &F : M.getFunctionList()) {
6934	if (F.isDeclaration())
6935	continue;
6936
6937	if (SRA) {
6938	setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address", Value: SignTypeValue);
6939	setFunctionAttrIfNotSet(F, FnAttrName: "sign-return-address-key", Value: SignKeyValue);
6940	} else {
6941	if (auto A = F.getFnAttribute(Kind: "sign-return-address");
6942	A.isValid() && "none" == A.getValueAsString()) {
6943	F.removeFnAttr(Kind: "sign-return-address");
6944	F.removeFnAttr(Kind: "sign-return-address-key");
6945	}
6946	}
6947	ConvertFunctionAttr(F, Set: BTE, FnAttrName: "branch-target-enforcement");
6948	ConvertFunctionAttr(F, Set: BPPLR, FnAttrName: "branch-protection-pauth-lr");
6949	ConvertFunctionAttr(F, Set: GCS, FnAttrName: "guarded-control-stack");
6950	}
6951
6952	if (BTE)
6953	M.setModuleFlag(Behavior: llvm::Module::Min, Key: "branch-target-enforcement", Val: `2`);
6954	if (BPPLR)
6955	M.setModuleFlag(Behavior: llvm::Module::Min, Key: "branch-protection-pauth-lr", Val: `2`);
6956	if (GCS)
6957	M.setModuleFlag(Behavior: llvm::Module::Min, Key: "guarded-control-stack", Val: `2`);
6958	if (SRA) {
6959	M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address", Val: `2`);
6960	if (SRAALLValue == `1`)
6961	M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-all", Val: `2`);
6962	if (SRABKeyValue == `1`)
6963	M.setModuleFlag(Behavior: llvm::Module::Min, Key: "sign-return-address-with-bkey", Val: `2`);
6964	}
6965	}
6966
6967	static bool isOldLoopArgument(Metadata *MD) {
6968	auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
6969	if (!T)
6970	return false;
6971	if (T->getNumOperands() < `1`)
6972	return false;
6973	auto *S = dyn_cast_or_null<MDString>(Val: T->getOperand(I: `0`));
6974	if (!S)
6975	return false;
6976	return S->getString().starts_with(Prefix: "llvm.vectorizer.");
6977	}
6978
6979	static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
6980	StringRef OldPrefix = "llvm.vectorizer.";
6981	assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
6982
6983	if (OldTag == "llvm.vectorizer.unroll")
6984	return MDString::get(Context&: C, Str: "llvm.loop.interleave.count");
6985
6986	return MDString::get(
6987	Context&: C, Str: (Twine ("llvm.loop.vectorize.") + OldTag.drop_front(N: OldPrefix.size()))
6988	.str());
6989	}
6990
6991	static Metadata upgradeLoopArgument(Metadata MD) {
6992	auto *T = dyn_cast_or_null<MDTuple>(Val: MD);
6993	if (!T)
6994	return MD;
6995	if (T->getNumOperands() < `1`)
6996	return MD;
6997	auto *OldTag = dyn_cast_or_null<MDString>(Val: T->getOperand(I: `0`));
6998	if (!OldTag)
6999	return MD;
7000	if (!OldTag->getString().starts_with(Prefix: "llvm.vectorizer."))
7001	return MD;
7002
7003	// This has an old tag. Upgrade it.
7004	SmallVector<Metadata *, `8`> Ops;
7005	Ops.reserve(N: T->getNumOperands());
7006	Ops.push_back(Elt: upgradeLoopTag(C&: T->getContext(), OldTag: OldTag->getString()));
7007	for (unsigned I = `1`, E = T->getNumOperands(); I != E; ++I)
7008	Ops.push_back(Elt: T->getOperand(I));
7009
7010	return MDTuple::get(Context&: T->getContext(), MDs: Ops);
7011	}
7012
7013	MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7014	auto *T = dyn_cast<MDTuple>(Val: &N);
7015	if (!T)
7016	return &N;
7017
7018	if (none_of(Range: T->operands(), P: isOldLoopArgument))
7019	return &N;
7020
7021	SmallVector<Metadata *, `8`> Ops;
7022	Ops.reserve(N: T->getNumOperands());
7023	for (Metadata *MD : T->operands())
7024	Ops.push_back(Elt: upgradeLoopArgument(MD));
7025
7026	return MDTuple::get(Context&: T->getContext(), MDs: Ops);
7027	}
7028
7029	std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7030	Triple T(TT);
7031	// The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7032	// the address space of globals to 1. This does not apply to SPIRV Logical.
7033	if ((T.isSPIR() \|\| (T.isSPIRV() && !T.isSPIRVLogical())) &&
7034	!DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G")) {
7035	return DL.empty() ? std::string ("G1") : (DL + "-G1").str();
7036	}
7037
7038	if (T.isLoongArch64() \|\| T.isRISCV64()) {
7039	// Make i32 a native type for 64-bit LoongArch and RISC-V.
7040	auto I = DL.find(Str: "-n64-");
7041	if (I != StringRef::npos)
7042	return (DL.take_front(N: I) + "-n32:64-" + DL.drop_front(N: I + `5`)).str();
7043	return DL.str();
7044	}
7045
7046	// AMDGPU data layout upgrades.
7047	std::string Res = DL.str();
7048	if (T.isAMDGPU()) {
7049	// Define address spaces for constants.
7050	if (!DL.contains(Other: "-G") && !DL.starts_with(Prefix: "G"))
7051	Res.append(s: Res.empty() ? "G1" : "-G1");
7052
7053	// AMDGCN data layout upgrades.
7054	if (T.isAMDGCN()) {
7055
7056	// Add missing non-integral declarations.
7057	// This goes before adding new address spaces to prevent incoherent string
7058	// values.
7059	if (!DL.contains(Other: "-ni") && !DL.starts_with(Prefix: "ni"))
7060	Res.append(s: "-ni:7:8:9");
7061	// Update ni:7 to ni:7:8:9.
7062	if (DL.ends_with(Suffix: "ni:7"))
7063	Res.append(s: ":8:9");
7064	if (DL.ends_with(Suffix: "ni:7:8"))
7065	Res.append(s: ":9");
7066
7067	// Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7068	// resources) An empty data layout has already been upgraded to G1 by now.
7069	if (!DL.contains(Other: "-p7") && !DL.starts_with(Prefix: "p7"))
7070	Res.append(s: "-p7:160:256:256:32");
7071	if (!DL.contains(Other: "-p8") && !DL.starts_with(Prefix: "p8"))
7072	Res.append(s: "-p8:128:128:128:48");
7073	constexpr StringRef OldP8("-p8:128:128-");
7074	if (DL.contains(Other: OldP8))
7075	Res.replace(pos: Res.find(svt: OldP8), n1: OldP8.size(), s: "-p8:128:128:128:48-");
7076	if (!DL.contains(Other: "-p9") && !DL.starts_with(Prefix: "p9"))
7077	Res.append(s: "-p9:192:256:256:32");
7078	}
7079
7080	// Upgrade the ELF mangling mode.
7081	if (!DL.contains(Other: "m:e"))
7082	Res = Res.empty() ? "m:e" : "m:e-" + Res;
7083
7084	return Res;
7085	}
7086
7087	if (T.isSystemZ() && !DL.empty()) {
7088	// Make sure the stack alignment is present.
7089	if (!DL.contains(Other: "-S64"))
7090	return "E-S64" + DL.drop_front(N: `1`).str();
7091	return DL.str();
7092	}
7093
7094	auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7095	// If the datalayout matches the expected format, add pointer size address
7096	// spaces to the datalayout.
7097	StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7098	if (!DL.contains(Other: AddrSpaces)) {
7099	SmallVector<StringRef, `4`> Groups;
7100	Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7101	if (R.match(String: Res, Matches: &Groups))
7102	Res = (Groups [`1`] + AddrSpaces + Groups [`3`]).str();
7103	}
7104	};
7105
7106	// AArch64 data layout upgrades.
7107	if (T.isAArch64()) {
7108	// Add "-Fn32"
7109	if (!DL.empty() && !DL.contains(Other: "-Fn32"))
7110	Res.append(s: "-Fn32");
7111	AddPtr32Ptr64AddrSpaces ();
7112	return Res;
7113	}
7114
7115	if (T.isSPARC() \|\| (T.isMIPS64() && !DL.contains(Other: "m:m")) \|\| T.isPPC64() \|\|
7116	T.isWasm()) {
7117	// Mips64 with o32 ABI did not add "-i128:128".
7118	// Add "-i128:128"
7119	std::string I64 = "-i64:64";
7120	std::string I128 = "-i128:128";
7121	if (!StringRef (Res).contains(Other: I128)) {
7122	size_t Pos = Res.find(str: I64);
7123	if (Pos != size_t(-`1`))
7124	Res.insert(pos1: Pos + I64.size(), str: I128);
7125	}
7126	}
7127
7128	if (T.isPPC() && T.isOSAIX() && !DL.contains(Other: "f64:32:64") && !DL.empty()) {
7129	size_t Pos = Res.find(s: "-S128");
7130	if (Pos == StringRef::npos)
7131	Pos = Res.size();
7132	Res.insert(pos: Pos, s: "-f64:32:64");
7133	}
7134
7135	if (!T.isX86())
7136	return Res;
7137
7138	AddPtr32Ptr64AddrSpaces ();
7139
7140	// i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7141	// for i128 operations prior to this being reflected in the data layout, and
7142	// clang mostly produced LLVM IR that already aligned i128 to 16 byte
7143	// boundaries, so although this is a breaking change, the upgrade is expected
7144	// to fix more IR than it breaks.
7145	// Intel MCU is an exception and uses 4-byte-alignment.
7146	if (!T.isOSIAMCU()) {
7147	std::string I128 = "-i128:128";
7148	if (StringRef Ref = Res; !Ref.contains(Other: I128)) {
7149	SmallVector<StringRef, `4`> Groups;
7150	Regex R("^(e(-[mpi][^-]))((-[^mpi][^-]))$");
7151	if (R.match(String: Res, Matches: &Groups))
7152	Res = (Groups [`1`] + I128 + Groups [`3`]).str();
7153	}
7154	}
7155
7156	// For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7157	// Raising the alignment is safe because Clang did not produce f80 values in
7158	// the MSVC environment before this upgrade was added.
7159	if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7160	StringRef Ref = Res;
7161	auto I = Ref.find(Str: "-f80:32-");
7162	if (I != StringRef::npos)
7163	Res = (Ref.take_front(N: I) + "-f80:128-" + Ref.drop_front(N: I + `8`)).str();
7164	}
7165
7166	return Res;
7167	}
7168
7169	void llvm::UpgradeAttributes(AttrBuilder &B) {
7170	StringRef FramePointer;
7171	Attribute A = B.getAttribute(Kind: "no-frame-pointer-elim");
7172	if (A.isValid()) {
7173	// The value can be "true" or "false".
7174	FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7175	B.removeAttribute(A: "no-frame-pointer-elim");
7176	}
7177	if (B.contains(A: "no-frame-pointer-elim-non-leaf")) {
7178	// The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7179	if (FramePointer != "all")
7180	FramePointer = "non-leaf";
7181	B.removeAttribute(A: "no-frame-pointer-elim-non-leaf");
7182	}
7183	if (!FramePointer.empty())
7184	B.addAttribute(A: "frame-pointer", V: FramePointer);
7185
7186	A = B.getAttribute(Kind: "null-pointer-is-valid");
7187	if (A.isValid()) {
7188	// The value can be "true" or "false".
7189	bool NullPointerIsValid = A.getValueAsString() == "true";
7190	B.removeAttribute(A: "null-pointer-is-valid");
7191	if (NullPointerIsValid)
7192	B.addAttribute(Val: Attribute::NullPointerIsValid);
7193	}
7194
7195	A = B.getAttribute(Kind: "uniform-work-group-size");
7196	if (A.isValid()) {
7197	StringRef Val = A.getValueAsString();
7198	if (!Val.empty()) {
7199	bool IsTrue = Val == "true";
7200	B.removeAttribute(A: "uniform-work-group-size");
7201	if (IsTrue)
7202	B.addAttribute(A: "uniform-work-group-size");
7203	}
7204	}
7205	}
7206
7207	void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7208	// clang.arc.attachedcall bundles are now required to have an operand.
7209	// If they don't, it's okay to drop them entirely: when there is an operand,
7210	// the "attachedcall" is meaningful and required, but without an operand,
7211	// it's just a marker NOP. Dropping it merely prevents an optimization.
7212	erase_if(C&: Bundles, P: [&](OperandBundleDef &OBD) {
7213	return OBD.getTag() == "clang.arc.attachedcall" &&
7214	OBD.inputs().empty();
7215	});
7216	}
7217

Browse the source code of llvm_projects/llvm/lib/IR/AutoUpgrade.cpp