1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions required for supporting intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Intrinsics.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringTable.h"
16#include "llvm/IR/ConstantRange.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/IR/IntrinsicsPowerPC.h"
27#include "llvm/IR/IntrinsicsR600.h"
28#include "llvm/IR/IntrinsicsRISCV.h"
29#include "llvm/IR/IntrinsicsS390.h"
30#include "llvm/IR/IntrinsicsSPIRV.h"
31#include "llvm/IR/IntrinsicsVE.h"
32#include "llvm/IR/IntrinsicsX86.h"
33#include "llvm/IR/IntrinsicsXCore.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NVVMIntrinsicUtils.h"
36#include "llvm/IR/Type.h"
37
38using namespace llvm;
39
40/// Table of string intrinsic names indexed by enum value.
41#define GET_INTRINSIC_NAME_TABLE
42#include "llvm/IR/IntrinsicImpl.inc"
43#undef GET_INTRINSIC_NAME_TABLE
44
45StringRef Intrinsic::getBaseName(ID id) {
46 assert(id < num_intrinsics && "Invalid intrinsic ID!");
47 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
48}
49
50StringRef Intrinsic::getName(ID id) {
51 assert(id < num_intrinsics && "Invalid intrinsic ID!");
52 assert(!Intrinsic::isOverloaded(id) &&
53 "This version of getName does not support overloading");
54 return getBaseName(id);
55}
56
57/// Returns a stable mangling for the type specified for use in the name
58/// mangling scheme used by 'any' types in intrinsic signatures. The mangling
59/// of named types is simply their name. Manglings for unnamed types consist
60/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
61/// combined with the mangling of their component types. A vararg function
62/// type will have a suffix of 'vararg'. Since function types can contain
63/// other function types, we close a function type mangling with suffix 'f'
64/// which can't be confused with it's prefix. This ensures we don't have
65/// collisions between two unrelated function types. Otherwise, you might
66/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)
67/// The HasUnnamedType boolean is set if an unnamed type was encountered,
68/// indicating that extra care must be taken to ensure a unique name.
69static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
70 std::string Result;
71 if (PointerType *PTyp = dyn_cast<PointerType>(Val: Ty)) {
72 Result += "p" + utostr(X: PTyp->getAddressSpace());
73 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Val: Ty)) {
74 Result += "a" + utostr(X: ATyp->getNumElements()) +
75 getMangledTypeStr(Ty: ATyp->getElementType(), HasUnnamedType);
76 } else if (StructType *STyp = dyn_cast<StructType>(Val: Ty)) {
77 if (!STyp->isLiteral()) {
78 Result += "s_";
79 if (STyp->hasName())
80 Result += STyp->getName();
81 else
82 HasUnnamedType = true;
83 } else {
84 Result += "sl_";
85 for (auto *Elem : STyp->elements())
86 Result += getMangledTypeStr(Ty: Elem, HasUnnamedType);
87 }
88 // Ensure nested structs are distinguishable.
89 Result += "s";
90 } else if (FunctionType *FT = dyn_cast<FunctionType>(Val: Ty)) {
91 Result += "f_" + getMangledTypeStr(Ty: FT->getReturnType(), HasUnnamedType);
92 for (size_t i = 0; i < FT->getNumParams(); i++)
93 Result += getMangledTypeStr(Ty: FT->getParamType(i), HasUnnamedType);
94 if (FT->isVarArg())
95 Result += "vararg";
96 // Ensure nested function types are distinguishable.
97 Result += "f";
98 } else if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
99 ElementCount EC = VTy->getElementCount();
100 if (EC.isScalable())
101 Result += "nx";
102 Result += "v" + utostr(X: EC.getKnownMinValue()) +
103 getMangledTypeStr(Ty: VTy->getElementType(), HasUnnamedType);
104 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Val: Ty)) {
105 Result += "t";
106 Result += TETy->getName();
107 for (Type *ParamTy : TETy->type_params())
108 Result += "_" + getMangledTypeStr(Ty: ParamTy, HasUnnamedType);
109 for (unsigned IntParam : TETy->int_params())
110 Result += "_" + utostr(X: IntParam);
111 // Ensure nested target extension types are distinguishable.
112 Result += "t";
113 } else if (Ty) {
114 switch (Ty->getTypeID()) {
115 default:
116 llvm_unreachable("Unhandled type");
117 case Type::VoidTyID:
118 Result += "isVoid";
119 break;
120 case Type::MetadataTyID:
121 Result += "Metadata";
122 break;
123 case Type::HalfTyID:
124 Result += "f16";
125 break;
126 case Type::BFloatTyID:
127 Result += "bf16";
128 break;
129 case Type::FloatTyID:
130 Result += "f32";
131 break;
132 case Type::DoubleTyID:
133 Result += "f64";
134 break;
135 case Type::X86_FP80TyID:
136 Result += "f80";
137 break;
138 case Type::FP128TyID:
139 Result += "f128";
140 break;
141 case Type::PPC_FP128TyID:
142 Result += "ppcf128";
143 break;
144 case Type::X86_AMXTyID:
145 Result += "x86amx";
146 break;
147 case Type::IntegerTyID:
148 Result += "i" + utostr(X: cast<IntegerType>(Val: Ty)->getBitWidth());
149 break;
150 }
151 }
152 return Result;
153}
154
155static std::string getIntrinsicNameImpl(Intrinsic::ID Id, ArrayRef<Type *> Tys,
156 Module *M, FunctionType *FT,
157 bool EarlyModuleCheck) {
158
159 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
160 assert((Tys.empty() || Intrinsic::isOverloaded(Id)) &&
161 "This version of getName is for overloaded intrinsics only");
162 (void)EarlyModuleCheck;
163 assert((!EarlyModuleCheck || M ||
164 !any_of(Tys, [](Type *T) { return isa<PointerType>(T); })) &&
165 "Intrinsic overloading on pointer types need to provide a Module");
166 bool HasUnnamedType = false;
167 std::string Result(Intrinsic::getBaseName(id: Id));
168 for (Type *Ty : Tys)
169 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
170 if (HasUnnamedType) {
171 assert(M && "unnamed types need a module");
172 if (!FT)
173 FT = Intrinsic::getType(Context&: M->getContext(), id: Id, Tys);
174 else
175 assert((FT == Intrinsic::getType(M->getContext(), Id, Tys)) &&
176 "Provided FunctionType must match arguments");
177 return M->getUniqueIntrinsicName(BaseName: Result, Id, Proto: FT);
178 }
179 return Result;
180}
181
182std::string Intrinsic::getName(ID Id, ArrayRef<Type *> Tys, Module *M,
183 FunctionType *FT) {
184 assert(M && "We need to have a Module");
185 return getIntrinsicNameImpl(Id, Tys, M, FT, EarlyModuleCheck: true);
186}
187
188std::string Intrinsic::getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys) {
189 return getIntrinsicNameImpl(Id, Tys, M: nullptr, FT: nullptr, EarlyModuleCheck: false);
190}
191
192/// IIT_Info - These are enumerators that describe the entries returned by the
193/// getIntrinsicInfoTableEntries function.
194///
195/// Defined in Intrinsics.td.
196enum IIT_Info {
197#define GET_INTRINSIC_IITINFO
198#include "llvm/IR/IntrinsicImpl.inc"
199#undef GET_INTRINSIC_IITINFO
200};
201
202static void
203DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
204 IIT_Info LastInfo,
205 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
206 using namespace Intrinsic;
207
208 bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC);
209
210 IIT_Info Info = IIT_Info(Infos[NextElt++]);
211
212 switch (Info) {
213 case IIT_Done:
214 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Void, Field: 0));
215 return;
216 case IIT_VARARG:
217 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VarArg, Field: 0));
218 return;
219 case IIT_MMX:
220 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::MMX, Field: 0));
221 return;
222 case IIT_AMX:
223 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AMX, Field: 0));
224 return;
225 case IIT_TOKEN:
226 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Token, Field: 0));
227 return;
228 case IIT_METADATA:
229 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Metadata, Field: 0));
230 return;
231 case IIT_F16:
232 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Half, Field: 0));
233 return;
234 case IIT_BF16:
235 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::BFloat, Field: 0));
236 return;
237 case IIT_F32:
238 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Float, Field: 0));
239 return;
240 case IIT_F64:
241 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Double, Field: 0));
242 return;
243 case IIT_F128:
244 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Quad, Field: 0));
245 return;
246 case IIT_PPCF128:
247 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::PPCQuad, Field: 0));
248 return;
249 case IIT_I1:
250 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 1));
251 return;
252 case IIT_I2:
253 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 2));
254 return;
255 case IIT_I4:
256 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 4));
257 return;
258 case IIT_AARCH64_SVCOUNT:
259 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AArch64Svcount, Field: 0));
260 return;
261 case IIT_I8:
262 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 8));
263 return;
264 case IIT_I16:
265 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 16));
266 return;
267 case IIT_I32:
268 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 32));
269 return;
270 case IIT_I64:
271 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 64));
272 return;
273 case IIT_I128:
274 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 128));
275 return;
276 case IIT_V1:
277 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1, IsScalable: IsScalableVector));
278 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
279 return;
280 case IIT_V2:
281 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2, IsScalable: IsScalableVector));
282 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
283 return;
284 case IIT_V3:
285 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 3, IsScalable: IsScalableVector));
286 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
287 return;
288 case IIT_V4:
289 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4, IsScalable: IsScalableVector));
290 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
291 return;
292 case IIT_V6:
293 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 6, IsScalable: IsScalableVector));
294 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
295 return;
296 case IIT_V8:
297 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 8, IsScalable: IsScalableVector));
298 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
299 return;
300 case IIT_V10:
301 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 10, IsScalable: IsScalableVector));
302 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
303 return;
304 case IIT_V16:
305 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 16, IsScalable: IsScalableVector));
306 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
307 return;
308 case IIT_V32:
309 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 32, IsScalable: IsScalableVector));
310 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
311 return;
312 case IIT_V64:
313 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 64, IsScalable: IsScalableVector));
314 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
315 return;
316 case IIT_V128:
317 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 128, IsScalable: IsScalableVector));
318 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
319 return;
320 case IIT_V256:
321 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 256, IsScalable: IsScalableVector));
322 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
323 return;
324 case IIT_V512:
325 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 512, IsScalable: IsScalableVector));
326 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
327 return;
328 case IIT_V1024:
329 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1024, IsScalable: IsScalableVector));
330 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
331 return;
332 case IIT_V2048:
333 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2048, IsScalable: IsScalableVector));
334 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
335 return;
336 case IIT_V4096:
337 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4096, IsScalable: IsScalableVector));
338 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
339 return;
340 case IIT_EXTERNREF:
341 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 10));
342 return;
343 case IIT_FUNCREF:
344 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 20));
345 return;
346 case IIT_PTR:
347 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 0));
348 return;
349 case IIT_ANYPTR: // [ANYPTR addrspace]
350 OutputTable.push_back(
351 Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: Infos[NextElt++]));
352 return;
353 case IIT_ARG: {
354 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
355 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Argument, Field: ArgInfo));
356 return;
357 }
358 case IIT_EXTEND_ARG: {
359 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
360 OutputTable.push_back(
361 Elt: IITDescriptor::get(K: IITDescriptor::ExtendArgument, Field: ArgInfo));
362 return;
363 }
364 case IIT_TRUNC_ARG: {
365 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
366 OutputTable.push_back(
367 Elt: IITDescriptor::get(K: IITDescriptor::TruncArgument, Field: ArgInfo));
368 return;
369 }
370 case IIT_ONE_NTH_ELTS_VEC_ARG: {
371 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
372 unsigned short N = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
373 OutputTable.push_back(
374 Elt: IITDescriptor::get(K: IITDescriptor::OneNthEltsVecArgument, Hi: N, Lo: ArgNo));
375 return;
376 }
377 case IIT_SAME_VEC_WIDTH_ARG: {
378 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
379 OutputTable.push_back(
380 Elt: IITDescriptor::get(K: IITDescriptor::SameVecWidthArgument, Field: ArgInfo));
381 return;
382 }
383 case IIT_VEC_OF_ANYPTRS_TO_ELT: {
384 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
385 unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
386 OutputTable.push_back(
387 Elt: IITDescriptor::get(K: IITDescriptor::VecOfAnyPtrsToElt, Hi: ArgNo, Lo: RefNo));
388 return;
389 }
390 case IIT_EMPTYSTRUCT:
391 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: 0));
392 return;
393 case IIT_STRUCT: {
394 unsigned StructElts = Infos[NextElt++] + 2;
395
396 OutputTable.push_back(
397 Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: StructElts));
398
399 for (unsigned i = 0; i != StructElts; ++i)
400 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
401 return;
402 }
403 case IIT_SUBDIVIDE2_ARG: {
404 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
405 OutputTable.push_back(
406 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide2Argument, Field: ArgInfo));
407 return;
408 }
409 case IIT_SUBDIVIDE4_ARG: {
410 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
411 OutputTable.push_back(
412 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide4Argument, Field: ArgInfo));
413 return;
414 }
415 case IIT_VEC_ELEMENT: {
416 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
417 OutputTable.push_back(
418 Elt: IITDescriptor::get(K: IITDescriptor::VecElementArgument, Field: ArgInfo));
419 return;
420 }
421 case IIT_SCALABLE_VEC: {
422 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
423 return;
424 }
425 case IIT_VEC_OF_BITCASTS_TO_INT: {
426 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
427 OutputTable.push_back(
428 Elt: IITDescriptor::get(K: IITDescriptor::VecOfBitcastsToInt, Field: ArgInfo));
429 return;
430 }
431 }
432 llvm_unreachable("unhandled");
433}
434
435#define GET_INTRINSIC_GENERATOR_GLOBAL
436#include "llvm/IR/IntrinsicImpl.inc"
437#undef GET_INTRINSIC_GENERATOR_GLOBAL
438
439void Intrinsic::getIntrinsicInfoTableEntries(
440 ID id, SmallVectorImpl<IITDescriptor> &T) {
441 // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
442 // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
443 // IntrinsicEmitter.cpp.
444 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
445 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
446 // Mask with all bits 1 except the most significant bit.
447 constexpr unsigned Mask = (1U << MSBPosition) - 1;
448
449 FixedEncodingTy TableVal = IIT_Table[id - 1];
450
451 // Decode the TableVal into an array of IITValues.
452 SmallVector<unsigned char> IITValues;
453 ArrayRef<unsigned char> IITEntries;
454 unsigned NextElt = 0;
455 // Check to see if the intrinsic's type was inlined in the fixed encoding
456 // table.
457 if (TableVal >> MSBPosition) {
458 // This is an offset into the IIT_LongEncodingTable.
459 IITEntries = IIT_LongEncodingTable;
460
461 // Strip sentinel bit.
462 NextElt = TableVal & Mask;
463 } else {
464 // If the entry was encoded into a single word in the table itself, decode
465 // it from an array of nibbles to an array of bytes.
466 do {
467 IITValues.push_back(Elt: TableVal & 0xF);
468 TableVal >>= 4;
469 } while (TableVal);
470
471 IITEntries = IITValues;
472 NextElt = 0;
473 }
474
475 // Okay, decode the table into the output vector of IITDescriptors.
476 DecodeIITType(NextElt, Infos: IITEntries, LastInfo: IIT_Done, OutputTable&: T);
477 while (NextElt != IITEntries.size() && IITEntries[NextElt] != 0)
478 DecodeIITType(NextElt, Infos: IITEntries, LastInfo: IIT_Done, OutputTable&: T);
479}
480
481static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
482 ArrayRef<Type *> Tys, LLVMContext &Context) {
483 using namespace Intrinsic;
484
485 IITDescriptor D = Infos.front();
486 Infos = Infos.slice(N: 1);
487
488 switch (D.Kind) {
489 case IITDescriptor::Void:
490 return Type::getVoidTy(C&: Context);
491 case IITDescriptor::VarArg:
492 return Type::getVoidTy(C&: Context);
493 case IITDescriptor::MMX:
494 return llvm::FixedVectorType::get(ElementType: llvm::IntegerType::get(C&: Context, NumBits: 64), NumElts: 1);
495 case IITDescriptor::AMX:
496 return Type::getX86_AMXTy(C&: Context);
497 case IITDescriptor::Token:
498 return Type::getTokenTy(C&: Context);
499 case IITDescriptor::Metadata:
500 return Type::getMetadataTy(C&: Context);
501 case IITDescriptor::Half:
502 return Type::getHalfTy(C&: Context);
503 case IITDescriptor::BFloat:
504 return Type::getBFloatTy(C&: Context);
505 case IITDescriptor::Float:
506 return Type::getFloatTy(C&: Context);
507 case IITDescriptor::Double:
508 return Type::getDoubleTy(C&: Context);
509 case IITDescriptor::Quad:
510 return Type::getFP128Ty(C&: Context);
511 case IITDescriptor::PPCQuad:
512 return Type::getPPC_FP128Ty(C&: Context);
513 case IITDescriptor::AArch64Svcount:
514 return TargetExtType::get(Context, Name: "aarch64.svcount");
515
516 case IITDescriptor::Integer:
517 return IntegerType::get(C&: Context, NumBits: D.Integer_Width);
518 case IITDescriptor::Vector:
519 return VectorType::get(ElementType: DecodeFixedType(Infos, Tys, Context),
520 EC: D.Vector_Width);
521 case IITDescriptor::Pointer:
522 return PointerType::get(C&: Context, AddressSpace: D.Pointer_AddressSpace);
523 case IITDescriptor::Struct: {
524 SmallVector<Type *, 8> Elts;
525 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)
526 Elts.push_back(Elt: DecodeFixedType(Infos, Tys, Context));
527 return StructType::get(Context, Elements: Elts);
528 }
529 case IITDescriptor::Argument:
530 return Tys[D.getArgumentNumber()];
531 case IITDescriptor::ExtendArgument: {
532 Type *Ty = Tys[D.getArgumentNumber()];
533 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
534 return VectorType::getExtendedElementVectorType(VTy);
535
536 return IntegerType::get(C&: Context, NumBits: 2 * cast<IntegerType>(Val: Ty)->getBitWidth());
537 }
538 case IITDescriptor::TruncArgument: {
539 Type *Ty = Tys[D.getArgumentNumber()];
540 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
541 return VectorType::getTruncatedElementVectorType(VTy);
542
543 IntegerType *ITy = cast<IntegerType>(Val: Ty);
544 assert(ITy->getBitWidth() % 2 == 0);
545 return IntegerType::get(C&: Context, NumBits: ITy->getBitWidth() / 2);
546 }
547 case IITDescriptor::Subdivide2Argument:
548 case IITDescriptor::Subdivide4Argument: {
549 Type *Ty = Tys[D.getArgumentNumber()];
550 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
551 assert(VTy && "Expected an argument of Vector Type");
552 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
553 return VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
554 }
555 case IITDescriptor::OneNthEltsVecArgument:
556 return VectorType::getOneNthElementsVectorType(
557 VTy: cast<VectorType>(Val: Tys[D.getRefArgNumber()]), Denominator: D.getVectorDivisor());
558 case IITDescriptor::SameVecWidthArgument: {
559 Type *EltTy = DecodeFixedType(Infos, Tys, Context);
560 Type *Ty = Tys[D.getArgumentNumber()];
561 if (auto *VTy = dyn_cast<VectorType>(Val: Ty))
562 return VectorType::get(ElementType: EltTy, EC: VTy->getElementCount());
563 return EltTy;
564 }
565 case IITDescriptor::VecElementArgument: {
566 Type *Ty = Tys[D.getArgumentNumber()];
567 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
568 return VTy->getElementType();
569 llvm_unreachable("Expected an argument of Vector Type");
570 }
571 case IITDescriptor::VecOfBitcastsToInt: {
572 Type *Ty = Tys[D.getArgumentNumber()];
573 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
574 assert(VTy && "Expected an argument of Vector Type");
575 return VectorType::getInteger(VTy);
576 }
577 case IITDescriptor::VecOfAnyPtrsToElt:
578 // Return the overloaded type (which determines the pointers address space)
579 return Tys[D.getOverloadArgNumber()];
580 }
581 llvm_unreachable("unhandled");
582}
583
584FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
585 ArrayRef<Type *> Tys) {
586 SmallVector<IITDescriptor, 8> Table;
587 getIntrinsicInfoTableEntries(id, T&: Table);
588
589 ArrayRef<IITDescriptor> TableRef = Table;
590 Type *ResultTy = DecodeFixedType(Infos&: TableRef, Tys, Context);
591
592 SmallVector<Type *, 8> ArgTys;
593 while (!TableRef.empty())
594 ArgTys.push_back(Elt: DecodeFixedType(Infos&: TableRef, Tys, Context));
595
596 // DecodeFixedType returns Void for IITDescriptor::Void and
597 // IITDescriptor::VarArg If we see void type as the type of the last argument,
598 // it is vararg intrinsic
599 if (!ArgTys.empty() && ArgTys.back()->isVoidTy()) {
600 ArgTys.pop_back();
601 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: true);
602 }
603 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
604}
605
606bool Intrinsic::isOverloaded(ID id) {
607#define GET_INTRINSIC_OVERLOAD_TABLE
608#include "llvm/IR/IntrinsicImpl.inc"
609#undef GET_INTRINSIC_OVERLOAD_TABLE
610}
611
612bool Intrinsic::hasPrettyPrintedArgs(ID id){
613#define GET_INTRINSIC_PRETTY_PRINT_TABLE
614#include "llvm/IR/IntrinsicImpl.inc"
615#undef GET_INTRINSIC_PRETTY_PRINT_TABLE
616}
617
618/// Table of per-target intrinsic name tables.
619#define GET_INTRINSIC_TARGET_DATA
620#include "llvm/IR/IntrinsicImpl.inc"
621#undef GET_INTRINSIC_TARGET_DATA
622
623bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
624 return IID > TargetInfos[0].Count;
625}
626
627/// Looks up Name in NameTable via binary search. NameTable must be sorted
628/// and all entries must start with "llvm.". If NameTable contains an exact
629/// match for Name or a prefix of Name followed by a dot, its index in
630/// NameTable is returned. Otherwise, -1 is returned.
631static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
632 StringRef Name, StringRef Target = "") {
633 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
634 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
635
636 // Do successive binary searches of the dotted name components. For
637 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
638 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
639 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
640 // size 1. During the search, we can skip the prefix that we already know is
641 // identical. By using strncmp we consider names with differing suffixes to
642 // be part of the equal range.
643 size_t CmpEnd = 4; // Skip the "llvm" component.
644 if (!Target.empty())
645 CmpEnd += 1 + Target.size(); // skip the .target component.
646
647 const unsigned *Low = NameOffsetTable.begin();
648 const unsigned *High = NameOffsetTable.end();
649 const unsigned *LastLow = Low;
650 while (CmpEnd < Name.size() && High - Low > 0) {
651 size_t CmpStart = CmpEnd;
652 CmpEnd = Name.find(C: '.', From: CmpStart + 1);
653 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
654 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
655 // `equal_range` requires the comparison to work with either side being an
656 // offset or the value. Detect which kind each side is to set up the
657 // compared strings.
658 const char *LHSStr;
659 if constexpr (std::is_integral_v<decltype(LHS)>)
660 LHSStr = IntrinsicNameTable.getCString(O: LHS);
661 else
662 LHSStr = LHS;
663
664 const char *RHSStr;
665 if constexpr (std::is_integral_v<decltype(RHS)>)
666 RHSStr = IntrinsicNameTable.getCString(O: RHS);
667 else
668 RHSStr = RHS;
669
670 return strncmp(s1: LHSStr + CmpStart, s2: RHSStr + CmpStart, n: CmpEnd - CmpStart) <
671 0;
672 };
673 LastLow = Low;
674 std::tie(args&: Low, args&: High) = std::equal_range(first: Low, last: High, val: Name.data(), comp: Cmp);
675 }
676 if (High - Low > 0)
677 LastLow = Low;
678
679 if (LastLow == NameOffsetTable.end())
680 return -1;
681 StringRef NameFound = IntrinsicNameTable[*LastLow];
682 if (Name == NameFound ||
683 (Name.starts_with(Prefix: NameFound) && Name[NameFound.size()] == '.'))
684 return LastLow - NameOffsetTable.begin();
685 return -1;
686}
687
688/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
689/// target as \c Name, or the generic table if \c Name is not target specific.
690///
691/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
692/// name.
693static std::pair<ArrayRef<unsigned>, StringRef>
694findTargetSubtable(StringRef Name) {
695 assert(Name.starts_with("llvm."));
696
697 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
698 // Drop "llvm." and take the first dotted component. That will be the target
699 // if this is target specific.
700 StringRef Target = Name.drop_front(N: 5).split(Separator: '.').first;
701 auto It = partition_point(
702 Range&: Targets, P: [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
703 // We've either found the target or just fall back to the generic set, which
704 // is always first.
705 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
706 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
707 TI.Name};
708}
709
710/// This does the actual lookup of an intrinsic ID which matches the given
711/// function name.
712Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
713 auto [NameOffsetTable, Target] = findTargetSubtable(Name);
714 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
715 if (Idx == -1)
716 return Intrinsic::not_intrinsic;
717
718 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
719 // an index into a sub-table.
720 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
721 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
722
723 // If the intrinsic is not overloaded, require an exact match. If it is
724 // overloaded, require either exact or prefix match.
725 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
726 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
727 bool IsExactMatch = Name.size() == MatchSize;
728 return IsExactMatch || Intrinsic::isOverloaded(id: ID) ? ID
729 : Intrinsic::not_intrinsic;
730}
731
732/// This defines the "Intrinsic::getAttributes(ID id)" method.
733#define GET_INTRINSIC_ATTRIBUTES
734#include "llvm/IR/IntrinsicImpl.inc"
735#undef GET_INTRINSIC_ATTRIBUTES
736
737static Function *getOrInsertIntrinsicDeclarationImpl(Module *M,
738 Intrinsic::ID id,
739 ArrayRef<Type *> Tys,
740 FunctionType *FT) {
741 Function *F = cast<Function>(
742 Val: M->getOrInsertFunction(Name: Tys.empty() ? Intrinsic::getName(id)
743 : Intrinsic::getName(Id: id, Tys, M, FT),
744 T: FT)
745 .getCallee());
746 if (F->getFunctionType() == FT)
747 return F;
748
749 // It's possible that a declaration for this intrinsic already exists with an
750 // incorrect signature, if the signature has changed, but this particular
751 // declaration has not been auto-upgraded yet. In that case, rename the
752 // invalid declaration and insert a new one with the correct signature. The
753 // invalid declaration will get upgraded later.
754 F->setName(F->getName() + ".invalid");
755 return cast<Function>(
756 Val: M->getOrInsertFunction(Name: Tys.empty() ? Intrinsic::getName(id)
757 : Intrinsic::getName(Id: id, Tys, M, FT),
758 T: FT)
759 .getCallee());
760}
761
762Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
763 ArrayRef<Type *> Tys) {
764 // There can never be multiple globals with the same name of different types,
765 // because intrinsics must be a specific type.
766 FunctionType *FT = getType(Context&: M->getContext(), id, Tys);
767 return getOrInsertIntrinsicDeclarationImpl(M, id, Tys, FT);
768}
769
770Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
771 ArrayRef<Type *> ArgTys) {
772 // If the intrinsic is not overloaded, use the non-overloaded version.
773 if (!Intrinsic::isOverloaded(id))
774 return getOrInsertDeclaration(M, id);
775
776 // Get the intrinsic signature metadata.
777 SmallVector<Intrinsic::IITDescriptor, 8> Table;
778 getIntrinsicInfoTableEntries(id, T&: Table);
779 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
780
781 FunctionType *FTy = FunctionType::get(Result: RetTy, Params: ArgTys, /*isVarArg=*/false);
782
783 // Automatically determine the overloaded types.
784 SmallVector<Type *, 4> OverloadTys;
785 [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult Res =
786 matchIntrinsicSignature(FTy, Infos&: TableRef, ArgTys&: OverloadTys);
787 assert(Res == Intrinsic::MatchIntrinsicTypes_Match &&
788 "intrinsic signature mismatch");
789
790 // If intrinsic requires vararg, recreate the FunctionType accordingly.
791 if (!matchIntrinsicVarArg(/*isVarArg=*/true, Infos&: TableRef))
792 FTy = FunctionType::get(Result: RetTy, Params: ArgTys, /*isVarArg=*/true);
793
794 assert(TableRef.empty() && "Unprocessed descriptors remain");
795
796 return getOrInsertIntrinsicDeclarationImpl(M, id, Tys: OverloadTys, FT: FTy);
797}
798
799Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
800 return M->getFunction(Name: getName(id));
801}
802
803Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
804 ArrayRef<Type *> Tys,
805 FunctionType *FT) {
806 return M->getFunction(Name: getName(Id: id, Tys, M, FT));
807}
808
809// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
810#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
811#include "llvm/IR/IntrinsicImpl.inc"
812#undef GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
813
814// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
815#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
816#include "llvm/IR/IntrinsicImpl.inc"
817#undef GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
818
819bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
820 switch (QID) {
821#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
822 case Intrinsic::INTRINSIC:
823#include "llvm/IR/ConstrainedOps.def"
824#undef INSTRUCTION
825 return true;
826 default:
827 return false;
828 }
829}
830
831bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
832 switch (QID) {
833#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
834 case Intrinsic::INTRINSIC: \
835 return ROUND_MODE == 1;
836#include "llvm/IR/ConstrainedOps.def"
837#undef INSTRUCTION
838 default:
839 return false;
840 }
841}
842
843using DeferredIntrinsicMatchPair =
844 std::pair<Type *, ArrayRef<Intrinsic::IITDescriptor>>;
845
846static bool
847matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
848 SmallVectorImpl<Type *> &ArgTys,
849 SmallVectorImpl<DeferredIntrinsicMatchPair> &DeferredChecks,
850 bool IsDeferredCheck) {
851 using namespace Intrinsic;
852
853 // If we ran out of descriptors, there are too many arguments.
854 if (Infos.empty())
855 return true;
856
857 // Do this before slicing off the 'front' part
858 auto InfosRef = Infos;
859 auto DeferCheck = [&DeferredChecks, &InfosRef](Type *T) {
860 DeferredChecks.emplace_back(Args&: T, Args&: InfosRef);
861 return false;
862 };
863
864 IITDescriptor D = Infos.front();
865 Infos = Infos.slice(N: 1);
866
867 switch (D.Kind) {
868 case IITDescriptor::Void:
869 return !Ty->isVoidTy();
870 case IITDescriptor::VarArg:
871 return true;
872 case IITDescriptor::MMX: {
873 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
874 return !VT || VT->getNumElements() != 1 ||
875 !VT->getElementType()->isIntegerTy(Bitwidth: 64);
876 }
877 case IITDescriptor::AMX:
878 return !Ty->isX86_AMXTy();
879 case IITDescriptor::Token:
880 return !Ty->isTokenTy();
881 case IITDescriptor::Metadata:
882 return !Ty->isMetadataTy();
883 case IITDescriptor::Half:
884 return !Ty->isHalfTy();
885 case IITDescriptor::BFloat:
886 return !Ty->isBFloatTy();
887 case IITDescriptor::Float:
888 return !Ty->isFloatTy();
889 case IITDescriptor::Double:
890 return !Ty->isDoubleTy();
891 case IITDescriptor::Quad:
892 return !Ty->isFP128Ty();
893 case IITDescriptor::PPCQuad:
894 return !Ty->isPPC_FP128Ty();
895 case IITDescriptor::Integer:
896 return !Ty->isIntegerTy(Bitwidth: D.Integer_Width);
897 case IITDescriptor::AArch64Svcount:
898 return !isa<TargetExtType>(Val: Ty) ||
899 cast<TargetExtType>(Val: Ty)->getName() != "aarch64.svcount";
900 case IITDescriptor::Vector: {
901 VectorType *VT = dyn_cast<VectorType>(Val: Ty);
902 return !VT || VT->getElementCount() != D.Vector_Width ||
903 matchIntrinsicType(Ty: VT->getElementType(), Infos, ArgTys,
904 DeferredChecks, IsDeferredCheck);
905 }
906 case IITDescriptor::Pointer: {
907 PointerType *PT = dyn_cast<PointerType>(Val: Ty);
908 return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace;
909 }
910
911 case IITDescriptor::Struct: {
912 StructType *ST = dyn_cast<StructType>(Val: Ty);
913 if (!ST || !ST->isLiteral() || ST->isPacked() ||
914 ST->getNumElements() != D.Struct_NumElements)
915 return true;
916
917 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)
918 if (matchIntrinsicType(Ty: ST->getElementType(N: i), Infos, ArgTys,
919 DeferredChecks, IsDeferredCheck))
920 return true;
921 return false;
922 }
923
924 case IITDescriptor::Argument:
925 // If this is the second occurrence of an argument,
926 // verify that the later instance matches the previous instance.
927 if (D.getArgumentNumber() < ArgTys.size())
928 return Ty != ArgTys[D.getArgumentNumber()];
929
930 if (D.getArgumentNumber() > ArgTys.size() ||
931 D.getArgumentKind() == IITDescriptor::AK_MatchType)
932 return IsDeferredCheck || DeferCheck(Ty);
933
934 assert(D.getArgumentNumber() == ArgTys.size() && !IsDeferredCheck &&
935 "Table consistency error");
936 ArgTys.push_back(Elt: Ty);
937
938 switch (D.getArgumentKind()) {
939 case IITDescriptor::AK_Any:
940 return false; // Success
941 case IITDescriptor::AK_AnyInteger:
942 return !Ty->isIntOrIntVectorTy();
943 case IITDescriptor::AK_AnyFloat:
944 return !Ty->isFPOrFPVectorTy();
945 case IITDescriptor::AK_AnyVector:
946 return !isa<VectorType>(Val: Ty);
947 case IITDescriptor::AK_AnyPointer:
948 return !isa<PointerType>(Val: Ty);
949 default:
950 break;
951 }
952 llvm_unreachable("all argument kinds not covered");
953
954 case IITDescriptor::ExtendArgument: {
955 // If this is a forward reference, defer the check for later.
956 if (D.getArgumentNumber() >= ArgTys.size())
957 return IsDeferredCheck || DeferCheck(Ty);
958
959 Type *NewTy = ArgTys[D.getArgumentNumber()];
960 if (VectorType *VTy = dyn_cast<VectorType>(Val: NewTy))
961 NewTy = VectorType::getExtendedElementVectorType(VTy);
962 else if (IntegerType *ITy = dyn_cast<IntegerType>(Val: NewTy))
963 NewTy = IntegerType::get(C&: ITy->getContext(), NumBits: 2 * ITy->getBitWidth());
964 else
965 return true;
966
967 return Ty != NewTy;
968 }
969 case IITDescriptor::TruncArgument: {
970 // If this is a forward reference, defer the check for later.
971 if (D.getArgumentNumber() >= ArgTys.size())
972 return IsDeferredCheck || DeferCheck(Ty);
973
974 Type *NewTy = ArgTys[D.getArgumentNumber()];
975 if (VectorType *VTy = dyn_cast<VectorType>(Val: NewTy))
976 NewTy = VectorType::getTruncatedElementVectorType(VTy);
977 else if (IntegerType *ITy = dyn_cast<IntegerType>(Val: NewTy))
978 NewTy = IntegerType::get(C&: ITy->getContext(), NumBits: ITy->getBitWidth() / 2);
979 else
980 return true;
981
982 return Ty != NewTy;
983 }
984 case IITDescriptor::OneNthEltsVecArgument: {
985 // If this is a forward reference, defer the check for later.
986 if (D.getRefArgNumber() >= ArgTys.size())
987 return IsDeferredCheck || DeferCheck(Ty);
988 auto *VTy = dyn_cast<VectorType>(Val: ArgTys[D.getRefArgNumber()]);
989 if (!VTy)
990 return true;
991 if (!VTy->getElementCount().isKnownMultipleOf(RHS: D.getVectorDivisor()))
992 return true;
993 return VectorType::getOneNthElementsVectorType(VTy, Denominator: D.getVectorDivisor()) !=
994 Ty;
995 }
996 case IITDescriptor::SameVecWidthArgument: {
997 if (D.getArgumentNumber() >= ArgTys.size()) {
998 // Defer check and subsequent check for the vector element type.
999 Infos = Infos.slice(N: 1);
1000 return IsDeferredCheck || DeferCheck(Ty);
1001 }
1002 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1003 auto *ThisArgType = dyn_cast<VectorType>(Val: Ty);
1004 // Both must be vectors of the same number of elements or neither.
1005 if ((ReferenceType != nullptr) != (ThisArgType != nullptr))
1006 return true;
1007 Type *EltTy = Ty;
1008 if (ThisArgType) {
1009 if (ReferenceType->getElementCount() != ThisArgType->getElementCount())
1010 return true;
1011 EltTy = ThisArgType->getElementType();
1012 }
1013 return matchIntrinsicType(Ty: EltTy, Infos, ArgTys, DeferredChecks,
1014 IsDeferredCheck);
1015 }
1016 case IITDescriptor::VecOfAnyPtrsToElt: {
1017 unsigned RefArgNumber = D.getRefArgNumber();
1018 if (RefArgNumber >= ArgTys.size()) {
1019 if (IsDeferredCheck)
1020 return true;
1021 // If forward referencing, already add the pointer-vector type and
1022 // defer the checks for later.
1023 ArgTys.push_back(Elt: Ty);
1024 return DeferCheck(Ty);
1025 }
1026
1027 if (!IsDeferredCheck) {
1028 assert(D.getOverloadArgNumber() == ArgTys.size() &&
1029 "Table consistency error");
1030 ArgTys.push_back(Elt: Ty);
1031 }
1032
1033 // Verify the overloaded type "matches" the Ref type.
1034 // i.e. Ty is a vector with the same width as Ref.
1035 // Composed of pointers to the same element type as Ref.
1036 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[RefArgNumber]);
1037 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1038 if (!ThisArgVecTy || !ReferenceType ||
1039 (ReferenceType->getElementCount() != ThisArgVecTy->getElementCount()))
1040 return true;
1041 return !ThisArgVecTy->getElementType()->isPointerTy();
1042 }
1043 case IITDescriptor::VecElementArgument: {
1044 if (D.getArgumentNumber() >= ArgTys.size())
1045 return IsDeferredCheck ? true : DeferCheck(Ty);
1046 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1047 return !ReferenceType || Ty != ReferenceType->getElementType();
1048 }
1049 case IITDescriptor::Subdivide2Argument:
1050 case IITDescriptor::Subdivide4Argument: {
1051 // If this is a forward reference, defer the check for later.
1052 if (D.getArgumentNumber() >= ArgTys.size())
1053 return IsDeferredCheck || DeferCheck(Ty);
1054
1055 Type *NewTy = ArgTys[D.getArgumentNumber()];
1056 if (auto *VTy = dyn_cast<VectorType>(Val: NewTy)) {
1057 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
1058 NewTy = VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
1059 return Ty != NewTy;
1060 }
1061 return true;
1062 }
1063 case IITDescriptor::VecOfBitcastsToInt: {
1064 if (D.getArgumentNumber() >= ArgTys.size())
1065 return IsDeferredCheck || DeferCheck(Ty);
1066 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1067 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1068 if (!ThisArgVecTy || !ReferenceType)
1069 return true;
1070 return ThisArgVecTy != VectorType::getInteger(VTy: ReferenceType);
1071 }
1072 }
1073 llvm_unreachable("unhandled");
1074}
1075
1076Intrinsic::MatchIntrinsicTypesResult
1077Intrinsic::matchIntrinsicSignature(FunctionType *FTy,
1078 ArrayRef<Intrinsic::IITDescriptor> &Infos,
1079 SmallVectorImpl<Type *> &ArgTys) {
1080 SmallVector<DeferredIntrinsicMatchPair, 2> DeferredChecks;
1081 if (matchIntrinsicType(Ty: FTy->getReturnType(), Infos, ArgTys, DeferredChecks,
1082 IsDeferredCheck: false))
1083 return MatchIntrinsicTypes_NoMatchRet;
1084
1085 unsigned NumDeferredReturnChecks = DeferredChecks.size();
1086
1087 for (auto *Ty : FTy->params())
1088 if (matchIntrinsicType(Ty, Infos, ArgTys, DeferredChecks, IsDeferredCheck: false))
1089 return MatchIntrinsicTypes_NoMatchArg;
1090
1091 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
1092 DeferredIntrinsicMatchPair &Check = DeferredChecks[I];
1093 if (matchIntrinsicType(Ty: Check.first, Infos&: Check.second, ArgTys, DeferredChecks,
1094 IsDeferredCheck: true))
1095 return I < NumDeferredReturnChecks ? MatchIntrinsicTypes_NoMatchRet
1096 : MatchIntrinsicTypes_NoMatchArg;
1097 }
1098
1099 return MatchIntrinsicTypes_Match;
1100}
1101
1102bool Intrinsic::matchIntrinsicVarArg(
1103 bool isVarArg, ArrayRef<Intrinsic::IITDescriptor> &Infos) {
1104 // If there are no descriptors left, then it can't be a vararg.
1105 if (Infos.empty())
1106 return isVarArg;
1107
1108 // There should be only one descriptor remaining at this point.
1109 if (Infos.size() != 1)
1110 return true;
1111
1112 // Check and verify the descriptor.
1113 IITDescriptor D = Infos.front();
1114 Infos = Infos.slice(N: 1);
1115 if (D.Kind == IITDescriptor::VarArg)
1116 return !isVarArg;
1117
1118 return true;
1119}
1120
1121bool Intrinsic::getIntrinsicSignature(Intrinsic::ID ID, FunctionType *FT,
1122 SmallVectorImpl<Type *> &ArgTys) {
1123 if (!ID)
1124 return false;
1125
1126 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1127 getIntrinsicInfoTableEntries(id: ID, T&: Table);
1128 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
1129
1130 if (Intrinsic::matchIntrinsicSignature(FTy: FT, Infos&: TableRef, ArgTys) !=
1131 Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) {
1132 return false;
1133 }
1134 if (Intrinsic::matchIntrinsicVarArg(isVarArg: FT->isVarArg(), Infos&: TableRef))
1135 return false;
1136 return true;
1137}
1138
1139bool Intrinsic::getIntrinsicSignature(Function *F,
1140 SmallVectorImpl<Type *> &ArgTys) {
1141 return getIntrinsicSignature(ID: F->getIntrinsicID(), FT: F->getFunctionType(),
1142 ArgTys);
1143}
1144
1145std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
1146 SmallVector<Type *, 4> ArgTys;
1147 if (!getIntrinsicSignature(F, ArgTys))
1148 return std::nullopt;
1149
1150 Intrinsic::ID ID = F->getIntrinsicID();
1151 StringRef Name = F->getName();
1152 std::string WantedName =
1153 Intrinsic::getName(Id: ID, Tys: ArgTys, M: F->getParent(), FT: F->getFunctionType());
1154 if (Name == WantedName)
1155 return std::nullopt;
1156
1157 Function *NewDecl = [&] {
1158 if (auto *ExistingGV = F->getParent()->getNamedValue(Name: WantedName)) {
1159 if (auto *ExistingF = dyn_cast<Function>(Val: ExistingGV))
1160 if (ExistingF->getFunctionType() == F->getFunctionType())
1161 return ExistingF;
1162
1163 // The name already exists, but is not a function or has the wrong
1164 // prototype. Make place for the new one by renaming the old version.
1165 // Either this old version will be removed later on or the module is
1166 // invalid and we'll get an error.
1167 ExistingGV->setName(WantedName + ".renamed");
1168 }
1169 return Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, Tys: ArgTys);
1170 }();
1171
1172 NewDecl->setCallingConv(F->getCallingConv());
1173 assert(NewDecl->getFunctionType() == F->getFunctionType() &&
1174 "Shouldn't change the signature");
1175 return NewDecl;
1176}
1177
1178struct InterleaveIntrinsic {
1179 Intrinsic::ID Interleave, Deinterleave;
1180};
1181
1182static InterleaveIntrinsic InterleaveIntrinsics[] = {
1183 {.Interleave: Intrinsic::vector_interleave2, .Deinterleave: Intrinsic::vector_deinterleave2},
1184 {.Interleave: Intrinsic::vector_interleave3, .Deinterleave: Intrinsic::vector_deinterleave3},
1185 {.Interleave: Intrinsic::vector_interleave4, .Deinterleave: Intrinsic::vector_deinterleave4},
1186 {.Interleave: Intrinsic::vector_interleave5, .Deinterleave: Intrinsic::vector_deinterleave5},
1187 {.Interleave: Intrinsic::vector_interleave6, .Deinterleave: Intrinsic::vector_deinterleave6},
1188 {.Interleave: Intrinsic::vector_interleave7, .Deinterleave: Intrinsic::vector_deinterleave7},
1189 {.Interleave: Intrinsic::vector_interleave8, .Deinterleave: Intrinsic::vector_deinterleave8},
1190};
1191
1192Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
1193 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1194 return InterleaveIntrinsics[Factor - 2].Interleave;
1195}
1196
1197Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
1198 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1199 return InterleaveIntrinsics[Factor - 2].Deinterleave;
1200}
1201
1202#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1203#include "llvm/IR/IntrinsicImpl.inc"
1204#undef GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1205