1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions required for supporting intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Intrinsics.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringTable.h"
16#include "llvm/IR/ConstantRange.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/IR/IntrinsicsPowerPC.h"
27#include "llvm/IR/IntrinsicsR600.h"
28#include "llvm/IR/IntrinsicsRISCV.h"
29#include "llvm/IR/IntrinsicsS390.h"
30#include "llvm/IR/IntrinsicsSPIRV.h"
31#include "llvm/IR/IntrinsicsVE.h"
32#include "llvm/IR/IntrinsicsX86.h"
33#include "llvm/IR/IntrinsicsXCore.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NVVMIntrinsicUtils.h"
36#include "llvm/IR/Type.h"
37
38using namespace llvm;
39
40/// Table of string intrinsic names indexed by enum value.
41#define GET_INTRINSIC_NAME_TABLE
42#include "llvm/IR/IntrinsicImpl.inc"
43
44StringRef Intrinsic::getBaseName(ID id) {
45 assert(id < num_intrinsics && "Invalid intrinsic ID!");
46 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
47}
48
49StringRef Intrinsic::getName(ID id) {
50 assert(id < num_intrinsics && "Invalid intrinsic ID!");
51 assert(!Intrinsic::isOverloaded(id) &&
52 "This version of getName does not support overloading");
53 return getBaseName(id);
54}
55
56/// Returns a stable mangling for the type specified for use in the name
57/// mangling scheme used by 'any' types in intrinsic signatures. The mangling
58/// of named types is simply their name. Manglings for unnamed types consist
59/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
60/// combined with the mangling of their component types. A vararg function
61/// type will have a suffix of 'vararg'. Since function types can contain
62/// other function types, we close a function type mangling with suffix 'f'
63/// which can't be confused with it's prefix. This ensures we don't have
64/// collisions between two unrelated function types. Otherwise, you might
65/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)
66/// The HasUnnamedType boolean is set if an unnamed type was encountered,
67/// indicating that extra care must be taken to ensure a unique name.
68static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
69 std::string Result;
70 if (PointerType *PTyp = dyn_cast<PointerType>(Val: Ty)) {
71 Result += "p" + utostr(X: PTyp->getAddressSpace());
72 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Val: Ty)) {
73 Result += "a" + utostr(X: ATyp->getNumElements()) +
74 getMangledTypeStr(Ty: ATyp->getElementType(), HasUnnamedType);
75 } else if (StructType *STyp = dyn_cast<StructType>(Val: Ty)) {
76 if (!STyp->isLiteral()) {
77 Result += "s_";
78 if (STyp->hasName())
79 Result += STyp->getName();
80 else
81 HasUnnamedType = true;
82 } else {
83 Result += "sl_";
84 for (auto *Elem : STyp->elements())
85 Result += getMangledTypeStr(Ty: Elem, HasUnnamedType);
86 }
87 // Ensure nested structs are distinguishable.
88 Result += "s";
89 } else if (FunctionType *FT = dyn_cast<FunctionType>(Val: Ty)) {
90 Result += "f_" + getMangledTypeStr(Ty: FT->getReturnType(), HasUnnamedType);
91 for (size_t i = 0; i < FT->getNumParams(); i++)
92 Result += getMangledTypeStr(Ty: FT->getParamType(i), HasUnnamedType);
93 if (FT->isVarArg())
94 Result += "vararg";
95 // Ensure nested function types are distinguishable.
96 Result += "f";
97 } else if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
98 ElementCount EC = VTy->getElementCount();
99 if (EC.isScalable())
100 Result += "nx";
101 Result += "v" + utostr(X: EC.getKnownMinValue()) +
102 getMangledTypeStr(Ty: VTy->getElementType(), HasUnnamedType);
103 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Val: Ty)) {
104 Result += "t";
105 Result += TETy->getName();
106 for (Type *ParamTy : TETy->type_params())
107 Result += "_" + getMangledTypeStr(Ty: ParamTy, HasUnnamedType);
108 for (unsigned IntParam : TETy->int_params())
109 Result += "_" + utostr(X: IntParam);
110 // Ensure nested target extension types are distinguishable.
111 Result += "t";
112 } else if (Ty) {
113 switch (Ty->getTypeID()) {
114 default:
115 llvm_unreachable("Unhandled type");
116 case Type::VoidTyID:
117 Result += "isVoid";
118 break;
119 case Type::MetadataTyID:
120 Result += "Metadata";
121 break;
122 case Type::HalfTyID:
123 Result += "f16";
124 break;
125 case Type::BFloatTyID:
126 Result += "bf16";
127 break;
128 case Type::FloatTyID:
129 Result += "f32";
130 break;
131 case Type::DoubleTyID:
132 Result += "f64";
133 break;
134 case Type::X86_FP80TyID:
135 Result += "f80";
136 break;
137 case Type::FP128TyID:
138 Result += "f128";
139 break;
140 case Type::PPC_FP128TyID:
141 Result += "ppcf128";
142 break;
143 case Type::X86_AMXTyID:
144 Result += "x86amx";
145 break;
146 case Type::IntegerTyID:
147 Result += "i" + utostr(X: cast<IntegerType>(Val: Ty)->getBitWidth());
148 break;
149 }
150 }
151 return Result;
152}
153
154static std::string getIntrinsicNameImpl(Intrinsic::ID Id, ArrayRef<Type *> Tys,
155 Module *M, FunctionType *FT,
156 bool EarlyModuleCheck) {
157
158 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
159 assert((Tys.empty() || Intrinsic::isOverloaded(Id)) &&
160 "This version of getName is for overloaded intrinsics only");
161 (void)EarlyModuleCheck;
162 assert((!EarlyModuleCheck || M ||
163 !any_of(Tys, [](Type *T) { return isa<PointerType>(T); })) &&
164 "Intrinsic overloading on pointer types need to provide a Module");
165 bool HasUnnamedType = false;
166 std::string Result(Intrinsic::getBaseName(id: Id));
167 for (Type *Ty : Tys)
168 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
169 if (HasUnnamedType) {
170 assert(M && "unnamed types need a module");
171 if (!FT)
172 FT = Intrinsic::getType(Context&: M->getContext(), id: Id, Tys);
173 else
174 assert((FT == Intrinsic::getType(M->getContext(), Id, Tys)) &&
175 "Provided FunctionType must match arguments");
176 return M->getUniqueIntrinsicName(BaseName: Result, Id, Proto: FT);
177 }
178 return Result;
179}
180
181std::string Intrinsic::getName(ID Id, ArrayRef<Type *> Tys, Module *M,
182 FunctionType *FT) {
183 assert(M && "We need to have a Module");
184 return getIntrinsicNameImpl(Id, Tys, M, FT, EarlyModuleCheck: true);
185}
186
187std::string Intrinsic::getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys) {
188 return getIntrinsicNameImpl(Id, Tys, M: nullptr, FT: nullptr, EarlyModuleCheck: false);
189}
190
191/// IIT_Info - These are enumerators that describe the entries returned by the
192/// getIntrinsicInfoTableEntries function.
193///
194/// Defined in Intrinsics.td.
195enum IIT_Info {
196#define GET_INTRINSIC_IITINFO
197#include "llvm/IR/IntrinsicImpl.inc"
198};
199
200static void
201DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
202 IIT_Info LastInfo,
203 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
204 using namespace Intrinsic;
205
206 bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC);
207
208 IIT_Info Info = IIT_Info(Infos[NextElt++]);
209
210 switch (Info) {
211 case IIT_Done:
212 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Void, Field: 0));
213 return;
214 case IIT_VARARG:
215 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VarArg, Field: 0));
216 return;
217 case IIT_MMX:
218 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::MMX, Field: 0));
219 return;
220 case IIT_AMX:
221 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AMX, Field: 0));
222 return;
223 case IIT_TOKEN:
224 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Token, Field: 0));
225 return;
226 case IIT_METADATA:
227 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Metadata, Field: 0));
228 return;
229 case IIT_F16:
230 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Half, Field: 0));
231 return;
232 case IIT_BF16:
233 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::BFloat, Field: 0));
234 return;
235 case IIT_F32:
236 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Float, Field: 0));
237 return;
238 case IIT_F64:
239 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Double, Field: 0));
240 return;
241 case IIT_F128:
242 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Quad, Field: 0));
243 return;
244 case IIT_PPCF128:
245 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::PPCQuad, Field: 0));
246 return;
247 case IIT_I1:
248 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 1));
249 return;
250 case IIT_I2:
251 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 2));
252 return;
253 case IIT_I4:
254 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 4));
255 return;
256 case IIT_AARCH64_SVCOUNT:
257 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AArch64Svcount, Field: 0));
258 return;
259 case IIT_I8:
260 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 8));
261 return;
262 case IIT_I16:
263 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 16));
264 return;
265 case IIT_I32:
266 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 32));
267 return;
268 case IIT_I64:
269 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 64));
270 return;
271 case IIT_I128:
272 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 128));
273 return;
274 case IIT_V1:
275 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1, IsScalable: IsScalableVector));
276 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
277 return;
278 case IIT_V2:
279 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2, IsScalable: IsScalableVector));
280 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
281 return;
282 case IIT_V3:
283 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 3, IsScalable: IsScalableVector));
284 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
285 return;
286 case IIT_V4:
287 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4, IsScalable: IsScalableVector));
288 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
289 return;
290 case IIT_V6:
291 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 6, IsScalable: IsScalableVector));
292 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
293 return;
294 case IIT_V8:
295 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 8, IsScalable: IsScalableVector));
296 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
297 return;
298 case IIT_V10:
299 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 10, IsScalable: IsScalableVector));
300 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
301 return;
302 case IIT_V16:
303 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 16, IsScalable: IsScalableVector));
304 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
305 return;
306 case IIT_V32:
307 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 32, IsScalable: IsScalableVector));
308 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
309 return;
310 case IIT_V64:
311 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 64, IsScalable: IsScalableVector));
312 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
313 return;
314 case IIT_V128:
315 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 128, IsScalable: IsScalableVector));
316 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
317 return;
318 case IIT_V256:
319 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 256, IsScalable: IsScalableVector));
320 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
321 return;
322 case IIT_V512:
323 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 512, IsScalable: IsScalableVector));
324 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
325 return;
326 case IIT_V1024:
327 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1024, IsScalable: IsScalableVector));
328 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
329 return;
330 case IIT_V2048:
331 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2048, IsScalable: IsScalableVector));
332 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
333 return;
334 case IIT_V4096:
335 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4096, IsScalable: IsScalableVector));
336 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
337 return;
338 case IIT_EXTERNREF:
339 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 10));
340 return;
341 case IIT_FUNCREF:
342 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 20));
343 return;
344 case IIT_PTR:
345 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 0));
346 return;
347 case IIT_ANYPTR: // [ANYPTR addrspace]
348 OutputTable.push_back(
349 Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: Infos[NextElt++]));
350 return;
351 case IIT_ARG: {
352 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
353 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Argument, Field: ArgInfo));
354 return;
355 }
356 case IIT_EXTEND_ARG: {
357 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
358 OutputTable.push_back(
359 Elt: IITDescriptor::get(K: IITDescriptor::ExtendArgument, Field: ArgInfo));
360 return;
361 }
362 case IIT_TRUNC_ARG: {
363 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
364 OutputTable.push_back(
365 Elt: IITDescriptor::get(K: IITDescriptor::TruncArgument, Field: ArgInfo));
366 return;
367 }
368 case IIT_ONE_NTH_ELTS_VEC_ARG: {
369 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
370 unsigned short N = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
371 OutputTable.push_back(
372 Elt: IITDescriptor::get(K: IITDescriptor::OneNthEltsVecArgument, Hi: N, Lo: ArgNo));
373 return;
374 }
375 case IIT_SAME_VEC_WIDTH_ARG: {
376 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
377 OutputTable.push_back(
378 Elt: IITDescriptor::get(K: IITDescriptor::SameVecWidthArgument, Field: ArgInfo));
379 return;
380 }
381 case IIT_VEC_OF_ANYPTRS_TO_ELT: {
382 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
383 unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
384 OutputTable.push_back(
385 Elt: IITDescriptor::get(K: IITDescriptor::VecOfAnyPtrsToElt, Hi: ArgNo, Lo: RefNo));
386 return;
387 }
388 case IIT_EMPTYSTRUCT:
389 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: 0));
390 return;
391 case IIT_STRUCT: {
392 unsigned StructElts = Infos[NextElt++] + 2;
393
394 OutputTable.push_back(
395 Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: StructElts));
396
397 for (unsigned i = 0; i != StructElts; ++i)
398 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
399 return;
400 }
401 case IIT_SUBDIVIDE2_ARG: {
402 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
403 OutputTable.push_back(
404 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide2Argument, Field: ArgInfo));
405 return;
406 }
407 case IIT_SUBDIVIDE4_ARG: {
408 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
409 OutputTable.push_back(
410 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide4Argument, Field: ArgInfo));
411 return;
412 }
413 case IIT_VEC_ELEMENT: {
414 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
415 OutputTable.push_back(
416 Elt: IITDescriptor::get(K: IITDescriptor::VecElementArgument, Field: ArgInfo));
417 return;
418 }
419 case IIT_SCALABLE_VEC: {
420 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
421 return;
422 }
423 case IIT_VEC_OF_BITCASTS_TO_INT: {
424 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
425 OutputTable.push_back(
426 Elt: IITDescriptor::get(K: IITDescriptor::VecOfBitcastsToInt, Field: ArgInfo));
427 return;
428 }
429 }
430 llvm_unreachable("unhandled");
431}
432
433#define GET_INTRINSIC_GENERATOR_GLOBAL
434#include "llvm/IR/IntrinsicImpl.inc"
435
436void Intrinsic::getIntrinsicInfoTableEntries(
437 ID id, SmallVectorImpl<IITDescriptor> &T) {
438 // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
439 // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
440 // IntrinsicEmitter.cpp.
441 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
442 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
443 // Mask with all bits 1 except the most significant bit.
444 constexpr unsigned Mask = (1U << MSBPosition) - 1;
445
446 FixedEncodingTy TableVal = IIT_Table[id - 1];
447
448 // Array to hold the inlined fixed encoding values expanded from nibbles to
449 // bytes. Its size can be be atmost FixedEncodingBits / 4 i.e., number
450 // of nibbles that can fit in `FixedEncodingTy`.
451 unsigned char IITValues[FixedEncodingBits / 4];
452
453 ArrayRef<unsigned char> IITEntries;
454 unsigned NextElt = 0;
455 // Check to see if the intrinsic's type was inlined in the fixed encoding
456 // table.
457 if (TableVal >> MSBPosition) {
458 // This is an offset into the IIT_LongEncodingTable.
459 IITEntries = IIT_LongEncodingTable;
460
461 // Strip sentinel bit.
462 NextElt = TableVal & Mask;
463 } else {
464 // If the entry was encoded into a single word in the table itself, decode
465 // it from an array of nibbles to an array of bytes.
466 do {
467 IITValues[NextElt++] = TableVal & 0xF;
468 TableVal >>= 4;
469 } while (TableVal);
470
471 IITEntries = ArrayRef(IITValues).take_front(N: NextElt);
472 NextElt = 0;
473 }
474
475 // Okay, decode the table into the output vector of IITDescriptors.
476 DecodeIITType(NextElt, Infos: IITEntries, LastInfo: IIT_Done, OutputTable&: T);
477 while (NextElt != IITEntries.size() && IITEntries[NextElt] != 0)
478 DecodeIITType(NextElt, Infos: IITEntries, LastInfo: IIT_Done, OutputTable&: T);
479}
480
481static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
482 ArrayRef<Type *> Tys, LLVMContext &Context) {
483 using namespace Intrinsic;
484
485 IITDescriptor D = Infos.front();
486 Infos = Infos.slice(N: 1);
487
488 switch (D.Kind) {
489 case IITDescriptor::Void:
490 return Type::getVoidTy(C&: Context);
491 case IITDescriptor::VarArg:
492 return Type::getVoidTy(C&: Context);
493 case IITDescriptor::MMX:
494 return llvm::FixedVectorType::get(ElementType: llvm::IntegerType::get(C&: Context, NumBits: 64), NumElts: 1);
495 case IITDescriptor::AMX:
496 return Type::getX86_AMXTy(C&: Context);
497 case IITDescriptor::Token:
498 return Type::getTokenTy(C&: Context);
499 case IITDescriptor::Metadata:
500 return Type::getMetadataTy(C&: Context);
501 case IITDescriptor::Half:
502 return Type::getHalfTy(C&: Context);
503 case IITDescriptor::BFloat:
504 return Type::getBFloatTy(C&: Context);
505 case IITDescriptor::Float:
506 return Type::getFloatTy(C&: Context);
507 case IITDescriptor::Double:
508 return Type::getDoubleTy(C&: Context);
509 case IITDescriptor::Quad:
510 return Type::getFP128Ty(C&: Context);
511 case IITDescriptor::PPCQuad:
512 return Type::getPPC_FP128Ty(C&: Context);
513 case IITDescriptor::AArch64Svcount:
514 return TargetExtType::get(Context, Name: "aarch64.svcount");
515
516 case IITDescriptor::Integer:
517 return IntegerType::get(C&: Context, NumBits: D.Integer_Width);
518 case IITDescriptor::Vector:
519 return VectorType::get(ElementType: DecodeFixedType(Infos, Tys, Context),
520 EC: D.Vector_Width);
521 case IITDescriptor::Pointer:
522 return PointerType::get(C&: Context, AddressSpace: D.Pointer_AddressSpace);
523 case IITDescriptor::Struct: {
524 SmallVector<Type *, 8> Elts;
525 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)
526 Elts.push_back(Elt: DecodeFixedType(Infos, Tys, Context));
527 return StructType::get(Context, Elements: Elts);
528 }
529 case IITDescriptor::Argument:
530 return Tys[D.getArgumentNumber()];
531 case IITDescriptor::ExtendArgument: {
532 Type *Ty = Tys[D.getArgumentNumber()];
533 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
534 return VectorType::getExtendedElementVectorType(VTy);
535
536 return IntegerType::get(C&: Context, NumBits: 2 * cast<IntegerType>(Val: Ty)->getBitWidth());
537 }
538 case IITDescriptor::TruncArgument: {
539 Type *Ty = Tys[D.getArgumentNumber()];
540 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
541 return VectorType::getTruncatedElementVectorType(VTy);
542
543 IntegerType *ITy = cast<IntegerType>(Val: Ty);
544 assert(ITy->getBitWidth() % 2 == 0);
545 return IntegerType::get(C&: Context, NumBits: ITy->getBitWidth() / 2);
546 }
547 case IITDescriptor::Subdivide2Argument:
548 case IITDescriptor::Subdivide4Argument: {
549 Type *Ty = Tys[D.getArgumentNumber()];
550 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
551 assert(VTy && "Expected an argument of Vector Type");
552 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
553 return VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
554 }
555 case IITDescriptor::OneNthEltsVecArgument:
556 return VectorType::getOneNthElementsVectorType(
557 VTy: cast<VectorType>(Val: Tys[D.getRefArgNumber()]), Denominator: D.getVectorDivisor());
558 case IITDescriptor::SameVecWidthArgument: {
559 Type *EltTy = DecodeFixedType(Infos, Tys, Context);
560 Type *Ty = Tys[D.getArgumentNumber()];
561 if (auto *VTy = dyn_cast<VectorType>(Val: Ty))
562 return VectorType::get(ElementType: EltTy, EC: VTy->getElementCount());
563 return EltTy;
564 }
565 case IITDescriptor::VecElementArgument: {
566 Type *Ty = Tys[D.getArgumentNumber()];
567 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
568 return VTy->getElementType();
569 llvm_unreachable("Expected an argument of Vector Type");
570 }
571 case IITDescriptor::VecOfBitcastsToInt: {
572 Type *Ty = Tys[D.getArgumentNumber()];
573 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
574 assert(VTy && "Expected an argument of Vector Type");
575 return VectorType::getInteger(VTy);
576 }
577 case IITDescriptor::VecOfAnyPtrsToElt:
578 // Return the overloaded type (which determines the pointers address space)
579 return Tys[D.getOverloadArgNumber()];
580 }
581 llvm_unreachable("unhandled");
582}
583
584FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
585 ArrayRef<Type *> Tys) {
586 SmallVector<IITDescriptor, 8> Table;
587 getIntrinsicInfoTableEntries(id, T&: Table);
588
589 ArrayRef<IITDescriptor> TableRef = Table;
590 Type *ResultTy = DecodeFixedType(Infos&: TableRef, Tys, Context);
591
592 SmallVector<Type *, 8> ArgTys;
593 while (!TableRef.empty())
594 ArgTys.push_back(Elt: DecodeFixedType(Infos&: TableRef, Tys, Context));
595
596 // DecodeFixedType returns Void for IITDescriptor::Void and
597 // IITDescriptor::VarArg If we see void type as the type of the last argument,
598 // it is vararg intrinsic
599 if (!ArgTys.empty() && ArgTys.back()->isVoidTy()) {
600 ArgTys.pop_back();
601 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: true);
602 }
603 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
604}
605
606bool Intrinsic::isOverloaded(ID id) {
607#define GET_INTRINSIC_OVERLOAD_TABLE
608#include "llvm/IR/IntrinsicImpl.inc"
609}
610
611bool Intrinsic::hasPrettyPrintedArgs(ID id){
612#define GET_INTRINSIC_PRETTY_PRINT_TABLE
613#include "llvm/IR/IntrinsicImpl.inc"
614}
615
616/// Table of per-target intrinsic name tables.
617#define GET_INTRINSIC_TARGET_DATA
618#include "llvm/IR/IntrinsicImpl.inc"
619
620bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
621 return IID > TargetInfos[0].Count;
622}
623
624/// Looks up Name in NameTable via binary search. NameTable must be sorted
625/// and all entries must start with "llvm.". If NameTable contains an exact
626/// match for Name or a prefix of Name followed by a dot, its index in
627/// NameTable is returned. Otherwise, -1 is returned.
628static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
629 StringRef Name, StringRef Target = "") {
630 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
631 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
632
633 // Do successive binary searches of the dotted name components. For
634 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
635 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
636 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
637 // size 1. During the search, we can skip the prefix that we already know is
638 // identical. By using strncmp we consider names with differing suffixes to
639 // be part of the equal range.
640 size_t CmpEnd = 4; // Skip the "llvm" component.
641 if (!Target.empty())
642 CmpEnd += 1 + Target.size(); // skip the .target component.
643
644 const unsigned *Low = NameOffsetTable.begin();
645 const unsigned *High = NameOffsetTable.end();
646 const unsigned *LastLow = Low;
647 while (CmpEnd < Name.size() && High - Low > 0) {
648 size_t CmpStart = CmpEnd;
649 CmpEnd = Name.find(C: '.', From: CmpStart + 1);
650 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
651 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
652 // `equal_range` requires the comparison to work with either side being an
653 // offset or the value. Detect which kind each side is to set up the
654 // compared strings.
655 const char *LHSStr;
656 if constexpr (std::is_integral_v<decltype(LHS)>)
657 LHSStr = IntrinsicNameTable.getCString(O: LHS);
658 else
659 LHSStr = LHS;
660
661 const char *RHSStr;
662 if constexpr (std::is_integral_v<decltype(RHS)>)
663 RHSStr = IntrinsicNameTable.getCString(O: RHS);
664 else
665 RHSStr = RHS;
666
667 return strncmp(s1: LHSStr + CmpStart, s2: RHSStr + CmpStart, n: CmpEnd - CmpStart) <
668 0;
669 };
670 LastLow = Low;
671 std::tie(args&: Low, args&: High) = std::equal_range(first: Low, last: High, val: Name.data(), comp: Cmp);
672 }
673 if (High - Low > 0)
674 LastLow = Low;
675
676 if (LastLow == NameOffsetTable.end())
677 return -1;
678 StringRef NameFound = IntrinsicNameTable[*LastLow];
679 if (Name == NameFound ||
680 (Name.starts_with(Prefix: NameFound) && Name[NameFound.size()] == '.'))
681 return LastLow - NameOffsetTable.begin();
682 return -1;
683}
684
685/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
686/// target as \c Name, or the generic table if \c Name is not target specific.
687///
688/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
689/// name.
690static std::pair<ArrayRef<unsigned>, StringRef>
691findTargetSubtable(StringRef Name) {
692 assert(Name.starts_with("llvm."));
693
694 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
695 // Drop "llvm." and take the first dotted component. That will be the target
696 // if this is target specific.
697 StringRef Target = Name.drop_front(N: 5).split(Separator: '.').first;
698 auto It = partition_point(
699 Range&: Targets, P: [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
700 // We've either found the target or just fall back to the generic set, which
701 // is always first.
702 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
703 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
704 TI.Name};
705}
706
707/// This does the actual lookup of an intrinsic ID which matches the given
708/// function name.
709Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
710 auto [NameOffsetTable, Target] = findTargetSubtable(Name);
711 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
712 if (Idx == -1)
713 return Intrinsic::not_intrinsic;
714
715 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
716 // an index into a sub-table.
717 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
718 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
719
720 // If the intrinsic is not overloaded, require an exact match. If it is
721 // overloaded, require either exact or prefix match.
722 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
723 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
724 bool IsExactMatch = Name.size() == MatchSize;
725 return IsExactMatch || Intrinsic::isOverloaded(id: ID) ? ID
726 : Intrinsic::not_intrinsic;
727}
728
729/// This defines the "Intrinsic::getAttributes(ID id)" method.
730#define GET_INTRINSIC_ATTRIBUTES
731#include "llvm/IR/IntrinsicImpl.inc"
732
733static Function *getOrInsertIntrinsicDeclarationImpl(Module *M,
734 Intrinsic::ID id,
735 ArrayRef<Type *> Tys,
736 FunctionType *FT) {
737 Function *F = cast<Function>(
738 Val: M->getOrInsertFunction(Name: Tys.empty() ? Intrinsic::getName(id)
739 : Intrinsic::getName(Id: id, Tys, M, FT),
740 T: FT)
741 .getCallee());
742 if (F->getFunctionType() == FT)
743 return F;
744
745 // It's possible that a declaration for this intrinsic already exists with an
746 // incorrect signature, if the signature has changed, but this particular
747 // declaration has not been auto-upgraded yet. In that case, rename the
748 // invalid declaration and insert a new one with the correct signature. The
749 // invalid declaration will get upgraded later.
750 F->setName(F->getName() + ".invalid");
751 return cast<Function>(
752 Val: M->getOrInsertFunction(Name: Tys.empty() ? Intrinsic::getName(id)
753 : Intrinsic::getName(Id: id, Tys, M, FT),
754 T: FT)
755 .getCallee());
756}
757
758Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
759 ArrayRef<Type *> Tys) {
760 // There can never be multiple globals with the same name of different types,
761 // because intrinsics must be a specific type.
762 FunctionType *FT = getType(Context&: M->getContext(), id, Tys);
763 return getOrInsertIntrinsicDeclarationImpl(M, id, Tys, FT);
764}
765
766Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
767 ArrayRef<Type *> ArgTys) {
768 // If the intrinsic is not overloaded, use the non-overloaded version.
769 if (!Intrinsic::isOverloaded(id))
770 return getOrInsertDeclaration(M, id);
771
772 // Get the intrinsic signature metadata.
773 SmallVector<Intrinsic::IITDescriptor, 8> Table;
774 getIntrinsicInfoTableEntries(id, T&: Table);
775 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
776
777 FunctionType *FTy = FunctionType::get(Result: RetTy, Params: ArgTys, /*isVarArg=*/false);
778
779 // Automatically determine the overloaded types.
780 SmallVector<Type *, 4> OverloadTys;
781 [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult Res =
782 matchIntrinsicSignature(FTy, Infos&: TableRef, ArgTys&: OverloadTys);
783 assert(Res == Intrinsic::MatchIntrinsicTypes_Match &&
784 "intrinsic signature mismatch");
785
786 // If intrinsic requires vararg, recreate the FunctionType accordingly.
787 if (!matchIntrinsicVarArg(/*isVarArg=*/true, Infos&: TableRef))
788 FTy = FunctionType::get(Result: RetTy, Params: ArgTys, /*isVarArg=*/true);
789
790 assert(TableRef.empty() && "Unprocessed descriptors remain");
791
792 return getOrInsertIntrinsicDeclarationImpl(M, id, Tys: OverloadTys, FT: FTy);
793}
794
795Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
796 return M->getFunction(Name: getName(id));
797}
798
799Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
800 ArrayRef<Type *> Tys,
801 FunctionType *FT) {
802 return M->getFunction(Name: getName(Id: id, Tys, M, FT));
803}
804
805// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
806#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
807#include "llvm/IR/IntrinsicImpl.inc"
808
809// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
810#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
811#include "llvm/IR/IntrinsicImpl.inc"
812
813bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
814 switch (QID) {
815#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
816 case Intrinsic::INTRINSIC:
817#include "llvm/IR/ConstrainedOps.def"
818#undef INSTRUCTION
819 return true;
820 default:
821 return false;
822 }
823}
824
825bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
826 switch (QID) {
827#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
828 case Intrinsic::INTRINSIC: \
829 return ROUND_MODE == 1;
830#include "llvm/IR/ConstrainedOps.def"
831#undef INSTRUCTION
832 default:
833 return false;
834 }
835}
836
837using DeferredIntrinsicMatchPair =
838 std::pair<Type *, ArrayRef<Intrinsic::IITDescriptor>>;
839
840static bool
841matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
842 SmallVectorImpl<Type *> &ArgTys,
843 SmallVectorImpl<DeferredIntrinsicMatchPair> &DeferredChecks,
844 bool IsDeferredCheck) {
845 using namespace Intrinsic;
846
847 // If we ran out of descriptors, there are too many arguments.
848 if (Infos.empty())
849 return true;
850
851 // Do this before slicing off the 'front' part
852 auto InfosRef = Infos;
853 auto DeferCheck = [&DeferredChecks, &InfosRef](Type *T) {
854 DeferredChecks.emplace_back(Args&: T, Args&: InfosRef);
855 return false;
856 };
857
858 IITDescriptor D = Infos.front();
859 Infos = Infos.slice(N: 1);
860
861 switch (D.Kind) {
862 case IITDescriptor::Void:
863 return !Ty->isVoidTy();
864 case IITDescriptor::VarArg:
865 return true;
866 case IITDescriptor::MMX: {
867 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
868 return !VT || VT->getNumElements() != 1 ||
869 !VT->getElementType()->isIntegerTy(Bitwidth: 64);
870 }
871 case IITDescriptor::AMX:
872 return !Ty->isX86_AMXTy();
873 case IITDescriptor::Token:
874 return !Ty->isTokenTy();
875 case IITDescriptor::Metadata:
876 return !Ty->isMetadataTy();
877 case IITDescriptor::Half:
878 return !Ty->isHalfTy();
879 case IITDescriptor::BFloat:
880 return !Ty->isBFloatTy();
881 case IITDescriptor::Float:
882 return !Ty->isFloatTy();
883 case IITDescriptor::Double:
884 return !Ty->isDoubleTy();
885 case IITDescriptor::Quad:
886 return !Ty->isFP128Ty();
887 case IITDescriptor::PPCQuad:
888 return !Ty->isPPC_FP128Ty();
889 case IITDescriptor::Integer:
890 return !Ty->isIntegerTy(Bitwidth: D.Integer_Width);
891 case IITDescriptor::AArch64Svcount:
892 return !isa<TargetExtType>(Val: Ty) ||
893 cast<TargetExtType>(Val: Ty)->getName() != "aarch64.svcount";
894 case IITDescriptor::Vector: {
895 VectorType *VT = dyn_cast<VectorType>(Val: Ty);
896 return !VT || VT->getElementCount() != D.Vector_Width ||
897 matchIntrinsicType(Ty: VT->getElementType(), Infos, ArgTys,
898 DeferredChecks, IsDeferredCheck);
899 }
900 case IITDescriptor::Pointer: {
901 PointerType *PT = dyn_cast<PointerType>(Val: Ty);
902 return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace;
903 }
904
905 case IITDescriptor::Struct: {
906 StructType *ST = dyn_cast<StructType>(Val: Ty);
907 if (!ST || !ST->isLiteral() || ST->isPacked() ||
908 ST->getNumElements() != D.Struct_NumElements)
909 return true;
910
911 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)
912 if (matchIntrinsicType(Ty: ST->getElementType(N: i), Infos, ArgTys,
913 DeferredChecks, IsDeferredCheck))
914 return true;
915 return false;
916 }
917
918 case IITDescriptor::Argument:
919 // If this is the second occurrence of an argument,
920 // verify that the later instance matches the previous instance.
921 if (D.getArgumentNumber() < ArgTys.size())
922 return Ty != ArgTys[D.getArgumentNumber()];
923
924 if (D.getArgumentNumber() > ArgTys.size() ||
925 D.getArgumentKind() == IITDescriptor::AK_MatchType)
926 return IsDeferredCheck || DeferCheck(Ty);
927
928 assert(D.getArgumentNumber() == ArgTys.size() && !IsDeferredCheck &&
929 "Table consistency error");
930 ArgTys.push_back(Elt: Ty);
931
932 switch (D.getArgumentKind()) {
933 case IITDescriptor::AK_Any:
934 return false; // Success
935 case IITDescriptor::AK_AnyInteger:
936 return !Ty->isIntOrIntVectorTy();
937 case IITDescriptor::AK_AnyFloat:
938 return !Ty->isFPOrFPVectorTy();
939 case IITDescriptor::AK_AnyVector:
940 return !isa<VectorType>(Val: Ty);
941 case IITDescriptor::AK_AnyPointer:
942 return !isa<PointerType>(Val: Ty);
943 default:
944 break;
945 }
946 llvm_unreachable("all argument kinds not covered");
947
948 case IITDescriptor::ExtendArgument: {
949 // If this is a forward reference, defer the check for later.
950 if (D.getArgumentNumber() >= ArgTys.size())
951 return IsDeferredCheck || DeferCheck(Ty);
952
953 Type *NewTy = ArgTys[D.getArgumentNumber()];
954 if (VectorType *VTy = dyn_cast<VectorType>(Val: NewTy))
955 NewTy = VectorType::getExtendedElementVectorType(VTy);
956 else if (IntegerType *ITy = dyn_cast<IntegerType>(Val: NewTy))
957 NewTy = IntegerType::get(C&: ITy->getContext(), NumBits: 2 * ITy->getBitWidth());
958 else
959 return true;
960
961 return Ty != NewTy;
962 }
963 case IITDescriptor::TruncArgument: {
964 // If this is a forward reference, defer the check for later.
965 if (D.getArgumentNumber() >= ArgTys.size())
966 return IsDeferredCheck || DeferCheck(Ty);
967
968 Type *NewTy = ArgTys[D.getArgumentNumber()];
969 if (VectorType *VTy = dyn_cast<VectorType>(Val: NewTy))
970 NewTy = VectorType::getTruncatedElementVectorType(VTy);
971 else if (IntegerType *ITy = dyn_cast<IntegerType>(Val: NewTy))
972 NewTy = IntegerType::get(C&: ITy->getContext(), NumBits: ITy->getBitWidth() / 2);
973 else
974 return true;
975
976 return Ty != NewTy;
977 }
978 case IITDescriptor::OneNthEltsVecArgument: {
979 // If this is a forward reference, defer the check for later.
980 if (D.getRefArgNumber() >= ArgTys.size())
981 return IsDeferredCheck || DeferCheck(Ty);
982 auto *VTy = dyn_cast<VectorType>(Val: ArgTys[D.getRefArgNumber()]);
983 if (!VTy)
984 return true;
985 if (!VTy->getElementCount().isKnownMultipleOf(RHS: D.getVectorDivisor()))
986 return true;
987 return VectorType::getOneNthElementsVectorType(VTy, Denominator: D.getVectorDivisor()) !=
988 Ty;
989 }
990 case IITDescriptor::SameVecWidthArgument: {
991 if (D.getArgumentNumber() >= ArgTys.size()) {
992 // Defer check and subsequent check for the vector element type.
993 Infos = Infos.slice(N: 1);
994 return IsDeferredCheck || DeferCheck(Ty);
995 }
996 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
997 auto *ThisArgType = dyn_cast<VectorType>(Val: Ty);
998 // Both must be vectors of the same number of elements or neither.
999 if ((ReferenceType != nullptr) != (ThisArgType != nullptr))
1000 return true;
1001 Type *EltTy = Ty;
1002 if (ThisArgType) {
1003 if (ReferenceType->getElementCount() != ThisArgType->getElementCount())
1004 return true;
1005 EltTy = ThisArgType->getElementType();
1006 }
1007 return matchIntrinsicType(Ty: EltTy, Infos, ArgTys, DeferredChecks,
1008 IsDeferredCheck);
1009 }
1010 case IITDescriptor::VecOfAnyPtrsToElt: {
1011 unsigned RefArgNumber = D.getRefArgNumber();
1012 if (RefArgNumber >= ArgTys.size()) {
1013 if (IsDeferredCheck)
1014 return true;
1015 // If forward referencing, already add the pointer-vector type and
1016 // defer the checks for later.
1017 ArgTys.push_back(Elt: Ty);
1018 return DeferCheck(Ty);
1019 }
1020
1021 if (!IsDeferredCheck) {
1022 assert(D.getOverloadArgNumber() == ArgTys.size() &&
1023 "Table consistency error");
1024 ArgTys.push_back(Elt: Ty);
1025 }
1026
1027 // Verify the overloaded type "matches" the Ref type.
1028 // i.e. Ty is a vector with the same width as Ref.
1029 // Composed of pointers to the same element type as Ref.
1030 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[RefArgNumber]);
1031 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1032 if (!ThisArgVecTy || !ReferenceType ||
1033 (ReferenceType->getElementCount() != ThisArgVecTy->getElementCount()))
1034 return true;
1035 return !ThisArgVecTy->getElementType()->isPointerTy();
1036 }
1037 case IITDescriptor::VecElementArgument: {
1038 if (D.getArgumentNumber() >= ArgTys.size())
1039 return IsDeferredCheck ? true : DeferCheck(Ty);
1040 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1041 return !ReferenceType || Ty != ReferenceType->getElementType();
1042 }
1043 case IITDescriptor::Subdivide2Argument:
1044 case IITDescriptor::Subdivide4Argument: {
1045 // If this is a forward reference, defer the check for later.
1046 if (D.getArgumentNumber() >= ArgTys.size())
1047 return IsDeferredCheck || DeferCheck(Ty);
1048
1049 Type *NewTy = ArgTys[D.getArgumentNumber()];
1050 if (auto *VTy = dyn_cast<VectorType>(Val: NewTy)) {
1051 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
1052 NewTy = VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
1053 return Ty != NewTy;
1054 }
1055 return true;
1056 }
1057 case IITDescriptor::VecOfBitcastsToInt: {
1058 if (D.getArgumentNumber() >= ArgTys.size())
1059 return IsDeferredCheck || DeferCheck(Ty);
1060 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1061 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1062 if (!ThisArgVecTy || !ReferenceType)
1063 return true;
1064 return ThisArgVecTy != VectorType::getInteger(VTy: ReferenceType);
1065 }
1066 }
1067 llvm_unreachable("unhandled");
1068}
1069
1070Intrinsic::MatchIntrinsicTypesResult
1071Intrinsic::matchIntrinsicSignature(FunctionType *FTy,
1072 ArrayRef<Intrinsic::IITDescriptor> &Infos,
1073 SmallVectorImpl<Type *> &ArgTys) {
1074 SmallVector<DeferredIntrinsicMatchPair, 2> DeferredChecks;
1075 if (matchIntrinsicType(Ty: FTy->getReturnType(), Infos, ArgTys, DeferredChecks,
1076 IsDeferredCheck: false))
1077 return MatchIntrinsicTypes_NoMatchRet;
1078
1079 unsigned NumDeferredReturnChecks = DeferredChecks.size();
1080
1081 for (auto *Ty : FTy->params())
1082 if (matchIntrinsicType(Ty, Infos, ArgTys, DeferredChecks, IsDeferredCheck: false))
1083 return MatchIntrinsicTypes_NoMatchArg;
1084
1085 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
1086 DeferredIntrinsicMatchPair &Check = DeferredChecks[I];
1087 if (matchIntrinsicType(Ty: Check.first, Infos&: Check.second, ArgTys, DeferredChecks,
1088 IsDeferredCheck: true))
1089 return I < NumDeferredReturnChecks ? MatchIntrinsicTypes_NoMatchRet
1090 : MatchIntrinsicTypes_NoMatchArg;
1091 }
1092
1093 return MatchIntrinsicTypes_Match;
1094}
1095
1096bool Intrinsic::matchIntrinsicVarArg(
1097 bool isVarArg, ArrayRef<Intrinsic::IITDescriptor> &Infos) {
1098 // If there are no descriptors left, then it can't be a vararg.
1099 if (Infos.empty())
1100 return isVarArg;
1101
1102 // There should be only one descriptor remaining at this point.
1103 if (Infos.size() != 1)
1104 return true;
1105
1106 // Check and verify the descriptor.
1107 IITDescriptor D = Infos.front();
1108 Infos = Infos.slice(N: 1);
1109 if (D.Kind == IITDescriptor::VarArg)
1110 return !isVarArg;
1111
1112 return true;
1113}
1114
1115bool Intrinsic::getIntrinsicSignature(Intrinsic::ID ID, FunctionType *FT,
1116 SmallVectorImpl<Type *> &ArgTys) {
1117 if (!ID)
1118 return false;
1119
1120 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1121 getIntrinsicInfoTableEntries(id: ID, T&: Table);
1122 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
1123
1124 if (Intrinsic::matchIntrinsicSignature(FTy: FT, Infos&: TableRef, ArgTys) !=
1125 Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) {
1126 return false;
1127 }
1128 if (Intrinsic::matchIntrinsicVarArg(isVarArg: FT->isVarArg(), Infos&: TableRef))
1129 return false;
1130 return true;
1131}
1132
1133bool Intrinsic::getIntrinsicSignature(Function *F,
1134 SmallVectorImpl<Type *> &ArgTys) {
1135 return getIntrinsicSignature(ID: F->getIntrinsicID(), FT: F->getFunctionType(),
1136 ArgTys);
1137}
1138
1139std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
1140 SmallVector<Type *, 4> ArgTys;
1141 if (!getIntrinsicSignature(F, ArgTys))
1142 return std::nullopt;
1143
1144 Intrinsic::ID ID = F->getIntrinsicID();
1145 StringRef Name = F->getName();
1146 std::string WantedName =
1147 Intrinsic::getName(Id: ID, Tys: ArgTys, M: F->getParent(), FT: F->getFunctionType());
1148 if (Name == WantedName)
1149 return std::nullopt;
1150
1151 Function *NewDecl = [&] {
1152 if (auto *ExistingGV = F->getParent()->getNamedValue(Name: WantedName)) {
1153 if (auto *ExistingF = dyn_cast<Function>(Val: ExistingGV))
1154 if (ExistingF->getFunctionType() == F->getFunctionType())
1155 return ExistingF;
1156
1157 // The name already exists, but is not a function or has the wrong
1158 // prototype. Make place for the new one by renaming the old version.
1159 // Either this old version will be removed later on or the module is
1160 // invalid and we'll get an error.
1161 ExistingGV->setName(WantedName + ".renamed");
1162 }
1163 return Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, Tys: ArgTys);
1164 }();
1165
1166 NewDecl->setCallingConv(F->getCallingConv());
1167 assert(NewDecl->getFunctionType() == F->getFunctionType() &&
1168 "Shouldn't change the signature");
1169 return NewDecl;
1170}
1171
1172struct InterleaveIntrinsic {
1173 Intrinsic::ID Interleave, Deinterleave;
1174};
1175
1176static InterleaveIntrinsic InterleaveIntrinsics[] = {
1177 {.Interleave: Intrinsic::vector_interleave2, .Deinterleave: Intrinsic::vector_deinterleave2},
1178 {.Interleave: Intrinsic::vector_interleave3, .Deinterleave: Intrinsic::vector_deinterleave3},
1179 {.Interleave: Intrinsic::vector_interleave4, .Deinterleave: Intrinsic::vector_deinterleave4},
1180 {.Interleave: Intrinsic::vector_interleave5, .Deinterleave: Intrinsic::vector_deinterleave5},
1181 {.Interleave: Intrinsic::vector_interleave6, .Deinterleave: Intrinsic::vector_deinterleave6},
1182 {.Interleave: Intrinsic::vector_interleave7, .Deinterleave: Intrinsic::vector_deinterleave7},
1183 {.Interleave: Intrinsic::vector_interleave8, .Deinterleave: Intrinsic::vector_deinterleave8},
1184};
1185
1186Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
1187 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1188 return InterleaveIntrinsics[Factor - 2].Interleave;
1189}
1190
1191Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
1192 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1193 return InterleaveIntrinsics[Factor - 2].Deinterleave;
1194}
1195
1196#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1197#include "llvm/IR/IntrinsicImpl.inc"
1198