1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions required for supporting intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Intrinsics.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringTable.h"
16#include "llvm/IR/ConstantRange.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/IR/IntrinsicsPowerPC.h"
27#include "llvm/IR/IntrinsicsR600.h"
28#include "llvm/IR/IntrinsicsRISCV.h"
29#include "llvm/IR/IntrinsicsS390.h"
30#include "llvm/IR/IntrinsicsSPIRV.h"
31#include "llvm/IR/IntrinsicsVE.h"
32#include "llvm/IR/IntrinsicsX86.h"
33#include "llvm/IR/IntrinsicsXCore.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NVVMIntrinsicUtils.h"
36#include "llvm/IR/Type.h"
37
38using namespace llvm;
39
40/// Table of string intrinsic names indexed by enum value.
41#define GET_INTRINSIC_NAME_TABLE
42#include "llvm/IR/IntrinsicImpl.inc"
43
44StringRef Intrinsic::getBaseName(ID id) {
45 assert(id < num_intrinsics && "Invalid intrinsic ID!");
46 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
47}
48
49StringRef Intrinsic::getName(ID id) {
50 assert(id < num_intrinsics && "Invalid intrinsic ID!");
51 assert(!Intrinsic::isOverloaded(id) &&
52 "This version of getName does not support overloading");
53 return getBaseName(id);
54}
55
56/// Returns a stable mangling for the type specified for use in the name
57/// mangling scheme used by 'any' types in intrinsic signatures. The mangling
58/// of named types is simply their name. Manglings for unnamed types consist
59/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
60/// combined with the mangling of their component types. A vararg function
61/// type will have a suffix of 'vararg'. Since function types can contain
62/// other function types, we close a function type mangling with suffix 'f'
63/// which can't be confused with it's prefix. This ensures we don't have
64/// collisions between two unrelated function types. Otherwise, you might
65/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)
66/// The HasUnnamedType boolean is set if an unnamed type was encountered,
67/// indicating that extra care must be taken to ensure a unique name.
68static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
69 std::string Result;
70 if (PointerType *PTyp = dyn_cast<PointerType>(Val: Ty)) {
71 Result += "p" + utostr(X: PTyp->getAddressSpace());
72 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Val: Ty)) {
73 Result += "a" + utostr(X: ATyp->getNumElements()) +
74 getMangledTypeStr(Ty: ATyp->getElementType(), HasUnnamedType);
75 } else if (StructType *STyp = dyn_cast<StructType>(Val: Ty)) {
76 if (!STyp->isLiteral()) {
77 Result += "s_";
78 if (STyp->hasName())
79 Result += STyp->getName();
80 else
81 HasUnnamedType = true;
82 } else {
83 Result += "sl_";
84 for (auto *Elem : STyp->elements())
85 Result += getMangledTypeStr(Ty: Elem, HasUnnamedType);
86 }
87 // Ensure nested structs are distinguishable.
88 Result += "s";
89 } else if (FunctionType *FT = dyn_cast<FunctionType>(Val: Ty)) {
90 Result += "f_" + getMangledTypeStr(Ty: FT->getReturnType(), HasUnnamedType);
91 for (size_t i = 0; i < FT->getNumParams(); i++)
92 Result += getMangledTypeStr(Ty: FT->getParamType(i), HasUnnamedType);
93 if (FT->isVarArg())
94 Result += "vararg";
95 // Ensure nested function types are distinguishable.
96 Result += "f";
97 } else if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
98 ElementCount EC = VTy->getElementCount();
99 if (EC.isScalable())
100 Result += "nx";
101 Result += "v" + utostr(X: EC.getKnownMinValue()) +
102 getMangledTypeStr(Ty: VTy->getElementType(), HasUnnamedType);
103 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Val: Ty)) {
104 Result += "t";
105 Result += TETy->getName();
106 for (Type *ParamTy : TETy->type_params())
107 Result += "_" + getMangledTypeStr(Ty: ParamTy, HasUnnamedType);
108 for (unsigned IntParam : TETy->int_params())
109 Result += "_" + utostr(X: IntParam);
110 // Ensure nested target extension types are distinguishable.
111 Result += "t";
112 } else if (Ty) {
113 switch (Ty->getTypeID()) {
114 default:
115 llvm_unreachable("Unhandled type");
116 case Type::VoidTyID:
117 Result += "isVoid";
118 break;
119 case Type::MetadataTyID:
120 Result += "Metadata";
121 break;
122 case Type::HalfTyID:
123 Result += "f16";
124 break;
125 case Type::BFloatTyID:
126 Result += "bf16";
127 break;
128 case Type::FloatTyID:
129 Result += "f32";
130 break;
131 case Type::DoubleTyID:
132 Result += "f64";
133 break;
134 case Type::X86_FP80TyID:
135 Result += "f80";
136 break;
137 case Type::FP128TyID:
138 Result += "f128";
139 break;
140 case Type::PPC_FP128TyID:
141 Result += "ppcf128";
142 break;
143 case Type::X86_AMXTyID:
144 Result += "x86amx";
145 break;
146 case Type::IntegerTyID:
147 Result += "i" + utostr(X: cast<IntegerType>(Val: Ty)->getBitWidth());
148 break;
149 case Type::ByteTyID:
150 Result += "b" + utostr(X: cast<ByteType>(Val: Ty)->getBitWidth());
151 break;
152 }
153 }
154 return Result;
155}
156
157static std::string getIntrinsicNameImpl(Intrinsic::ID Id, ArrayRef<Type *> Tys,
158 Module *M, FunctionType *FT,
159 bool EarlyModuleCheck) {
160
161 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
162 assert((Tys.empty() || Intrinsic::isOverloaded(Id)) &&
163 "This version of getName is for overloaded intrinsics only");
164 (void)EarlyModuleCheck;
165 assert((!EarlyModuleCheck || M ||
166 !any_of(Tys, [](Type *T) { return isa<PointerType>(T); })) &&
167 "Intrinsic overloading on pointer types need to provide a Module");
168 bool HasUnnamedType = false;
169 std::string Result(Intrinsic::getBaseName(id: Id));
170 for (Type *Ty : Tys)
171 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
172 if (HasUnnamedType) {
173 assert(M && "unnamed types need a module");
174 if (!FT)
175 FT = Intrinsic::getType(Context&: M->getContext(), id: Id, Tys);
176 else
177 assert((FT == Intrinsic::getType(M->getContext(), Id, Tys)) &&
178 "Provided FunctionType must match arguments");
179 return M->getUniqueIntrinsicName(BaseName: Result, Id, Proto: FT);
180 }
181 return Result;
182}
183
184std::string Intrinsic::getName(ID Id, ArrayRef<Type *> Tys, Module *M,
185 FunctionType *FT) {
186 assert(M && "We need to have a Module");
187 return getIntrinsicNameImpl(Id, Tys, M, FT, EarlyModuleCheck: true);
188}
189
190std::string Intrinsic::getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys) {
191 return getIntrinsicNameImpl(Id, Tys, M: nullptr, FT: nullptr, EarlyModuleCheck: false);
192}
193
194/// IIT_Info - These are enumerators that describe the entries returned by the
195/// getIntrinsicInfoTableEntries function.
196///
197/// Defined in Intrinsics.td.
198enum IIT_Info {
199#define GET_INTRINSIC_IITINFO
200#include "llvm/IR/IntrinsicImpl.inc"
201};
202
203static void
204DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
205 IIT_Info LastInfo,
206 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
207 using namespace Intrinsic;
208
209 bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC);
210
211 IIT_Info Info = IIT_Info(Infos[NextElt++]);
212
213 switch (Info) {
214 case IIT_Done:
215 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Void, Field: 0));
216 return;
217 case IIT_VARARG:
218 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VarArg, Field: 0));
219 return;
220 case IIT_MMX:
221 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::MMX, Field: 0));
222 return;
223 case IIT_AMX:
224 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AMX, Field: 0));
225 return;
226 case IIT_TOKEN:
227 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Token, Field: 0));
228 return;
229 case IIT_METADATA:
230 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Metadata, Field: 0));
231 return;
232 case IIT_F16:
233 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Half, Field: 0));
234 return;
235 case IIT_BF16:
236 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::BFloat, Field: 0));
237 return;
238 case IIT_F32:
239 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Float, Field: 0));
240 return;
241 case IIT_F64:
242 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Double, Field: 0));
243 return;
244 case IIT_F128:
245 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Quad, Field: 0));
246 return;
247 case IIT_PPCF128:
248 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::PPCQuad, Field: 0));
249 return;
250 case IIT_I1:
251 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 1));
252 return;
253 case IIT_I2:
254 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 2));
255 return;
256 case IIT_I4:
257 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 4));
258 return;
259 case IIT_AARCH64_SVCOUNT:
260 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AArch64Svcount, Field: 0));
261 return;
262 case IIT_I8:
263 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 8));
264 return;
265 case IIT_I16:
266 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 16));
267 return;
268 case IIT_I32:
269 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 32));
270 return;
271 case IIT_I64:
272 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 64));
273 return;
274 case IIT_I128:
275 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 128));
276 return;
277 case IIT_V1:
278 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1, IsScalable: IsScalableVector));
279 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
280 return;
281 case IIT_V2:
282 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2, IsScalable: IsScalableVector));
283 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
284 return;
285 case IIT_V3:
286 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 3, IsScalable: IsScalableVector));
287 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
288 return;
289 case IIT_V4:
290 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4, IsScalable: IsScalableVector));
291 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
292 return;
293 case IIT_V6:
294 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 6, IsScalable: IsScalableVector));
295 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
296 return;
297 case IIT_V8:
298 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 8, IsScalable: IsScalableVector));
299 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
300 return;
301 case IIT_V10:
302 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 10, IsScalable: IsScalableVector));
303 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
304 return;
305 case IIT_V16:
306 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 16, IsScalable: IsScalableVector));
307 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
308 return;
309 case IIT_V32:
310 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 32, IsScalable: IsScalableVector));
311 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
312 return;
313 case IIT_V64:
314 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 64, IsScalable: IsScalableVector));
315 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
316 return;
317 case IIT_V128:
318 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 128, IsScalable: IsScalableVector));
319 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
320 return;
321 case IIT_V256:
322 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 256, IsScalable: IsScalableVector));
323 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
324 return;
325 case IIT_V512:
326 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 512, IsScalable: IsScalableVector));
327 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
328 return;
329 case IIT_V1024:
330 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1024, IsScalable: IsScalableVector));
331 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
332 return;
333 case IIT_V2048:
334 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2048, IsScalable: IsScalableVector));
335 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
336 return;
337 case IIT_V4096:
338 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4096, IsScalable: IsScalableVector));
339 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
340 return;
341 case IIT_EXTERNREF:
342 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 10));
343 return;
344 case IIT_FUNCREF:
345 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 20));
346 return;
347 case IIT_PTR:
348 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 0));
349 return;
350 case IIT_ANYPTR: // [ANYPTR addrspace]
351 OutputTable.push_back(
352 Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: Infos[NextElt++]));
353 return;
354 case IIT_ARG: {
355 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
356 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Argument, Field: ArgInfo));
357 return;
358 }
359 case IIT_EXTEND_ARG: {
360 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
361 OutputTable.push_back(
362 Elt: IITDescriptor::get(K: IITDescriptor::ExtendArgument, Field: ArgInfo));
363 return;
364 }
365 case IIT_TRUNC_ARG: {
366 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
367 OutputTable.push_back(
368 Elt: IITDescriptor::get(K: IITDescriptor::TruncArgument, Field: ArgInfo));
369 return;
370 }
371 case IIT_ONE_NTH_ELTS_VEC_ARG: {
372 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
373 unsigned short N = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
374 OutputTable.push_back(
375 Elt: IITDescriptor::get(K: IITDescriptor::OneNthEltsVecArgument, Hi: N, Lo: ArgNo));
376 return;
377 }
378 case IIT_SAME_VEC_WIDTH_ARG: {
379 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
380 OutputTable.push_back(
381 Elt: IITDescriptor::get(K: IITDescriptor::SameVecWidthArgument, Field: ArgInfo));
382 return;
383 }
384 case IIT_VEC_OF_ANYPTRS_TO_ELT: {
385 unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
386 unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
387 OutputTable.push_back(
388 Elt: IITDescriptor::get(K: IITDescriptor::VecOfAnyPtrsToElt, Hi: ArgNo, Lo: RefNo));
389 return;
390 }
391 case IIT_EMPTYSTRUCT:
392 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: 0));
393 return;
394 case IIT_STRUCT: {
395 unsigned StructElts = Infos[NextElt++] + 2;
396
397 OutputTable.push_back(
398 Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: StructElts));
399
400 for (unsigned i = 0; i != StructElts; ++i)
401 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
402 return;
403 }
404 case IIT_SUBDIVIDE2_ARG: {
405 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
406 OutputTable.push_back(
407 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide2Argument, Field: ArgInfo));
408 return;
409 }
410 case IIT_SUBDIVIDE4_ARG: {
411 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
412 OutputTable.push_back(
413 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide4Argument, Field: ArgInfo));
414 return;
415 }
416 case IIT_VEC_ELEMENT: {
417 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
418 OutputTable.push_back(
419 Elt: IITDescriptor::get(K: IITDescriptor::VecElementArgument, Field: ArgInfo));
420 return;
421 }
422 case IIT_SCALABLE_VEC: {
423 DecodeIITType(NextElt, Infos, LastInfo: Info, OutputTable);
424 return;
425 }
426 case IIT_VEC_OF_BITCASTS_TO_INT: {
427 unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
428 OutputTable.push_back(
429 Elt: IITDescriptor::get(K: IITDescriptor::VecOfBitcastsToInt, Field: ArgInfo));
430 return;
431 }
432 }
433 llvm_unreachable("unhandled");
434}
435
436#define GET_INTRINSIC_GENERATOR_GLOBAL
437#include "llvm/IR/IntrinsicImpl.inc"
438
439void Intrinsic::getIntrinsicInfoTableEntries(
440 ID id, SmallVectorImpl<IITDescriptor> &T) {
441 // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
442 // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
443 // IntrinsicEmitter.cpp.
444 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
445 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
446 // Mask with all bits 1 except the most significant bit.
447 constexpr unsigned Mask = (1U << MSBPosition) - 1;
448
449 FixedEncodingTy TableVal = IIT_Table[id - 1];
450
451 // Array to hold the inlined fixed encoding values expanded from nibbles to
452 // bytes. Its size can be be atmost FixedEncodingBits / 4 i.e., number
453 // of nibbles that can fit in `FixedEncodingTy`.
454 unsigned char IITValues[FixedEncodingBits / 4];
455
456 ArrayRef<unsigned char> IITEntries;
457 unsigned NextElt = 0;
458 // Check to see if the intrinsic's type was inlined in the fixed encoding
459 // table.
460 if (TableVal >> MSBPosition) {
461 // This is an offset into the IIT_LongEncodingTable.
462 IITEntries = IIT_LongEncodingTable;
463
464 // Strip sentinel bit.
465 NextElt = TableVal & Mask;
466 } else {
467 // If the entry was encoded into a single word in the table itself, decode
468 // it from an array of nibbles to an array of bytes.
469 do {
470 IITValues[NextElt++] = TableVal & 0xF;
471 TableVal >>= 4;
472 } while (TableVal);
473
474 IITEntries = ArrayRef(IITValues).take_front(N: NextElt);
475 NextElt = 0;
476 }
477
478 // Okay, decode the table into the output vector of IITDescriptors.
479 DecodeIITType(NextElt, Infos: IITEntries, LastInfo: IIT_Done, OutputTable&: T);
480 while (NextElt != IITEntries.size() && IITEntries[NextElt] != 0)
481 DecodeIITType(NextElt, Infos: IITEntries, LastInfo: IIT_Done, OutputTable&: T);
482}
483
484static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
485 ArrayRef<Type *> Tys, LLVMContext &Context) {
486 using namespace Intrinsic;
487
488 IITDescriptor D = Infos.front();
489 Infos = Infos.slice(N: 1);
490
491 switch (D.Kind) {
492 case IITDescriptor::Void:
493 return Type::getVoidTy(C&: Context);
494 case IITDescriptor::VarArg:
495 return Type::getVoidTy(C&: Context);
496 case IITDescriptor::MMX:
497 return llvm::FixedVectorType::get(ElementType: llvm::IntegerType::get(C&: Context, NumBits: 64), NumElts: 1);
498 case IITDescriptor::AMX:
499 return Type::getX86_AMXTy(C&: Context);
500 case IITDescriptor::Token:
501 return Type::getTokenTy(C&: Context);
502 case IITDescriptor::Metadata:
503 return Type::getMetadataTy(C&: Context);
504 case IITDescriptor::Half:
505 return Type::getHalfTy(C&: Context);
506 case IITDescriptor::BFloat:
507 return Type::getBFloatTy(C&: Context);
508 case IITDescriptor::Float:
509 return Type::getFloatTy(C&: Context);
510 case IITDescriptor::Double:
511 return Type::getDoubleTy(C&: Context);
512 case IITDescriptor::Quad:
513 return Type::getFP128Ty(C&: Context);
514 case IITDescriptor::PPCQuad:
515 return Type::getPPC_FP128Ty(C&: Context);
516 case IITDescriptor::AArch64Svcount:
517 return TargetExtType::get(Context, Name: "aarch64.svcount");
518
519 case IITDescriptor::Integer:
520 return IntegerType::get(C&: Context, NumBits: D.Integer_Width);
521 case IITDescriptor::Vector:
522 return VectorType::get(ElementType: DecodeFixedType(Infos, Tys, Context),
523 EC: D.Vector_Width);
524 case IITDescriptor::Pointer:
525 return PointerType::get(C&: Context, AddressSpace: D.Pointer_AddressSpace);
526 case IITDescriptor::Struct: {
527 SmallVector<Type *, 8> Elts;
528 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)
529 Elts.push_back(Elt: DecodeFixedType(Infos, Tys, Context));
530 return StructType::get(Context, Elements: Elts);
531 }
532 case IITDescriptor::Argument:
533 return Tys[D.getArgumentNumber()];
534 case IITDescriptor::ExtendArgument: {
535 Type *Ty = Tys[D.getArgumentNumber()];
536 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
537 return VectorType::getExtendedElementVectorType(VTy);
538
539 return IntegerType::get(C&: Context, NumBits: 2 * cast<IntegerType>(Val: Ty)->getBitWidth());
540 }
541 case IITDescriptor::TruncArgument: {
542 Type *Ty = Tys[D.getArgumentNumber()];
543 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
544 return VectorType::getTruncatedElementVectorType(VTy);
545
546 IntegerType *ITy = cast<IntegerType>(Val: Ty);
547 assert(ITy->getBitWidth() % 2 == 0);
548 return IntegerType::get(C&: Context, NumBits: ITy->getBitWidth() / 2);
549 }
550 case IITDescriptor::Subdivide2Argument:
551 case IITDescriptor::Subdivide4Argument: {
552 Type *Ty = Tys[D.getArgumentNumber()];
553 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
554 assert(VTy && "Expected an argument of Vector Type");
555 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
556 return VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
557 }
558 case IITDescriptor::OneNthEltsVecArgument:
559 return VectorType::getOneNthElementsVectorType(
560 VTy: cast<VectorType>(Val: Tys[D.getRefArgNumber()]), Denominator: D.getVectorDivisor());
561 case IITDescriptor::SameVecWidthArgument: {
562 Type *EltTy = DecodeFixedType(Infos, Tys, Context);
563 Type *Ty = Tys[D.getArgumentNumber()];
564 if (auto *VTy = dyn_cast<VectorType>(Val: Ty))
565 return VectorType::get(ElementType: EltTy, EC: VTy->getElementCount());
566 return EltTy;
567 }
568 case IITDescriptor::VecElementArgument: {
569 Type *Ty = Tys[D.getArgumentNumber()];
570 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
571 return VTy->getElementType();
572 llvm_unreachable("Expected an argument of Vector Type");
573 }
574 case IITDescriptor::VecOfBitcastsToInt: {
575 Type *Ty = Tys[D.getArgumentNumber()];
576 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
577 assert(VTy && "Expected an argument of Vector Type");
578 return VectorType::getInteger(VTy);
579 }
580 case IITDescriptor::VecOfAnyPtrsToElt:
581 // Return the overloaded type (which determines the pointers address space)
582 return Tys[D.getOverloadArgNumber()];
583 }
584 llvm_unreachable("unhandled");
585}
586
587FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
588 ArrayRef<Type *> Tys) {
589 SmallVector<IITDescriptor, 8> Table;
590 getIntrinsicInfoTableEntries(id, T&: Table);
591
592 ArrayRef<IITDescriptor> TableRef = Table;
593 Type *ResultTy = DecodeFixedType(Infos&: TableRef, Tys, Context);
594
595 SmallVector<Type *, 8> ArgTys;
596 while (!TableRef.empty())
597 ArgTys.push_back(Elt: DecodeFixedType(Infos&: TableRef, Tys, Context));
598
599 // DecodeFixedType returns Void for IITDescriptor::Void and
600 // IITDescriptor::VarArg If we see void type as the type of the last argument,
601 // it is vararg intrinsic
602 if (!ArgTys.empty() && ArgTys.back()->isVoidTy()) {
603 ArgTys.pop_back();
604 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: true);
605 }
606 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: false);
607}
608
609bool Intrinsic::isOverloaded(ID id) {
610#define GET_INTRINSIC_OVERLOAD_TABLE
611#include "llvm/IR/IntrinsicImpl.inc"
612}
613
614bool Intrinsic::hasPrettyPrintedArgs(ID id){
615#define GET_INTRINSIC_PRETTY_PRINT_TABLE
616#include "llvm/IR/IntrinsicImpl.inc"
617}
618
619/// Table of per-target intrinsic name tables.
620#define GET_INTRINSIC_TARGET_DATA
621#include "llvm/IR/IntrinsicImpl.inc"
622
623bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
624 return IID > TargetInfos[0].Count;
625}
626
627/// Looks up Name in NameTable via binary search. NameTable must be sorted
628/// and all entries must start with "llvm.". If NameTable contains an exact
629/// match for Name or a prefix of Name followed by a dot, its index in
630/// NameTable is returned. Otherwise, -1 is returned.
631static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
632 StringRef Name, StringRef Target = "") {
633 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
634 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
635
636 // Do successive binary searches of the dotted name components. For
637 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
638 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
639 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
640 // size 1. During the search, we can skip the prefix that we already know is
641 // identical. By using strncmp we consider names with differing suffixes to
642 // be part of the equal range.
643 size_t CmpEnd = 4; // Skip the "llvm" component.
644 if (!Target.empty())
645 CmpEnd += 1 + Target.size(); // skip the .target component.
646
647 const unsigned *Low = NameOffsetTable.begin();
648 const unsigned *High = NameOffsetTable.end();
649 const unsigned *LastLow = Low;
650 while (CmpEnd < Name.size() && High - Low > 0) {
651 size_t CmpStart = CmpEnd;
652 CmpEnd = Name.find(C: '.', From: CmpStart + 1);
653 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
654 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
655 // `equal_range` requires the comparison to work with either side being an
656 // offset or the value. Detect which kind each side is to set up the
657 // compared strings.
658 const char *LHSStr;
659 if constexpr (std::is_integral_v<decltype(LHS)>)
660 LHSStr = IntrinsicNameTable.getCString(O: LHS);
661 else
662 LHSStr = LHS;
663
664 const char *RHSStr;
665 if constexpr (std::is_integral_v<decltype(RHS)>)
666 RHSStr = IntrinsicNameTable.getCString(O: RHS);
667 else
668 RHSStr = RHS;
669
670 return strncmp(s1: LHSStr + CmpStart, s2: RHSStr + CmpStart, n: CmpEnd - CmpStart) <
671 0;
672 };
673 LastLow = Low;
674 std::tie(args&: Low, args&: High) = std::equal_range(first: Low, last: High, val: Name.data(), comp: Cmp);
675 }
676 if (High - Low > 0)
677 LastLow = Low;
678
679 if (LastLow == NameOffsetTable.end())
680 return -1;
681 StringRef NameFound = IntrinsicNameTable[*LastLow];
682 if (Name == NameFound ||
683 (Name.starts_with(Prefix: NameFound) && Name[NameFound.size()] == '.'))
684 return LastLow - NameOffsetTable.begin();
685 return -1;
686}
687
688/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
689/// target as \c Name, or the generic table if \c Name is not target specific.
690///
691/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
692/// name.
693static std::pair<ArrayRef<unsigned>, StringRef>
694findTargetSubtable(StringRef Name) {
695 assert(Name.starts_with("llvm."));
696
697 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
698 // Drop "llvm." and take the first dotted component. That will be the target
699 // if this is target specific.
700 StringRef Target = Name.drop_front(N: 5).split(Separator: '.').first;
701 auto It = partition_point(
702 Range&: Targets, P: [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
703 // We've either found the target or just fall back to the generic set, which
704 // is always first.
705 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
706 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
707 TI.Name};
708}
709
710/// This does the actual lookup of an intrinsic ID which matches the given
711/// function name.
712Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
713 auto [NameOffsetTable, Target] = findTargetSubtable(Name);
714 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
715 if (Idx == -1)
716 return Intrinsic::not_intrinsic;
717
718 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
719 // an index into a sub-table.
720 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
721 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
722
723 // If the intrinsic is not overloaded, require an exact match. If it is
724 // overloaded, require either exact or prefix match.
725 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
726 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
727 bool IsExactMatch = Name.size() == MatchSize;
728 return IsExactMatch || Intrinsic::isOverloaded(id: ID) ? ID
729 : Intrinsic::not_intrinsic;
730}
731
732/// This defines the "Intrinsic::getAttributes(ID id)" method.
733#define GET_INTRINSIC_ATTRIBUTES
734#include "llvm/IR/IntrinsicImpl.inc"
735
736static Function *getOrInsertIntrinsicDeclarationImpl(Module *M,
737 Intrinsic::ID id,
738 ArrayRef<Type *> Tys,
739 FunctionType *FT) {
740 Function *F = cast<Function>(
741 Val: M->getOrInsertFunction(Name: Tys.empty() ? Intrinsic::getName(id)
742 : Intrinsic::getName(Id: id, Tys, M, FT),
743 T: FT)
744 .getCallee());
745 if (F->getFunctionType() == FT)
746 return F;
747
748 // It's possible that a declaration for this intrinsic already exists with an
749 // incorrect signature, if the signature has changed, but this particular
750 // declaration has not been auto-upgraded yet. In that case, rename the
751 // invalid declaration and insert a new one with the correct signature. The
752 // invalid declaration will get upgraded later.
753 F->setName(F->getName() + ".invalid");
754 return cast<Function>(
755 Val: M->getOrInsertFunction(Name: Tys.empty() ? Intrinsic::getName(id)
756 : Intrinsic::getName(Id: id, Tys, M, FT),
757 T: FT)
758 .getCallee());
759}
760
761Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
762 ArrayRef<Type *> Tys) {
763 // There can never be multiple globals with the same name of different types,
764 // because intrinsics must be a specific type.
765 FunctionType *FT = getType(Context&: M->getContext(), id, Tys);
766 return getOrInsertIntrinsicDeclarationImpl(M, id, Tys, FT);
767}
768
769Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
770 ArrayRef<Type *> ArgTys) {
771 // If the intrinsic is not overloaded, use the non-overloaded version.
772 if (!Intrinsic::isOverloaded(id))
773 return getOrInsertDeclaration(M, id);
774
775 // Get the intrinsic signature metadata.
776 SmallVector<Intrinsic::IITDescriptor, 8> Table;
777 getIntrinsicInfoTableEntries(id, T&: Table);
778 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
779
780 FunctionType *FTy = FunctionType::get(Result: RetTy, Params: ArgTys, /*isVarArg=*/false);
781
782 // Automatically determine the overloaded types.
783 SmallVector<Type *, 4> OverloadTys;
784 [[maybe_unused]] Intrinsic::MatchIntrinsicTypesResult Res =
785 matchIntrinsicSignature(FTy, Infos&: TableRef, ArgTys&: OverloadTys);
786 assert(Res == Intrinsic::MatchIntrinsicTypes_Match &&
787 "intrinsic signature mismatch");
788
789 // If intrinsic requires vararg, recreate the FunctionType accordingly.
790 if (!matchIntrinsicVarArg(/*isVarArg=*/true, Infos&: TableRef))
791 FTy = FunctionType::get(Result: RetTy, Params: ArgTys, /*isVarArg=*/true);
792
793 assert(TableRef.empty() && "Unprocessed descriptors remain");
794
795 return getOrInsertIntrinsicDeclarationImpl(M, id, Tys: OverloadTys, FT: FTy);
796}
797
798Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
799 return M->getFunction(Name: getName(id));
800}
801
802Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
803 ArrayRef<Type *> Tys,
804 FunctionType *FT) {
805 return M->getFunction(Name: getName(Id: id, Tys, M, FT));
806}
807
808// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
809#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
810#include "llvm/IR/IntrinsicImpl.inc"
811
812// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
813#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
814#include "llvm/IR/IntrinsicImpl.inc"
815
816bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
817 switch (QID) {
818#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
819 case Intrinsic::INTRINSIC:
820#include "llvm/IR/ConstrainedOps.def"
821#undef INSTRUCTION
822 return true;
823 default:
824 return false;
825 }
826}
827
828bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
829 switch (QID) {
830#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
831 case Intrinsic::INTRINSIC: \
832 return ROUND_MODE == 1;
833#include "llvm/IR/ConstrainedOps.def"
834#undef INSTRUCTION
835 default:
836 return false;
837 }
838}
839
840using DeferredIntrinsicMatchPair =
841 std::pair<Type *, ArrayRef<Intrinsic::IITDescriptor>>;
842
843static bool
844matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
845 SmallVectorImpl<Type *> &ArgTys,
846 SmallVectorImpl<DeferredIntrinsicMatchPair> &DeferredChecks,
847 bool IsDeferredCheck) {
848 using namespace Intrinsic;
849
850 // If we ran out of descriptors, there are too many arguments.
851 if (Infos.empty())
852 return true;
853
854 // Do this before slicing off the 'front' part
855 auto InfosRef = Infos;
856 auto DeferCheck = [&DeferredChecks, &InfosRef](Type *T) {
857 DeferredChecks.emplace_back(Args&: T, Args&: InfosRef);
858 return false;
859 };
860
861 IITDescriptor D = Infos.front();
862 Infos = Infos.slice(N: 1);
863
864 switch (D.Kind) {
865 case IITDescriptor::Void:
866 return !Ty->isVoidTy();
867 case IITDescriptor::VarArg:
868 return true;
869 case IITDescriptor::MMX: {
870 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
871 return !VT || VT->getNumElements() != 1 ||
872 !VT->getElementType()->isIntegerTy(Bitwidth: 64);
873 }
874 case IITDescriptor::AMX:
875 return !Ty->isX86_AMXTy();
876 case IITDescriptor::Token:
877 return !Ty->isTokenTy();
878 case IITDescriptor::Metadata:
879 return !Ty->isMetadataTy();
880 case IITDescriptor::Half:
881 return !Ty->isHalfTy();
882 case IITDescriptor::BFloat:
883 return !Ty->isBFloatTy();
884 case IITDescriptor::Float:
885 return !Ty->isFloatTy();
886 case IITDescriptor::Double:
887 return !Ty->isDoubleTy();
888 case IITDescriptor::Quad:
889 return !Ty->isFP128Ty();
890 case IITDescriptor::PPCQuad:
891 return !Ty->isPPC_FP128Ty();
892 case IITDescriptor::Integer:
893 return !Ty->isIntegerTy(Bitwidth: D.Integer_Width);
894 case IITDescriptor::AArch64Svcount:
895 return !isa<TargetExtType>(Val: Ty) ||
896 cast<TargetExtType>(Val: Ty)->getName() != "aarch64.svcount";
897 case IITDescriptor::Vector: {
898 VectorType *VT = dyn_cast<VectorType>(Val: Ty);
899 return !VT || VT->getElementCount() != D.Vector_Width ||
900 matchIntrinsicType(Ty: VT->getElementType(), Infos, ArgTys,
901 DeferredChecks, IsDeferredCheck);
902 }
903 case IITDescriptor::Pointer: {
904 PointerType *PT = dyn_cast<PointerType>(Val: Ty);
905 return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace;
906 }
907
908 case IITDescriptor::Struct: {
909 StructType *ST = dyn_cast<StructType>(Val: Ty);
910 if (!ST || !ST->isLiteral() || ST->isPacked() ||
911 ST->getNumElements() != D.Struct_NumElements)
912 return true;
913
914 for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i)
915 if (matchIntrinsicType(Ty: ST->getElementType(N: i), Infos, ArgTys,
916 DeferredChecks, IsDeferredCheck))
917 return true;
918 return false;
919 }
920
921 case IITDescriptor::Argument:
922 // If this is the second occurrence of an argument,
923 // verify that the later instance matches the previous instance.
924 if (D.getArgumentNumber() < ArgTys.size())
925 return Ty != ArgTys[D.getArgumentNumber()];
926
927 if (D.getArgumentNumber() > ArgTys.size() ||
928 D.getArgumentKind() == IITDescriptor::AK_MatchType)
929 return IsDeferredCheck || DeferCheck(Ty);
930
931 assert(D.getArgumentNumber() == ArgTys.size() && !IsDeferredCheck &&
932 "Table consistency error");
933 ArgTys.push_back(Elt: Ty);
934
935 switch (D.getArgumentKind()) {
936 case IITDescriptor::AK_Any:
937 return false; // Success
938 case IITDescriptor::AK_AnyInteger:
939 return !Ty->isIntOrIntVectorTy();
940 case IITDescriptor::AK_AnyFloat:
941 return !Ty->isFPOrFPVectorTy();
942 case IITDescriptor::AK_AnyVector:
943 return !isa<VectorType>(Val: Ty);
944 case IITDescriptor::AK_AnyPointer:
945 return !isa<PointerType>(Val: Ty);
946 default:
947 break;
948 }
949 llvm_unreachable("all argument kinds not covered");
950
951 case IITDescriptor::ExtendArgument: {
952 // If this is a forward reference, defer the check for later.
953 if (D.getArgumentNumber() >= ArgTys.size())
954 return IsDeferredCheck || DeferCheck(Ty);
955
956 Type *NewTy = ArgTys[D.getArgumentNumber()];
957 if (VectorType *VTy = dyn_cast<VectorType>(Val: NewTy))
958 NewTy = VectorType::getExtendedElementVectorType(VTy);
959 else if (IntegerType *ITy = dyn_cast<IntegerType>(Val: NewTy))
960 NewTy = IntegerType::get(C&: ITy->getContext(), NumBits: 2 * ITy->getBitWidth());
961 else
962 return true;
963
964 return Ty != NewTy;
965 }
966 case IITDescriptor::TruncArgument: {
967 // If this is a forward reference, defer the check for later.
968 if (D.getArgumentNumber() >= ArgTys.size())
969 return IsDeferredCheck || DeferCheck(Ty);
970
971 Type *NewTy = ArgTys[D.getArgumentNumber()];
972 if (VectorType *VTy = dyn_cast<VectorType>(Val: NewTy))
973 NewTy = VectorType::getTruncatedElementVectorType(VTy);
974 else if (IntegerType *ITy = dyn_cast<IntegerType>(Val: NewTy))
975 NewTy = IntegerType::get(C&: ITy->getContext(), NumBits: ITy->getBitWidth() / 2);
976 else
977 return true;
978
979 return Ty != NewTy;
980 }
981 case IITDescriptor::OneNthEltsVecArgument: {
982 // If this is a forward reference, defer the check for later.
983 if (D.getRefArgNumber() >= ArgTys.size())
984 return IsDeferredCheck || DeferCheck(Ty);
985 auto *VTy = dyn_cast<VectorType>(Val: ArgTys[D.getRefArgNumber()]);
986 if (!VTy)
987 return true;
988 if (!VTy->getElementCount().isKnownMultipleOf(RHS: D.getVectorDivisor()))
989 return true;
990 return VectorType::getOneNthElementsVectorType(VTy, Denominator: D.getVectorDivisor()) !=
991 Ty;
992 }
993 case IITDescriptor::SameVecWidthArgument: {
994 if (D.getArgumentNumber() >= ArgTys.size()) {
995 // Defer check and subsequent check for the vector element type.
996 Infos = Infos.slice(N: 1);
997 return IsDeferredCheck || DeferCheck(Ty);
998 }
999 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1000 auto *ThisArgType = dyn_cast<VectorType>(Val: Ty);
1001 // Both must be vectors of the same number of elements or neither.
1002 if ((ReferenceType != nullptr) != (ThisArgType != nullptr))
1003 return true;
1004 Type *EltTy = Ty;
1005 if (ThisArgType) {
1006 if (ReferenceType->getElementCount() != ThisArgType->getElementCount())
1007 return true;
1008 EltTy = ThisArgType->getElementType();
1009 }
1010 return matchIntrinsicType(Ty: EltTy, Infos, ArgTys, DeferredChecks,
1011 IsDeferredCheck);
1012 }
1013 case IITDescriptor::VecOfAnyPtrsToElt: {
1014 unsigned RefArgNumber = D.getRefArgNumber();
1015 if (RefArgNumber >= ArgTys.size()) {
1016 if (IsDeferredCheck)
1017 return true;
1018 // If forward referencing, already add the pointer-vector type and
1019 // defer the checks for later.
1020 ArgTys.push_back(Elt: Ty);
1021 return DeferCheck(Ty);
1022 }
1023
1024 if (!IsDeferredCheck) {
1025 assert(D.getOverloadArgNumber() == ArgTys.size() &&
1026 "Table consistency error");
1027 ArgTys.push_back(Elt: Ty);
1028 }
1029
1030 // Verify the overloaded type "matches" the Ref type.
1031 // i.e. Ty is a vector with the same width as Ref.
1032 // Composed of pointers to the same element type as Ref.
1033 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[RefArgNumber]);
1034 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1035 if (!ThisArgVecTy || !ReferenceType ||
1036 (ReferenceType->getElementCount() != ThisArgVecTy->getElementCount()))
1037 return true;
1038 return !ThisArgVecTy->getElementType()->isPointerTy();
1039 }
1040 case IITDescriptor::VecElementArgument: {
1041 if (D.getArgumentNumber() >= ArgTys.size())
1042 return IsDeferredCheck ? true : DeferCheck(Ty);
1043 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1044 return !ReferenceType || Ty != ReferenceType->getElementType();
1045 }
1046 case IITDescriptor::Subdivide2Argument:
1047 case IITDescriptor::Subdivide4Argument: {
1048 // If this is a forward reference, defer the check for later.
1049 if (D.getArgumentNumber() >= ArgTys.size())
1050 return IsDeferredCheck || DeferCheck(Ty);
1051
1052 Type *NewTy = ArgTys[D.getArgumentNumber()];
1053 if (auto *VTy = dyn_cast<VectorType>(Val: NewTy)) {
1054 int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
1055 NewTy = VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
1056 return Ty != NewTy;
1057 }
1058 return true;
1059 }
1060 case IITDescriptor::VecOfBitcastsToInt: {
1061 if (D.getArgumentNumber() >= ArgTys.size())
1062 return IsDeferredCheck || DeferCheck(Ty);
1063 auto *ReferenceType = dyn_cast<VectorType>(Val: ArgTys[D.getArgumentNumber()]);
1064 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1065 if (!ThisArgVecTy || !ReferenceType)
1066 return true;
1067 return ThisArgVecTy != VectorType::getInteger(VTy: ReferenceType);
1068 }
1069 }
1070 llvm_unreachable("unhandled");
1071}
1072
1073Intrinsic::MatchIntrinsicTypesResult
1074Intrinsic::matchIntrinsicSignature(FunctionType *FTy,
1075 ArrayRef<Intrinsic::IITDescriptor> &Infos,
1076 SmallVectorImpl<Type *> &ArgTys) {
1077 SmallVector<DeferredIntrinsicMatchPair, 2> DeferredChecks;
1078 if (matchIntrinsicType(Ty: FTy->getReturnType(), Infos, ArgTys, DeferredChecks,
1079 IsDeferredCheck: false))
1080 return MatchIntrinsicTypes_NoMatchRet;
1081
1082 unsigned NumDeferredReturnChecks = DeferredChecks.size();
1083
1084 for (auto *Ty : FTy->params())
1085 if (matchIntrinsicType(Ty, Infos, ArgTys, DeferredChecks, IsDeferredCheck: false))
1086 return MatchIntrinsicTypes_NoMatchArg;
1087
1088 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
1089 DeferredIntrinsicMatchPair &Check = DeferredChecks[I];
1090 if (matchIntrinsicType(Ty: Check.first, Infos&: Check.second, ArgTys, DeferredChecks,
1091 IsDeferredCheck: true))
1092 return I < NumDeferredReturnChecks ? MatchIntrinsicTypes_NoMatchRet
1093 : MatchIntrinsicTypes_NoMatchArg;
1094 }
1095
1096 return MatchIntrinsicTypes_Match;
1097}
1098
1099bool Intrinsic::matchIntrinsicVarArg(
1100 bool isVarArg, ArrayRef<Intrinsic::IITDescriptor> &Infos) {
1101 // If there are no descriptors left, then it can't be a vararg.
1102 if (Infos.empty())
1103 return isVarArg;
1104
1105 // There should be only one descriptor remaining at this point.
1106 if (Infos.size() != 1)
1107 return true;
1108
1109 // Check and verify the descriptor.
1110 IITDescriptor D = Infos.front();
1111 Infos = Infos.slice(N: 1);
1112 if (D.Kind == IITDescriptor::VarArg)
1113 return !isVarArg;
1114
1115 return true;
1116}
1117
1118bool Intrinsic::getIntrinsicSignature(Intrinsic::ID ID, FunctionType *FT,
1119 SmallVectorImpl<Type *> &ArgTys) {
1120 if (!ID)
1121 return false;
1122
1123 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1124 getIntrinsicInfoTableEntries(id: ID, T&: Table);
1125 ArrayRef<Intrinsic::IITDescriptor> TableRef = Table;
1126
1127 if (Intrinsic::matchIntrinsicSignature(FTy: FT, Infos&: TableRef, ArgTys) !=
1128 Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) {
1129 return false;
1130 }
1131 if (Intrinsic::matchIntrinsicVarArg(isVarArg: FT->isVarArg(), Infos&: TableRef))
1132 return false;
1133 return true;
1134}
1135
1136bool Intrinsic::getIntrinsicSignature(Function *F,
1137 SmallVectorImpl<Type *> &ArgTys) {
1138 return getIntrinsicSignature(ID: F->getIntrinsicID(), FT: F->getFunctionType(),
1139 ArgTys);
1140}
1141
1142std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
1143 SmallVector<Type *, 4> ArgTys;
1144 if (!getIntrinsicSignature(F, ArgTys))
1145 return std::nullopt;
1146
1147 Intrinsic::ID ID = F->getIntrinsicID();
1148 StringRef Name = F->getName();
1149 std::string WantedName =
1150 Intrinsic::getName(Id: ID, Tys: ArgTys, M: F->getParent(), FT: F->getFunctionType());
1151 if (Name == WantedName)
1152 return std::nullopt;
1153
1154 Function *NewDecl = [&] {
1155 if (auto *ExistingGV = F->getParent()->getNamedValue(Name: WantedName)) {
1156 if (auto *ExistingF = dyn_cast<Function>(Val: ExistingGV))
1157 if (ExistingF->getFunctionType() == F->getFunctionType())
1158 return ExistingF;
1159
1160 // The name already exists, but is not a function or has the wrong
1161 // prototype. Make place for the new one by renaming the old version.
1162 // Either this old version will be removed later on or the module is
1163 // invalid and we'll get an error.
1164 ExistingGV->setName(WantedName + ".renamed");
1165 }
1166 return Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, Tys: ArgTys);
1167 }();
1168
1169 NewDecl->setCallingConv(F->getCallingConv());
1170 assert(NewDecl->getFunctionType() == F->getFunctionType() &&
1171 "Shouldn't change the signature");
1172 return NewDecl;
1173}
1174
1175struct InterleaveIntrinsic {
1176 Intrinsic::ID Interleave, Deinterleave;
1177};
1178
1179static InterleaveIntrinsic InterleaveIntrinsics[] = {
1180 {.Interleave: Intrinsic::vector_interleave2, .Deinterleave: Intrinsic::vector_deinterleave2},
1181 {.Interleave: Intrinsic::vector_interleave3, .Deinterleave: Intrinsic::vector_deinterleave3},
1182 {.Interleave: Intrinsic::vector_interleave4, .Deinterleave: Intrinsic::vector_deinterleave4},
1183 {.Interleave: Intrinsic::vector_interleave5, .Deinterleave: Intrinsic::vector_deinterleave5},
1184 {.Interleave: Intrinsic::vector_interleave6, .Deinterleave: Intrinsic::vector_deinterleave6},
1185 {.Interleave: Intrinsic::vector_interleave7, .Deinterleave: Intrinsic::vector_deinterleave7},
1186 {.Interleave: Intrinsic::vector_interleave8, .Deinterleave: Intrinsic::vector_deinterleave8},
1187};
1188
1189Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
1190 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1191 return InterleaveIntrinsics[Factor - 2].Interleave;
1192}
1193
1194Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
1195 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1196 return InterleaveIntrinsics[Factor - 2].Deinterleave;
1197}
1198
1199#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1200#include "llvm/IR/IntrinsicImpl.inc"
1201