1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions required for supporting intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Intrinsics.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringTable.h"
16#include "llvm/IR/ConstantRange.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/IR/IntrinsicsPowerPC.h"
27#include "llvm/IR/IntrinsicsR600.h"
28#include "llvm/IR/IntrinsicsRISCV.h"
29#include "llvm/IR/IntrinsicsS390.h"
30#include "llvm/IR/IntrinsicsSPIRV.h"
31#include "llvm/IR/IntrinsicsVE.h"
32#include "llvm/IR/IntrinsicsX86.h"
33#include "llvm/IR/IntrinsicsXCore.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NVVMIntrinsicUtils.h"
36#include "llvm/IR/Type.h"
37#include "llvm/Support/FormatVariadic.h"
38#include "llvm/Support/MathExtras.h"
39
40using namespace llvm;
41
42// Forward declaration of static functions.
43static bool isSignatureValid(FunctionType *FTy,
44 ArrayRef<Intrinsic::IITDescriptor> &Infos,
45 unsigned NumArgs, bool IsVarArg,
46 SmallVectorImpl<Type *> &OverloadTys,
47 raw_ostream &OS);
48
49/// Table of string intrinsic names indexed by enum value.
50#define GET_INTRINSIC_NAME_TABLE
51#include "llvm/IR/IntrinsicImpl.inc"
52
53/// Table of required target features indexed by enum value.
54#define GET_INTRINSIC_TARGET_FEATURES_TABLE
55#include "llvm/IR/IntrinsicImpl.inc"
56
57StringRef Intrinsic::getBaseName(ID id) {
58 assert(id < num_intrinsics && "Invalid intrinsic ID!");
59 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
60}
61
62StringRef Intrinsic::getRequiredTargetFeatures(ID id) {
63 assert(id < num_intrinsics && "invalid intrinsic ID!");
64 return IntrinsicTargetFeaturesTable[IntrinsicTargetFeaturesOffsetTable[id]];
65}
66
67StringRef Intrinsic::getName(ID id) {
68 assert(id < num_intrinsics && "Invalid intrinsic ID!");
69 assert(!Intrinsic::isOverloaded(id) &&
70 "This version of getName does not support overloading");
71 return getBaseName(id);
72}
73
74/// Returns a stable mangling for the type specified for use in the name
75/// mangling scheme used by 'any' types in intrinsic signatures. The mangling
76/// of named types is simply their name. Manglings for unnamed types consist
77/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
78/// combined with the mangling of their component types. A vararg function
79/// type will have a suffix of 'vararg'. Since function types can contain
80/// other function types, we close a function type mangling with suffix 'f'
81/// which can't be confused with it's prefix. This ensures we don't have
82/// collisions between two unrelated function types. Otherwise, you might
83/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)
84/// The HasUnnamedType boolean is set if an unnamed type was encountered,
85/// indicating that extra care must be taken to ensure a unique name.
86static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
87 std::string Result;
88 if (PointerType *PTyp = dyn_cast<PointerType>(Val: Ty)) {
89 Result += "p" + utostr(X: PTyp->getAddressSpace());
90 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Val: Ty)) {
91 Result += "a" + utostr(X: ATyp->getNumElements()) +
92 getMangledTypeStr(Ty: ATyp->getElementType(), HasUnnamedType);
93 } else if (StructType *STyp = dyn_cast<StructType>(Val: Ty)) {
94 if (!STyp->isLiteral()) {
95 Result += "s_";
96 if (STyp->hasName())
97 Result += STyp->getName();
98 else
99 HasUnnamedType = true;
100 } else {
101 Result += "sl_";
102 for (auto *Elem : STyp->elements())
103 Result += getMangledTypeStr(Ty: Elem, HasUnnamedType);
104 }
105 // Ensure nested structs are distinguishable.
106 Result += "s";
107 } else if (FunctionType *FT = dyn_cast<FunctionType>(Val: Ty)) {
108 Result += "f_" + getMangledTypeStr(Ty: FT->getReturnType(), HasUnnamedType);
109 for (size_t i = 0; i < FT->getNumParams(); i++)
110 Result += getMangledTypeStr(Ty: FT->getParamType(i), HasUnnamedType);
111 if (FT->isVarArg())
112 Result += "vararg";
113 // Ensure nested function types are distinguishable.
114 Result += "f";
115 } else if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
116 ElementCount EC = VTy->getElementCount();
117 if (EC.isScalable())
118 Result += "nx";
119 Result += "v" + utostr(X: EC.getKnownMinValue()) +
120 getMangledTypeStr(Ty: VTy->getElementType(), HasUnnamedType);
121 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Val: Ty)) {
122 Result += "t";
123 Result += TETy->getName();
124 for (Type *ParamTy : TETy->type_params())
125 Result += "_" + getMangledTypeStr(Ty: ParamTy, HasUnnamedType);
126 for (unsigned IntParam : TETy->int_params())
127 Result += "_" + utostr(X: IntParam);
128 // Ensure nested target extension types are distinguishable.
129 Result += "t";
130 } else if (Ty) {
131 switch (Ty->getTypeID()) {
132 default:
133 llvm_unreachable("Unhandled type");
134 case Type::VoidTyID:
135 Result += "isVoid";
136 break;
137 case Type::MetadataTyID:
138 Result += "Metadata";
139 break;
140 case Type::HalfTyID:
141 Result += "f16";
142 break;
143 case Type::BFloatTyID:
144 Result += "bf16";
145 break;
146 case Type::FloatTyID:
147 Result += "f32";
148 break;
149 case Type::DoubleTyID:
150 Result += "f64";
151 break;
152 case Type::X86_FP80TyID:
153 Result += "f80";
154 break;
155 case Type::FP128TyID:
156 Result += "f128";
157 break;
158 case Type::PPC_FP128TyID:
159 Result += "ppcf128";
160 break;
161 case Type::X86_AMXTyID:
162 Result += "x86amx";
163 break;
164 case Type::IntegerTyID:
165 Result += "i" + utostr(X: cast<IntegerType>(Val: Ty)->getBitWidth());
166 break;
167 case Type::ByteTyID:
168 Result += "b" + utostr(X: cast<ByteType>(Val: Ty)->getBitWidth());
169 break;
170 }
171 }
172 return Result;
173}
174
175static std::string getIntrinsicNameImpl(Intrinsic::ID Id,
176 ArrayRef<Type *> OverloadTys, Module *M,
177 FunctionType *FT,
178 bool EarlyModuleCheck) {
179
180 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
181 assert((OverloadTys.empty() || Intrinsic::isOverloaded(Id)) &&
182 "This version of getName is for overloaded intrinsics only");
183 (void)EarlyModuleCheck;
184 assert((!EarlyModuleCheck || M ||
185 !any_of(OverloadTys, llvm::IsaPred<PointerType>)) &&
186 "Intrinsic overloading on pointer types need to provide a Module");
187 bool HasUnnamedType = false;
188 std::string Result(Intrinsic::getBaseName(id: Id));
189 for (Type *Ty : OverloadTys)
190 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
191 if (HasUnnamedType) {
192 assert(M && "unnamed types need a module");
193 if (!FT)
194 FT = Intrinsic::getType(Context&: M->getContext(), id: Id, OverloadTys);
195 else
196 assert(FT == Intrinsic::getType(M->getContext(), Id, OverloadTys) &&
197 "Provided FunctionType must match arguments");
198 return M->getUniqueIntrinsicName(BaseName: Result, Id, Proto: FT);
199 }
200 return Result;
201}
202
203std::string Intrinsic::getName(ID Id, ArrayRef<Type *> OverloadTys, Module *M,
204 FunctionType *FT) {
205 assert(M && "We need to have a Module");
206 return getIntrinsicNameImpl(Id, OverloadTys, M, FT, EarlyModuleCheck: true);
207}
208
209std::string Intrinsic::getNameNoUnnamedTypes(ID Id,
210 ArrayRef<Type *> OverloadTys) {
211 return getIntrinsicNameImpl(Id, OverloadTys, M: nullptr, FT: nullptr, EarlyModuleCheck: false);
212}
213
214/// IIT_Info - These are enumerators that describe the entries returned by the
215/// getIntrinsicInfoTableEntries function.
216///
217/// Defined in Intrinsics.td.
218enum IIT_Info {
219#define GET_INTRINSIC_IITINFO
220#include "llvm/IR/IntrinsicImpl.inc"
221};
222
223static_assert(IIT_Done == 0, "IIT_Done expected to be 0");
224
225static void
226DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
227 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
228 using namespace Intrinsic;
229
230 auto IsScalableVector = [&]() {
231 IIT_Info NextInfo = IIT_Info(Infos[NextElt]);
232 if (NextInfo != IIT_SCALABLE_VEC)
233 return false;
234 // Eat the IIT_SCALABLE_VEC token.
235 ++NextElt;
236 return true;
237 };
238
239 IIT_Info Info = IIT_Info(Infos[NextElt++]);
240
241 switch (Info) {
242 case IIT_Done:
243 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Void, Field: 0));
244 return;
245 case IIT_VARARG:
246 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VarArg, Field: 0));
247 return;
248 case IIT_MMX:
249 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::MMX, Field: 0));
250 return;
251 case IIT_AMX:
252 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AMX, Field: 0));
253 return;
254 case IIT_TOKEN:
255 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Token, Field: 0));
256 return;
257 case IIT_METADATA:
258 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Metadata, Field: 0));
259 return;
260 case IIT_F16:
261 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Half, Field: 0));
262 return;
263 case IIT_BF16:
264 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::BFloat, Field: 0));
265 return;
266 case IIT_F32:
267 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Float, Field: 0));
268 return;
269 case IIT_F64:
270 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Double, Field: 0));
271 return;
272 case IIT_F128:
273 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Quad, Field: 0));
274 return;
275 case IIT_PPCF128:
276 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::PPCQuad, Field: 0));
277 return;
278 case IIT_I1:
279 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 1));
280 return;
281 case IIT_I2:
282 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 2));
283 return;
284 case IIT_I4:
285 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 4));
286 return;
287 case IIT_AARCH64_SVCOUNT:
288 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AArch64Svcount, Field: 0));
289 return;
290 case IIT_I8:
291 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 8));
292 return;
293 case IIT_I16:
294 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 16));
295 return;
296 case IIT_I32:
297 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 32));
298 return;
299 case IIT_I64:
300 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 64));
301 return;
302 case IIT_I128:
303 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 128));
304 return;
305 case IIT_V1:
306 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1, IsScalable: IsScalableVector()));
307 DecodeIITType(NextElt, Infos, OutputTable);
308 return;
309 case IIT_V2:
310 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2, IsScalable: IsScalableVector()));
311 DecodeIITType(NextElt, Infos, OutputTable);
312 return;
313 case IIT_V3:
314 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 3, IsScalable: IsScalableVector()));
315 DecodeIITType(NextElt, Infos, OutputTable);
316 return;
317 case IIT_V4:
318 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4, IsScalable: IsScalableVector()));
319 DecodeIITType(NextElt, Infos, OutputTable);
320 return;
321 case IIT_V6:
322 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 6, IsScalable: IsScalableVector()));
323 DecodeIITType(NextElt, Infos, OutputTable);
324 return;
325 case IIT_V8:
326 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 8, IsScalable: IsScalableVector()));
327 DecodeIITType(NextElt, Infos, OutputTable);
328 return;
329 case IIT_V10:
330 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 10, IsScalable: IsScalableVector()));
331 DecodeIITType(NextElt, Infos, OutputTable);
332 return;
333 case IIT_V16:
334 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 16, IsScalable: IsScalableVector()));
335 DecodeIITType(NextElt, Infos, OutputTable);
336 return;
337 case IIT_V32:
338 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 32, IsScalable: IsScalableVector()));
339 DecodeIITType(NextElt, Infos, OutputTable);
340 return;
341 case IIT_V64:
342 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 64, IsScalable: IsScalableVector()));
343 DecodeIITType(NextElt, Infos, OutputTable);
344 return;
345 case IIT_V128:
346 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 128, IsScalable: IsScalableVector()));
347 DecodeIITType(NextElt, Infos, OutputTable);
348 return;
349 case IIT_V256:
350 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 256, IsScalable: IsScalableVector()));
351 DecodeIITType(NextElt, Infos, OutputTable);
352 return;
353 case IIT_V512:
354 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 512, IsScalable: IsScalableVector()));
355 DecodeIITType(NextElt, Infos, OutputTable);
356 return;
357 case IIT_V1024:
358 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1024, IsScalable: IsScalableVector()));
359 DecodeIITType(NextElt, Infos, OutputTable);
360 return;
361 case IIT_V2048:
362 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2048, IsScalable: IsScalableVector()));
363 DecodeIITType(NextElt, Infos, OutputTable);
364 return;
365 case IIT_V4096:
366 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4096, IsScalable: IsScalableVector()));
367 DecodeIITType(NextElt, Infos, OutputTable);
368 return;
369 case IIT_EXTERNREF:
370 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmExternref, Field: 0));
371 return;
372 case IIT_FUNCREF:
373 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmFuncref, Field: 0));
374 return;
375 case IIT_PTR:
376 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 0));
377 return;
378 case IIT_PTR_AS: // pointer with address space.
379 OutputTable.push_back(
380 Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: Infos[NextElt++]));
381 return;
382 case IIT_ANY: {
383 unsigned OverloadIndex = Infos[NextElt++];
384 unsigned ArgKindEnums = Infos[NextElt++];
385 unsigned Packed = (ArgKindEnums << 8) | OverloadIndex;
386 OutputTable.push_back(
387 Elt: IITDescriptor::get(K: IITDescriptor::Overloaded, Field: Packed));
388 return;
389 }
390 case IIT_MATCH: {
391 unsigned OverloadIndex = Infos[NextElt++];
392 OutputTable.push_back(
393 Elt: IITDescriptor::get(K: IITDescriptor::Match, Field: OverloadIndex));
394 return;
395 }
396 case IIT_EXTEND_ARG: {
397 unsigned OverloadIndex = Infos[NextElt++];
398 OutputTable.push_back(
399 Elt: IITDescriptor::get(K: IITDescriptor::Extend, Field: OverloadIndex));
400 return;
401 }
402 case IIT_TRUNC_ARG: {
403 unsigned OverloadIndex = Infos[NextElt++];
404 OutputTable.push_back(
405 Elt: IITDescriptor::get(K: IITDescriptor::Trunc, Field: OverloadIndex));
406 return;
407 }
408 case IIT_ONE_NTH_ELTS_VEC_ARG: {
409 unsigned short OverloadIndex = Infos[NextElt++];
410 unsigned short N = Infos[NextElt++];
411 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::OneNthEltsVec,
412 /*Hi=*/N, /*Lo=*/OverloadIndex));
413 return;
414 }
415 case IIT_SAME_VEC_WIDTH_ARG: {
416 unsigned OverloadIndex = Infos[NextElt++];
417 OutputTable.push_back(
418 Elt: IITDescriptor::get(K: IITDescriptor::SameVecWidth, Field: OverloadIndex));
419 // IIT_SAME_VEC_WIDTH_ARG entry is followed by the element type.
420 DecodeIITType(NextElt, Infos, OutputTable);
421 return;
422 }
423 case IIT_VEC_OF_ANYPTRS_TO_ELT: {
424 unsigned short OverloadIndex = Infos[NextElt++];
425 unsigned short RefOverloadIndex = Infos[NextElt++];
426 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VecOfAnyPtrsToElt,
427 /*Hi=*/RefOverloadIndex,
428 /*Lo=*/OverloadIndex));
429 return;
430 }
431 case IIT_STRUCT: {
432 unsigned StructElts = Infos[NextElt++] + 2;
433
434 OutputTable.push_back(
435 Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: StructElts));
436
437 for (unsigned i = 0; i != StructElts; ++i)
438 DecodeIITType(NextElt, Infos, OutputTable);
439 return;
440 }
441 case IIT_SUBDIVIDE2_ARG: {
442 unsigned OverloadIndex = Infos[NextElt++];
443 OutputTable.push_back(
444 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide2, Field: OverloadIndex));
445 return;
446 }
447 case IIT_SUBDIVIDE4_ARG: {
448 unsigned OverloadIndex = Infos[NextElt++];
449 OutputTable.push_back(
450 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide4, Field: OverloadIndex));
451 return;
452 }
453 case IIT_VEC_ELEMENT: {
454 unsigned OverloadIndex = Infos[NextElt++];
455 OutputTable.push_back(
456 Elt: IITDescriptor::get(K: IITDescriptor::VecElement, Field: OverloadIndex));
457 return;
458 }
459 case IIT_VEC_OF_BITCASTS_TO_INT: {
460 unsigned OverloadIndex = Infos[NextElt++];
461 OutputTable.push_back(
462 Elt: IITDescriptor::get(K: IITDescriptor::VecOfBitcastsToInt, Field: OverloadIndex));
463 return;
464 }
465 case IIT_SCALABLE_VEC:
466 break;
467 }
468 llvm_unreachable("unhandled");
469}
470
471#define GET_INTRINSIC_GENERATOR_GLOBAL
472#include "llvm/IR/IntrinsicImpl.inc"
473
474std::tuple<ArrayRef<Intrinsic::IITDescriptor>, unsigned, bool>
475Intrinsic::getIntrinsicInfoTableEntries(ID id,
476 SmallVectorImpl<IITDescriptor> &T) {
477 // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
478 // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
479 // IntrinsicEmitter.cpp.
480 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
481 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
482 // Mask with all bits 1 except the most significant bit.
483 constexpr unsigned Mask = (1U << MSBPosition) - 1;
484
485 FixedEncodingTy TableVal = IIT_Table[id - 1];
486
487 // Array to hold the inlined fixed encoding values expanded from nibbles to
488 // bytes. Its size can be be atmost FixedEncodingBits / 4 i.e., number
489 // of nibbles that can fit in `FixedEncodingTy` + 1 (the IIT_Done terminator
490 // that is not explicitly encoded). Note that if there are trailing 0 bytes
491 // in the encoding (for example, payload following one of the IIT tokens),
492 // the inlined encoding does not encode the actual size of the encoding, so
493 // we always assume its size of this maximum length possible, followed by the
494 // IIT_Done terminator token (whose value is 0).
495 unsigned char IITValues[FixedEncodingBits / 4 + 1] = {0};
496
497 ArrayRef<unsigned char> IITEntries;
498 unsigned NextElt = 0;
499 // Check to see if the intrinsic's type was inlined in the fixed encoding
500 // table.
501 if (TableVal >> MSBPosition) {
502 // This is an offset into the IIT_LongEncodingTable.
503 IITEntries = IIT_LongEncodingTable;
504
505 // Strip sentinel bit.
506 NextElt = TableVal & Mask;
507 } else {
508 // If the entry was encoded into a single word in the table itself, decode
509 // it from an array of nibbles to an array of bytes.
510 do {
511 IITValues[NextElt++] = TableVal & 0xF;
512 TableVal >>= 4;
513 } while (TableVal);
514
515 IITEntries = IITValues;
516 NextElt = 0;
517 }
518
519 // Okay, decode the table into the output vector of IITDescriptors.
520 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
521 unsigned NumArgs = 0;
522 while (IITEntries[NextElt] != IIT_Done) {
523 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
524 ++NumArgs;
525 }
526
527 ArrayRef<IITDescriptor> TableRef = T;
528
529 bool IsVarArg = false;
530 if (TableRef.back().Kind == Intrinsic::IITDescriptor::VarArg) {
531 IsVarArg = true;
532 TableRef.consume_back();
533 --NumArgs;
534 }
535 return {TableRef, NumArgs, IsVarArg};
536}
537
538static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
539 ArrayRef<Type *> OverloadTys,
540 LLVMContext &Context) {
541 using namespace Intrinsic;
542
543 IITDescriptor D = Infos.consume_front();
544
545 switch (D.Kind) {
546 case IITDescriptor::Void:
547 return Type::getVoidTy(C&: Context);
548 case IITDescriptor::MMX:
549 return llvm::FixedVectorType::get(ElementType: llvm::IntegerType::get(C&: Context, NumBits: 64), NumElts: 1);
550 case IITDescriptor::AMX:
551 return Type::getX86_AMXTy(C&: Context);
552 case IITDescriptor::Token:
553 return Type::getTokenTy(C&: Context);
554 case IITDescriptor::Metadata:
555 return Type::getMetadataTy(C&: Context);
556 case IITDescriptor::Half:
557 return Type::getHalfTy(C&: Context);
558 case IITDescriptor::BFloat:
559 return Type::getBFloatTy(C&: Context);
560 case IITDescriptor::Float:
561 return Type::getFloatTy(C&: Context);
562 case IITDescriptor::Double:
563 return Type::getDoubleTy(C&: Context);
564 case IITDescriptor::Quad:
565 return Type::getFP128Ty(C&: Context);
566 case IITDescriptor::PPCQuad:
567 return Type::getPPC_FP128Ty(C&: Context);
568 case IITDescriptor::AArch64Svcount:
569 return TargetExtType::get(Context, Name: "aarch64.svcount");
570 case IITDescriptor::WasmExternref:
571 return TargetExtType::get(Context, Name: "wasm.externref");
572 case IITDescriptor::WasmFuncref:
573 return TargetExtType::get(Context, Name: "wasm.funcref");
574 case IITDescriptor::Integer:
575 return IntegerType::get(C&: Context, NumBits: D.IntegerWidth);
576 case IITDescriptor::Vector:
577 return VectorType::get(ElementType: DecodeFixedType(Infos, OverloadTys, Context),
578 EC: D.VectorWidth);
579 case IITDescriptor::Pointer:
580 return PointerType::get(C&: Context, AddressSpace: D.PointerAddressSpace);
581 case IITDescriptor::Struct: {
582 SmallVector<Type *, 8> Elts;
583 for (unsigned i = 0, e = D.StructNumElements; i != e; ++i)
584 Elts.push_back(Elt: DecodeFixedType(Infos, OverloadTys, Context));
585 return StructType::get(Context, Elements: Elts);
586 }
587 // For any overload type or partially dependent type, substitute it with the
588 // corresponding concrete type from OverloadTys. Additionally, do the same
589 // for the fully dependent type that matches an overload type.
590 case IITDescriptor::Overloaded:
591 case IITDescriptor::VecOfAnyPtrsToElt:
592 case IITDescriptor::Match:
593 return OverloadTys[D.getOverloadIndex()];
594 case IITDescriptor::Extend:
595 return OverloadTys[D.getOverloadIndex()]->getExtendedType();
596 case IITDescriptor::Trunc:
597 return OverloadTys[D.getOverloadIndex()]->getTruncatedType();
598 case IITDescriptor::Subdivide2:
599 case IITDescriptor::Subdivide4: {
600 Type *Ty = OverloadTys[D.getOverloadIndex()];
601 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
602 assert(VTy && "Expected overload type to be a Vector Type");
603 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
604 return VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
605 }
606 case IITDescriptor::OneNthEltsVec:
607 return VectorType::getOneNthElementsVectorType(
608 VTy: cast<VectorType>(Val: OverloadTys[D.getOverloadIndex()]),
609 Denominator: D.getVectorDivisor());
610 case IITDescriptor::SameVecWidth: {
611 Type *EltTy = DecodeFixedType(Infos, OverloadTys, Context);
612 Type *Ty = OverloadTys[D.getOverloadIndex()];
613 if (auto *VTy = dyn_cast<VectorType>(Val: Ty))
614 return VectorType::get(ElementType: EltTy, EC: VTy->getElementCount());
615 return EltTy;
616 }
617 case IITDescriptor::VecElement: {
618 Type *Ty = OverloadTys[D.getOverloadIndex()];
619 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
620 return VTy->getElementType();
621 llvm_unreachable("Expected overload type to be a Vector Type");
622 }
623 case IITDescriptor::VecOfBitcastsToInt: {
624 Type *Ty = OverloadTys[D.getOverloadIndex()];
625 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
626 assert(VTy && "Expected overload type to be a Vector Type");
627 return VectorType::getInteger(VTy);
628 }
629 case IITDescriptor::VarArg:
630 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
631 // should never see it here.
632 llvm_unreachable("IITDescriptor::VarArg not expected");
633 }
634 llvm_unreachable("unhandled");
635}
636
637FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
638 ArrayRef<Type *> OverloadTys) {
639 SmallVector<IITDescriptor, 8> Table;
640 auto [TableRef, _, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
641
642 Type *ResultTy = DecodeFixedType(Infos&: TableRef, OverloadTys, Context);
643
644 SmallVector<Type *, 8> ArgTys;
645 while (!TableRef.empty())
646 ArgTys.push_back(Elt: DecodeFixedType(Infos&: TableRef, OverloadTys, Context));
647 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: IsVarArg);
648}
649
650bool Intrinsic::isOverloaded(ID id) {
651#define GET_INTRINSIC_OVERLOAD_TABLE
652#include "llvm/IR/IntrinsicImpl.inc"
653}
654
655bool Intrinsic::isTriviallyScalarizable(ID id) {
656#define GET_INTRINSIC_SCALARIZABLE_TABLE
657#include "llvm/IR/IntrinsicImpl.inc"
658}
659
660bool Intrinsic::hasPrettyPrintedArgs(ID id){
661#define GET_INTRINSIC_PRETTY_PRINT_TABLE
662#include "llvm/IR/IntrinsicImpl.inc"
663}
664
665/// Table of per-target intrinsic name tables.
666#define GET_INTRINSIC_TARGET_DATA
667#include "llvm/IR/IntrinsicImpl.inc"
668
669bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
670 return IID > TargetInfos[0].Count;
671}
672
673/// Looks up Name in NameTable via binary search. NameTable must be sorted
674/// and all entries must start with "llvm.". If NameTable contains an exact
675/// match for Name or a prefix of Name followed by a dot, its index in
676/// NameTable is returned. Otherwise, -1 is returned.
677static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
678 StringRef Name, StringRef Target = "") {
679 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
680 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
681
682 // Do successive binary searches of the dotted name components. For
683 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
684 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
685 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
686 // size 1. During the search, we can skip the prefix that we already know is
687 // identical. By using strncmp we consider names with differing suffixes to
688 // be part of the equal range.
689 size_t CmpEnd = 4; // Skip the "llvm" component.
690 if (!Target.empty())
691 CmpEnd += 1 + Target.size(); // skip the .target component.
692
693 const unsigned *Low = NameOffsetTable.begin();
694 const unsigned *High = NameOffsetTable.end();
695 const unsigned *LastLow = Low;
696 while (CmpEnd < Name.size() && High - Low > 0) {
697 size_t CmpStart = CmpEnd;
698 CmpEnd = Name.find(C: '.', From: CmpStart + 1);
699 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
700 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
701 // `equal_range` requires the comparison to work with either side being an
702 // offset or the value. Detect which kind each side is to set up the
703 // compared strings.
704 const char *LHSStr;
705 if constexpr (std::is_integral_v<decltype(LHS)>)
706 LHSStr = IntrinsicNameTable.getCString(O: LHS);
707 else
708 LHSStr = LHS;
709
710 const char *RHSStr;
711 if constexpr (std::is_integral_v<decltype(RHS)>)
712 RHSStr = IntrinsicNameTable.getCString(O: RHS);
713 else
714 RHSStr = RHS;
715
716 return strncmp(s1: LHSStr + CmpStart, s2: RHSStr + CmpStart, n: CmpEnd - CmpStart) <
717 0;
718 };
719 LastLow = Low;
720 std::tie(args&: Low, args&: High) = std::equal_range(first: Low, last: High, val: Name.data(), comp: Cmp);
721 }
722 if (High - Low > 0)
723 LastLow = Low;
724
725 if (LastLow == NameOffsetTable.end())
726 return -1;
727 StringRef NameFound = IntrinsicNameTable[*LastLow];
728 if (Name == NameFound ||
729 (Name.starts_with(Prefix: NameFound) && Name[NameFound.size()] == '.'))
730 return LastLow - NameOffsetTable.begin();
731 return -1;
732}
733
734/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
735/// target as \c Name, or the generic table if \c Name is not target specific.
736///
737/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
738/// name.
739static std::pair<ArrayRef<unsigned>, StringRef>
740findTargetSubtable(StringRef Name) {
741 assert(Name.starts_with("llvm."));
742
743 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
744 // Drop "llvm." and take the first dotted component. That will be the target
745 // if this is target specific.
746 StringRef Target = Name.drop_front(N: 5).split(Separator: '.').first;
747 auto It = partition_point(
748 Range&: Targets, P: [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
749 // We've either found the target or just fall back to the generic set, which
750 // is always first.
751 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
752 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
753 TI.Name};
754}
755
756/// This does the actual lookup of an intrinsic ID which matches the given
757/// function name.
758Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
759 auto [NameOffsetTable, Target] = findTargetSubtable(Name);
760 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
761 if (Idx == -1)
762 return Intrinsic::not_intrinsic;
763
764 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
765 // an index into a sub-table.
766 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
767 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
768
769 // If the intrinsic is not overloaded, require an exact match. If it is
770 // overloaded, require either exact or prefix match.
771 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
772 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
773 bool IsExactMatch = Name.size() == MatchSize;
774 return IsExactMatch || Intrinsic::isOverloaded(id: ID) ? ID
775 : Intrinsic::not_intrinsic;
776}
777
778/// This defines the "Intrinsic::getAttributes(ID id)" method.
779#define GET_INTRINSIC_ATTRIBUTES
780#include "llvm/IR/IntrinsicImpl.inc"
781
782static Function *
783getOrInsertIntrinsicDeclarationImpl(Module *M, Intrinsic::ID id,
784 ArrayRef<Type *> OverloadTys,
785 FunctionType *FT) {
786 std::string Name = OverloadTys.empty()
787 ? Intrinsic::getName(id).str()
788 : Intrinsic::getName(Id: id, OverloadTys, M, FT);
789 Function *F = cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
790 if (F->getFunctionType() == FT)
791 return F;
792
793 // It's possible that a declaration for this intrinsic already exists with an
794 // incorrect signature, if the signature has changed, but this particular
795 // declaration has not been auto-upgraded yet. In that case, rename the
796 // invalid declaration and insert a new one with the correct signature. The
797 // invalid declaration will get upgraded later.
798 F->setName(F->getName() + ".invalid");
799 return cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
800}
801
802Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
803 ArrayRef<Type *> OverloadTys) {
804 // There can never be multiple globals with the same name of different types,
805 // because intrinsics must be a specific type.
806 FunctionType *FT = getType(Context&: M->getContext(), id, OverloadTys);
807 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT);
808}
809
810Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
811 ArrayRef<Type *> ArgTys) {
812 // If the intrinsic is not overloaded, use the non-overloaded version.
813 if (!Intrinsic::isOverloaded(id))
814 return getOrInsertDeclaration(M, id);
815
816 // Get the intrinsic signature metadata.
817 SmallVector<Intrinsic::IITDescriptor, 8> Table;
818 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
819 FunctionType *FTy = FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: IsVarArg);
820
821 // Automatically determine the overloaded types.
822 SmallVector<Type *, 4> OverloadTys;
823 [[maybe_unused]] bool IsValid = ::isSignatureValid(
824 FTy, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS&: nulls());
825 assert(IsValid && "intrinsic signature mismatch");
826 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT: FTy);
827}
828
829Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
830 return M->getFunction(Name: getName(id));
831}
832
833Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
834 ArrayRef<Type *> OverloadTys,
835 FunctionType *FT) {
836 return M->getFunction(Name: getName(Id: id, OverloadTys, M, FT));
837}
838
839// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
840#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
841#include "llvm/IR/IntrinsicImpl.inc"
842
843// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
844#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
845#include "llvm/IR/IntrinsicImpl.inc"
846
847bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
848 switch (QID) {
849#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
850 case Intrinsic::INTRINSIC:
851#include "llvm/IR/ConstrainedOps.def"
852#undef INSTRUCTION
853 return true;
854 default:
855 return false;
856 }
857}
858
859bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
860 switch (QID) {
861#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
862 case Intrinsic::INTRINSIC: \
863 return ROUND_MODE == 1;
864#include "llvm/IR/ConstrainedOps.def"
865#undef INSTRUCTION
866 default:
867 return false;
868 }
869}
870
871// This class represents a position in the intrinsic's type signature and is
872// used to generate error messages in `matchIntrinsicType`. The printed position
873// can be of the following forms:
874//
875// return
876// return struct element 3
877// return vector element
878// return struct element 3 vector element
879// argument 3
880// argument 3 vector element
881//
882// To support deferred checks also being able to generate these error messages
883// we need to encode the position compactly so that it can be stashed into
884// DeferredIntrinsicMatchInfo below (without materializing it into a string).
885// The class below serves that purpose.
886//
887namespace {
888struct MatchPosition {
889 uint16_t IsRet : 1;
890 uint16_t Num : 15; // Argument number (when IsRet = false).
891 struct Index {
892 uint16_t IsStruct : 1; // If true, this is a struct element with element
893 // index `Num`, else its a vector element.
894 uint16_t Num : 15; // Struct element index.
895 };
896 // We expect this to be just 2 levels deep, since nested structs are not
897 // supported.
898 static constexpr unsigned INDEX_TABLE_SIZE = 2;
899 Index Indices[INDEX_TABLE_SIZE];
900 uint16_t NumIndices = 0;
901
902 void pop_index() {
903 assert(NumIndices > 0 && "cannot pop from empty indices");
904 --NumIndices;
905 }
906
907 void push_struct_element(unsigned ElementNum) {
908 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
909 assert(isInt<15>(ElementNum) && "Element index overflow");
910 Indices[NumIndices].IsStruct = true;
911 Indices[NumIndices++].Num = ElementNum;
912 }
913
914 void push_vector_element() {
915 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
916 Indices[NumIndices].IsStruct = false;
917 Indices[NumIndices++].Num = 0;
918 }
919};
920} // namespace
921
922static raw_ostream &operator<<(raw_ostream &OS, const MatchPosition &Pos) {
923 OS << "intrinsic ";
924
925 if (Pos.IsRet)
926 OS << "return";
927 else
928 OS << "argument " << Pos.Num;
929
930 for (const MatchPosition::Index &Idx :
931 ArrayRef(Pos.Indices).take_front(N: Pos.NumIndices)) {
932 if (Idx.IsStruct)
933 OS << " struct element " << Idx.Num;
934 else
935 OS << " vector element";
936 }
937 return OS;
938}
939
940using DeferredIntrinsicMatchInfo =
941 std::tuple<Type *, ArrayRef<Intrinsic::IITDescriptor>, MatchPosition>;
942
943static bool
944matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
945 MatchPosition Position, SmallVectorImpl<Type *> &OverloadTys,
946 SmallVectorImpl<DeferredIntrinsicMatchInfo> &DeferredChecks,
947 bool IsDeferredCheck, raw_ostream &OS) {
948 using namespace Intrinsic;
949
950 // If we ran out of descriptors, there are too many arguments or returns.
951 if (Infos.empty()) {
952 OS << Position << " too many "
953 << (Position.IsRet ? "returns" : "arguments");
954 return true;
955 }
956
957 // Do this before slicing off the 'front' part
958 auto InfosRef = Infos;
959 auto DeferCheck = [&DeferredChecks, &InfosRef, &Position](Type *T) {
960 DeferredChecks.emplace_back(Args&: T, Args&: InfosRef, Args&: Position);
961 return false;
962 };
963
964 IITDescriptor D = Infos.consume_front();
965
966 // Print error message when the (non-dependent) type for current position is
967 // invalid.
968 auto PrintMsg = [&OS, &Position,
969 Ty](bool IsValid, const Twine &Expected,
970 std::optional<unsigned> OIdx = std::nullopt) -> bool {
971 if (IsValid)
972 return false;
973 OS << Position << " type";
974 if (OIdx)
975 OS << " (overload type " << *OIdx << ")";
976 OS << " expected " << Expected << ", but got " << *Ty;
977 return true;
978 };
979
980 // Print message when an overload type is invalid as a result of its use in
981 // current dependent type. DependentQualifier describes the "function" applied
982 // to the overload type to get the dependent type.
983 auto PrintMsgInvalidOverloadTy =
984 [&OS, &Position, &OverloadTys](const Twine &DependentQualifier,
985 const Twine &Expected,
986 unsigned OIdx) -> bool {
987 OS << Position << " is " << DependentQualifier << " overload type " << OIdx
988 << ", so overload type " << OIdx << " expected " << Expected
989 << ", but got " << *OverloadTys[OIdx];
990 return true;
991 };
992
993 // Print message when a dependent type is invalid.
994 auto PrintMsgInvalidDepType =
995 [&OS, &Position, &OverloadTys,
996 Ty](bool IsValid, const Twine &DependentQualifier, const Twine &Expected,
997 unsigned OIdx) -> bool {
998 if (IsValid)
999 return false;
1000 bool IsMatching = DependentQualifier.isSingleStringRef() &&
1001 DependentQualifier.getSingleStringRef() == "matching";
1002 OS << Position << " type (" << DependentQualifier << " overload type "
1003 << OIdx << ") expected " << Expected;
1004 if (!IsMatching)
1005 OS << " (overload type " << OIdx << " is " << *OverloadTys[OIdx] << ")";
1006 OS << ", but got " << *Ty;
1007 return true;
1008 };
1009
1010 switch (D.Kind) {
1011 case IITDescriptor::Void:
1012 assert(Position.IsRet && Position.NumIndices == 0 &&
1013 "void descriptor expected only for return type");
1014 return PrintMsg(Ty->isVoidTy(), "void");
1015 case IITDescriptor::MMX: {
1016 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
1017 return PrintMsg(VT && VT->getNumElements() == 1 &&
1018 VT->getElementType()->isIntegerTy(BitWidth: 64),
1019 "x86_mmx (<1 x i64>)");
1020 }
1021 case IITDescriptor::AMX:
1022 return PrintMsg(Ty->isX86_AMXTy(), "x86_amx");
1023 case IITDescriptor::Token:
1024 return PrintMsg(Ty->isTokenTy(), "token");
1025 case IITDescriptor::Metadata:
1026 return PrintMsg(Ty->isMetadataTy(), "metadata");
1027 case IITDescriptor::Half:
1028 return PrintMsg(Ty->isHalfTy(), "half");
1029 case IITDescriptor::BFloat:
1030 return PrintMsg(Ty->isBFloatTy(), "bfloat");
1031 case IITDescriptor::Float:
1032 return PrintMsg(Ty->isFloatTy(), "float");
1033 case IITDescriptor::Double:
1034 return PrintMsg(Ty->isDoubleTy(), "double");
1035 case IITDescriptor::Quad:
1036 return PrintMsg(Ty->isFP128Ty(), "fp128");
1037 case IITDescriptor::PPCQuad:
1038 return PrintMsg(Ty->isPPC_FP128Ty(), "ppc_fp128");
1039 case IITDescriptor::Integer:
1040 return PrintMsg(Ty->isIntegerTy(BitWidth: D.IntegerWidth),
1041 "i" + Twine(D.IntegerWidth));
1042 case IITDescriptor::AArch64Svcount:
1043 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1044 cast<TargetExtType>(Val: Ty)->getName() == "aarch64.svcount",
1045 "aarch64.svcount");
1046 case IITDescriptor::WasmExternref:
1047 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1048 cast<TargetExtType>(Val: Ty)->getName() == "wasm.externref",
1049 "wasm.externref");
1050 case IITDescriptor::WasmFuncref:
1051 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1052 cast<TargetExtType>(Val: Ty)->getName() == "wasm.funcref",
1053 "wasm.funcref");
1054 case IITDescriptor::Vector: {
1055 VectorType *VT = dyn_cast<VectorType>(Val: Ty);
1056 StringRef Scalable = D.VectorWidth.isScalable() ? "vscale " : "";
1057 bool HasError =
1058 PrintMsg(VT && VT->getElementCount() == D.VectorWidth,
1059 Twine(Scalable) + "vector with " +
1060 Twine(D.VectorWidth.getKnownMinValue()) + " elements");
1061 if (HasError)
1062 return true;
1063 Position.push_vector_element();
1064 return matchIntrinsicType(Ty: VT->getElementType(), Infos, Position,
1065 OverloadTys, DeferredChecks, IsDeferredCheck, OS);
1066 }
1067 case IITDescriptor::Pointer: {
1068 PointerType *PT = dyn_cast<PointerType>(Val: Ty);
1069 unsigned AS = D.PointerAddressSpace;
1070 bool IsValid = PT && PT->getAddressSpace() == AS;
1071 if (AS == 0)
1072 return PrintMsg(IsValid, "ptr");
1073 return PrintMsg(IsValid, "ptr addrspace(" + Twine(AS) + ")");
1074 }
1075
1076 case IITDescriptor::Struct: {
1077 StructType *ST = dyn_cast<StructType>(Val: Ty);
1078 unsigned EC = D.StructNumElements;
1079 bool HasError = PrintMsg(
1080 ST && ST->isLiteral() && !ST->isPacked() && ST->getNumElements() == EC,
1081 "literal non-packed struct with " + Twine(EC) + " elements");
1082 if (HasError)
1083 return true;
1084
1085 for (const auto &[Idx, ETy] : llvm::enumerate(First: ST->elements())) {
1086 Position.push_struct_element(ElementNum: Idx);
1087 if (matchIntrinsicType(Ty: ETy, Infos, Position, OverloadTys, DeferredChecks,
1088 IsDeferredCheck, OS))
1089 return true;
1090 Position.pop_index();
1091 }
1092 return false;
1093 }
1094
1095 case IITDescriptor::Overloaded: {
1096 unsigned OIdx = D.getOverloadIndex();
1097 assert(OIdx == OverloadTys.size() && !IsDeferredCheck &&
1098 "Table consistency error");
1099 OverloadTys.push_back(Elt: Ty);
1100
1101 IITDescriptor::AnyKindVectorConstraint VC;
1102 IITDescriptor::AnyKindElementConstraint EC;
1103 std::tie(args&: VC, args&: EC) = D.getOverloadConstraints();
1104
1105 bool IsValid = [&]() {
1106 switch (VC) {
1107 case IITDescriptor::VC_None:
1108 return true;
1109 case IITDescriptor::VC_Vector:
1110 return isa<VectorType>(Val: Ty);
1111 case IITDescriptor::VC_Scalar:
1112 return !isa<VectorType>(Val: Ty);
1113 }
1114 llvm_unreachable("invalid vector constraint");
1115 }();
1116
1117 IsValid &= [&]() {
1118 Type *ETy = Ty->getScalarType();
1119 switch (EC) {
1120 case IITDescriptor::EC_None:
1121 return true;
1122 case IITDescriptor::EC_Integer:
1123 return ETy->isIntegerTy();
1124 case IITDescriptor::EC_Float:
1125 return ETy->isFloatingPointTy();
1126 case IITDescriptor::EC_Pointer:
1127 return ETy->isPointerTy();
1128 }
1129 llvm_unreachable("invalid element constraint");
1130 }();
1131
1132 if (IsValid)
1133 return false;
1134
1135 static constexpr StringLiteral VectorKinds[] = {
1136 "",
1137 "vector",
1138 "scalar",
1139 };
1140 static constexpr StringLiteral ElementKinds[] = {
1141 "",
1142 "integer",
1143 "fp",
1144 "pointer",
1145 };
1146
1147 if (EC == IITDescriptor::EC_None) {
1148 // No constraint on element type.
1149 // Expected = any {vector | scalar} type.
1150 StringLiteral VK = ArrayRef(VectorKinds)[VC];
1151 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: VK), OIdx);
1152 }
1153
1154 StringLiteral EK = ArrayRef(ElementKinds)[EC];
1155 switch (VC) {
1156 case IITDescriptor::VC_None:
1157 // Expected = any EK or EK vector.
1158 return PrintMsg(false, formatv(Fmt: "any {0} or {0} vector", Vals&: EK), OIdx);
1159 case IITDescriptor::VC_Vector:
1160 return PrintMsg(false, formatv(Fmt: "any {} vector", Vals&: EK), OIdx);
1161 case IITDescriptor::VC_Scalar:
1162 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: EK), OIdx);
1163 }
1164 llvm_unreachable("invalid vector constraint");
1165 }
1166
1167 case IITDescriptor::Match: {
1168 unsigned OIdx = D.getOverloadIndex();
1169 if (OIdx >= OverloadTys.size())
1170 return IsDeferredCheck || DeferCheck(Ty);
1171 return PrintMsgInvalidDepType(Ty == OverloadTys[OIdx], "matching",
1172 formatv(Fmt: "{}", Vals&: *OverloadTys[OIdx]), OIdx);
1173 }
1174
1175 case IITDescriptor::Extend:
1176 case IITDescriptor::Trunc: {
1177 unsigned OIdx = D.getOverloadIndex();
1178 // If this is a forward reference, defer the check for later.
1179 if (OIdx >= OverloadTys.size())
1180 return IsDeferredCheck || DeferCheck(Ty);
1181
1182 Type *OTy = OverloadTys[OIdx];
1183 bool IsExtend = D.Kind == IITDescriptor::Extend;
1184 StringRef Qualifier = IsExtend ? "extended" : "truncated";
1185 if (!OTy->isIntOrIntVectorTy())
1186 return PrintMsgInvalidOverloadTy(Qualifier, "int or vector of int", OIdx);
1187
1188 Type *NewTy = IsExtend ? OTy->getExtendedType() : OTy->getTruncatedType();
1189 return PrintMsgInvalidDepType(Ty == NewTy, Qualifier, formatv(Fmt: "{}", Vals&: *NewTy),
1190 OIdx);
1191 }
1192 case IITDescriptor::OneNthEltsVec: {
1193 unsigned OIdx = D.getOverloadIndex();
1194 unsigned Divisor = D.getVectorDivisor();
1195 // If this is a forward reference, defer the check for later.
1196 if (OIdx >= OverloadTys.size())
1197 return IsDeferredCheck || DeferCheck(Ty);
1198 Type *OTy = OverloadTys[OIdx];
1199 auto *OVecTy = dyn_cast<VectorType>(Val: OTy);
1200 auto Qualifier = formatv(Fmt: "1/nth (n={}) elements vector of", Vals&: Divisor);
1201 if (!OVecTy)
1202 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1203 if (!OVecTy->getElementCount().isKnownMultipleOf(RHS: Divisor))
1204 return PrintMsgInvalidOverloadTy(
1205 Qualifier, formatv(Fmt: "vector with multiple of {} elements", Vals&: Divisor),
1206 OIdx);
1207 Type *Expected = VectorType::getOneNthElementsVectorType(VTy: OVecTy, Denominator: Divisor);
1208 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1209 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1210 }
1211 case IITDescriptor::SameVecWidth: {
1212 unsigned OIdx = D.getOverloadIndex();
1213 if (OIdx >= OverloadTys.size()) {
1214 // Defer check and subsequent check for the vector element type.
1215 Infos.consume_front();
1216 return IsDeferredCheck || DeferCheck(Ty);
1217 }
1218 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1219 auto *ThisArgVecType = dyn_cast<VectorType>(Val: Ty);
1220 // Both must be vectors of the same number of elements or neither.
1221 StringRef Qualifier = "same vector width of";
1222 if (OVecTy && !ThisArgVecType)
1223 return PrintMsgInvalidDepType(false, Qualifier, "vector", OIdx);
1224 if (!OVecTy && ThisArgVecType)
1225 return PrintMsgInvalidDepType(false, Qualifier, "scalar", OIdx);
1226 Type *EltTy = Ty;
1227 if (ThisArgVecType) {
1228 ElementCount Expected = OVecTy->getElementCount();
1229 if (Expected != ThisArgVecType->getElementCount())
1230 return PrintMsgInvalidDepType(
1231 false, Qualifier, formatv(Fmt: "vector with {} elements", Vals&: Expected),
1232 OIdx);
1233 EltTy = ThisArgVecType->getElementType();
1234 Position.push_vector_element();
1235 }
1236 return matchIntrinsicType(Ty: EltTy, Infos, Position, OverloadTys,
1237 DeferredChecks, IsDeferredCheck, OS);
1238 }
1239 case IITDescriptor::VecOfAnyPtrsToElt: {
1240 unsigned RefOverloadIndex = D.getRefOverloadIndex();
1241 if (RefOverloadIndex >= OverloadTys.size()) {
1242 if (IsDeferredCheck)
1243 return true;
1244 // If forward referencing, already add the pointer-vector type and
1245 // defer the checks for later.
1246 assert(D.getOverloadIndex() == OverloadTys.size() &&
1247 "Table consistency error");
1248 OverloadTys.push_back(Elt: Ty);
1249 return DeferCheck(Ty);
1250 }
1251
1252 if (!IsDeferredCheck) {
1253 assert(D.getOverloadIndex() == OverloadTys.size() &&
1254 "Table consistency error");
1255 OverloadTys.push_back(Elt: Ty);
1256 }
1257
1258 // Verify the overloaded type "matches" the Ref type.
1259 // i.e. Ty is a vector with the same width as Ref and composed of pointers.
1260
1261 StringRef Qualifier = "vector of pointers to elements of";
1262 auto *ReferenceType = dyn_cast<VectorType>(Val: OverloadTys[RefOverloadIndex]);
1263 if (!ReferenceType)
1264 return PrintMsgInvalidOverloadTy(Qualifier, "vector", RefOverloadIndex);
1265
1266 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1267 if (!ThisArgVecTy)
1268 return PrintMsgInvalidDepType(false, Qualifier, "vector",
1269 RefOverloadIndex);
1270
1271 auto ExpectedCount = ReferenceType->getElementCount();
1272 auto Expected =
1273 formatv(Fmt: "vector of pointers with {} elements", Vals&: ExpectedCount);
1274 bool IsValid = ThisArgVecTy->getElementCount() == ExpectedCount &&
1275 ThisArgVecTy->getElementType()->isPointerTy();
1276 return PrintMsgInvalidDepType(IsValid, Qualifier, Expected,
1277 RefOverloadIndex);
1278 }
1279 case IITDescriptor::VecElement: {
1280 unsigned OIdx = D.getOverloadIndex();
1281 if (OIdx >= OverloadTys.size())
1282 return IsDeferredCheck || DeferCheck(Ty);
1283 StringRef Qualifier = "vector element of";
1284 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1285 if (!OVecTy)
1286 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1287 Type *Expected = OVecTy->getElementType();
1288 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1289 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1290 }
1291 case IITDescriptor::Subdivide2:
1292 case IITDescriptor::Subdivide4: {
1293 unsigned OIdx = D.getOverloadIndex();
1294 // If this is a forward reference, defer the check for later.
1295 if (OIdx >= OverloadTys.size())
1296 return IsDeferredCheck || DeferCheck(Ty);
1297
1298 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
1299 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1300 auto Qualifier =
1301 formatv(Fmt: "subdivided by {} vector of", Vals: SubDivs == 1 ? 2 : 4);
1302 if (!OVecTy)
1303 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1304
1305 // TODO: Verify that the element type of the overload type is subdivisible
1306 // by 2 or 4.
1307 Type *Expected = VectorType::getSubdividedVectorType(VTy: OVecTy, NumSubdivs: SubDivs);
1308 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1309 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1310 }
1311 case IITDescriptor::VecOfBitcastsToInt: {
1312 unsigned OIdx = D.getOverloadIndex();
1313 if (OIdx >= OverloadTys.size())
1314 return IsDeferredCheck || DeferCheck(Ty);
1315 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1316 StringRef Qualifier = "vector of bitcasts to int of";
1317 if (!OVecTy)
1318 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1319 Type *Expected = VectorType::getInteger(VTy: OVecTy);
1320 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1321 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1322 }
1323 case IITDescriptor::VarArg:
1324 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
1325 // should never see it here.
1326 llvm_unreachable("IITDescriptor::VarArg not expected");
1327 }
1328 llvm_unreachable("unhandled");
1329}
1330
1331/// Return true if the function type \p FTy is a valid type signature for the
1332/// type constraints specified in the .td file, represented by \p Infos and
1333/// \p IsVarArg. The overloaded types for the intrinsic are pushed to the
1334/// \p OverloadTys vector.
1335///
1336/// If the type is not valid, returns false and prints an error message to
1337/// \p OS.
1338static bool isSignatureValid(FunctionType *FTy,
1339 ArrayRef<Intrinsic::IITDescriptor> &Infos,
1340 unsigned NumArgs, bool IsVarArg,
1341 SmallVectorImpl<Type *> &OverloadTys,
1342 raw_ostream &OS) {
1343 SmallVector<DeferredIntrinsicMatchInfo, 2> DeferredChecks;
1344
1345 assert(!Infos.empty() && "Table consistency error");
1346
1347 MatchPosition Pos;
1348 Pos.IsRet = true;
1349 Pos.Num = 0;
1350
1351 if (matchIntrinsicType(Ty: FTy->getReturnType(), Infos, Position: Pos, OverloadTys,
1352 DeferredChecks, IsDeferredCheck: false, OS))
1353 return false;
1354
1355 if (FTy->getNumParams() != NumArgs) {
1356 OS << "intrinsic has incorrect number of args. Expected " << NumArgs
1357 << ", but got " << FTy->getNumParams();
1358 return false;
1359 }
1360
1361 Pos.IsRet = false;
1362 for (const auto &[Idx, Ty] : llvm::enumerate(First: FTy->params())) {
1363 Pos.Num = Idx;
1364 if (matchIntrinsicType(Ty, Infos, Position: Pos, OverloadTys, DeferredChecks, IsDeferredCheck: false,
1365 OS))
1366 return false;
1367 }
1368
1369 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
1370 auto &[DefTy, DefInfos, DefPosition] = DeferredChecks[I];
1371 if (matchIntrinsicType(Ty: DefTy, Infos&: DefInfos, Position: DefPosition, OverloadTys,
1372 DeferredChecks, IsDeferredCheck: true, OS))
1373 return false;
1374 }
1375
1376 if (!Infos.empty()) {
1377 OS << "intrinsic has too few arguments!";
1378 return false;
1379 }
1380
1381 if (FTy->isVarArg() != IsVarArg) {
1382 if (IsVarArg)
1383 OS << "intrinsic was not defined with variable arguments!";
1384 else
1385 OS << "intrinsic was defined with variable arguments!";
1386 return false;
1387 }
1388
1389 return true;
1390}
1391
1392bool Intrinsic::hasStructReturnType(ID id) {
1393 using namespace Intrinsic;
1394 SmallVector<IITDescriptor> Table;
1395 getIntrinsicInfoTableEntries(id, T&: Table);
1396 return !Table.empty() && Table[0].Kind == IITDescriptor::Struct;
1397}
1398
1399bool Intrinsic::isSignatureValid(Intrinsic::ID ID, FunctionType *FT,
1400 SmallVectorImpl<Type *> &OverloadTys,
1401 raw_ostream &OS) {
1402 if (!ID)
1403 return false;
1404
1405 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1406 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id: ID, T&: Table);
1407
1408 return ::isSignatureValid(FTy: FT, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS);
1409}
1410
1411bool Intrinsic::isSignatureValid(Function *F,
1412 SmallVectorImpl<Type *> &OverloadTys,
1413 raw_ostream &OS) {
1414 return isSignatureValid(ID: F->getIntrinsicID(), FT: F->getFunctionType(),
1415 OverloadTys, OS);
1416}
1417
1418std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
1419 SmallVector<Type *, 4> OverloadTys;
1420 if (!isSignatureValid(F, OverloadTys))
1421 return std::nullopt;
1422
1423 Intrinsic::ID ID = F->getIntrinsicID();
1424 StringRef Name = F->getName();
1425 std::string WantedName =
1426 Intrinsic::getName(Id: ID, OverloadTys, M: F->getParent(), FT: F->getFunctionType());
1427 if (Name == WantedName)
1428 return std::nullopt;
1429
1430 Function *NewDecl = [&] {
1431 if (auto *ExistingGV = F->getParent()->getNamedValue(Name: WantedName)) {
1432 if (auto *ExistingF = dyn_cast<Function>(Val: ExistingGV))
1433 if (ExistingF->getFunctionType() == F->getFunctionType())
1434 return ExistingF;
1435
1436 // The name already exists, but is not a function or has the wrong
1437 // prototype. Make place for the new one by renaming the old version.
1438 // Either this old version will be removed later on or the module is
1439 // invalid and we'll get an error.
1440 ExistingGV->setName(WantedName + ".renamed");
1441 }
1442 return Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys);
1443 }();
1444
1445 NewDecl->setCallingConv(F->getCallingConv());
1446 assert(NewDecl->getFunctionType() == F->getFunctionType() &&
1447 "Shouldn't change the signature");
1448 return NewDecl;
1449}
1450
1451struct InterleaveIntrinsic {
1452 Intrinsic::ID Interleave, Deinterleave;
1453};
1454
1455static InterleaveIntrinsic InterleaveIntrinsics[] = {
1456 {.Interleave: Intrinsic::vector_interleave2, .Deinterleave: Intrinsic::vector_deinterleave2},
1457 {.Interleave: Intrinsic::vector_interleave3, .Deinterleave: Intrinsic::vector_deinterleave3},
1458 {.Interleave: Intrinsic::vector_interleave4, .Deinterleave: Intrinsic::vector_deinterleave4},
1459 {.Interleave: Intrinsic::vector_interleave5, .Deinterleave: Intrinsic::vector_deinterleave5},
1460 {.Interleave: Intrinsic::vector_interleave6, .Deinterleave: Intrinsic::vector_deinterleave6},
1461 {.Interleave: Intrinsic::vector_interleave7, .Deinterleave: Intrinsic::vector_deinterleave7},
1462 {.Interleave: Intrinsic::vector_interleave8, .Deinterleave: Intrinsic::vector_deinterleave8},
1463};
1464
1465Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
1466 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1467 return InterleaveIntrinsics[Factor - 2].Interleave;
1468}
1469
1470Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
1471 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1472 return InterleaveIntrinsics[Factor - 2].Deinterleave;
1473}
1474
1475LLVM_ABI void Intrinsic::printFPClassMask(raw_ostream &OS,
1476 const Constant *ImmArgVal) {
1477 uint64_t Val = cast<ConstantInt>(Val: ImmArgVal)->getZExtValue();
1478 OS << static_cast<FPClassTest>(Val);
1479}
1480
1481#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1482#include "llvm/IR/IntrinsicImpl.inc"
1483
1484// Emit the default-argument values table and lookup function
1485// (Intrinsic::getAllDefaultArgValues).
1486#define GET_INTRINSIC_DEFAULT_ARG_VALUES
1487#include "llvm/IR/IntrinsicImpl.inc"
1488