1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions required for supporting intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Intrinsics.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringTable.h"
16#include "llvm/IR/ConstantRange.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsPowerPC.h"
26#include "llvm/IR/IntrinsicsR600.h"
27#include "llvm/IR/IntrinsicsRISCV.h"
28#include "llvm/IR/IntrinsicsS390.h"
29#include "llvm/IR/IntrinsicsSPIRV.h"
30#include "llvm/IR/IntrinsicsVE.h"
31#include "llvm/IR/IntrinsicsX86.h"
32#include "llvm/IR/IntrinsicsXCore.h"
33#include "llvm/IR/Module.h"
34#include "llvm/IR/NVVMIntrinsicUtils.h"
35#include "llvm/IR/Type.h"
36#include "llvm/Support/FormatVariadic.h"
37#include "llvm/Support/MathExtras.h"
38
39using namespace llvm;
40
41// Forward declaration of static functions.
42static bool isSignatureValid(FunctionType *FTy,
43 ArrayRef<Intrinsic::IITDescriptor> &Infos,
44 unsigned NumArgs, bool IsVarArg,
45 SmallVectorImpl<Type *> &OverloadTys,
46 raw_ostream &OS,
47 unsigned NumMissingTrailingParams = 0);
48
49/// Table of string intrinsic names indexed by enum value.
50#define GET_INTRINSIC_NAME_TABLE
51#include "llvm/IR/IntrinsicImpl.inc"
52
53/// Table of required target features indexed by enum value.
54#define GET_INTRINSIC_TARGET_FEATURES_TABLE
55#include "llvm/IR/IntrinsicImpl.inc"
56
57StringRef Intrinsic::getBaseName(ID id) {
58 assert(id < num_intrinsics && "Invalid intrinsic ID!");
59 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
60}
61
62StringRef Intrinsic::getRequiredTargetFeatures(ID id) {
63 assert(id < num_intrinsics && "invalid intrinsic ID!");
64 return IntrinsicTargetFeaturesTable[IntrinsicTargetFeaturesOffsetTable[id]];
65}
66
67StringRef Intrinsic::getName(ID id) {
68 assert(id < num_intrinsics && "Invalid intrinsic ID!");
69 assert(!Intrinsic::isOverloaded(id) &&
70 "This version of getName does not support overloading");
71 return getBaseName(id);
72}
73
74/// Returns a stable mangling for the type specified for use in the name
75/// mangling scheme used by 'any' types in intrinsic signatures. The mangling
76/// of named types is simply their name. Manglings for unnamed types consist
77/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
78/// combined with the mangling of their component types. A vararg function
79/// type will have a suffix of 'vararg'. Since function types can contain
80/// other function types, we close a function type mangling with suffix 'f'
81/// which can't be confused with it's prefix. This ensures we don't have
82/// collisions between two unrelated function types. Otherwise, you might
83/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)
84/// The HasUnnamedType boolean is set if an unnamed type was encountered,
85/// indicating that extra care must be taken to ensure a unique name.
86static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
87 std::string Result;
88 if (PointerType *PTyp = dyn_cast<PointerType>(Val: Ty)) {
89 Result += "p" + utostr(X: PTyp->getAddressSpace());
90 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Val: Ty)) {
91 Result += "a" + utostr(X: ATyp->getNumElements()) +
92 getMangledTypeStr(Ty: ATyp->getElementType(), HasUnnamedType);
93 } else if (StructType *STyp = dyn_cast<StructType>(Val: Ty)) {
94 if (!STyp->isLiteral()) {
95 Result += "s_";
96 if (STyp->hasName())
97 Result += STyp->getName();
98 else
99 HasUnnamedType = true;
100 } else {
101 Result += "sl_";
102 for (auto *Elem : STyp->elements())
103 Result += getMangledTypeStr(Ty: Elem, HasUnnamedType);
104 }
105 // Ensure nested structs are distinguishable.
106 Result += "s";
107 } else if (FunctionType *FT = dyn_cast<FunctionType>(Val: Ty)) {
108 Result += "f_" + getMangledTypeStr(Ty: FT->getReturnType(), HasUnnamedType);
109 for (size_t i = 0; i < FT->getNumParams(); i++)
110 Result += getMangledTypeStr(Ty: FT->getParamType(i), HasUnnamedType);
111 if (FT->isVarArg())
112 Result += "vararg";
113 // Ensure nested function types are distinguishable.
114 Result += "f";
115 } else if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
116 ElementCount EC = VTy->getElementCount();
117 if (EC.isScalable())
118 Result += "nx";
119 Result += "v" + utostr(X: EC.getKnownMinValue()) +
120 getMangledTypeStr(Ty: VTy->getElementType(), HasUnnamedType);
121 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Val: Ty)) {
122 Result += "t";
123 Result += TETy->getName();
124 for (Type *ParamTy : TETy->type_params())
125 Result += "_" + getMangledTypeStr(Ty: ParamTy, HasUnnamedType);
126 for (unsigned IntParam : TETy->int_params())
127 Result += "_" + utostr(X: IntParam);
128 // Ensure nested target extension types are distinguishable.
129 Result += "t";
130 } else if (Ty) {
131 switch (Ty->getTypeID()) {
132 default:
133 llvm_unreachable("Unhandled type");
134 case Type::VoidTyID:
135 Result += "isVoid";
136 break;
137 case Type::MetadataTyID:
138 Result += "Metadata";
139 break;
140 case Type::HalfTyID:
141 Result += "f16";
142 break;
143 case Type::BFloatTyID:
144 Result += "bf16";
145 break;
146 case Type::FloatTyID:
147 Result += "f32";
148 break;
149 case Type::DoubleTyID:
150 Result += "f64";
151 break;
152 case Type::X86_FP80TyID:
153 Result += "f80";
154 break;
155 case Type::FP128TyID:
156 Result += "f128";
157 break;
158 case Type::PPC_FP128TyID:
159 Result += "ppcf128";
160 break;
161 case Type::X86_AMXTyID:
162 Result += "x86amx";
163 break;
164 case Type::IntegerTyID:
165 Result += "i" + utostr(X: cast<IntegerType>(Val: Ty)->getBitWidth());
166 break;
167 case Type::ByteTyID:
168 Result += "b" + utostr(X: cast<ByteType>(Val: Ty)->getBitWidth());
169 break;
170 }
171 }
172 return Result;
173}
174
175static std::string getIntrinsicNameImpl(Intrinsic::ID Id,
176 ArrayRef<Type *> OverloadTys, Module *M,
177 FunctionType *FT,
178 bool EarlyModuleCheck) {
179
180 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
181 assert((OverloadTys.empty() || Intrinsic::isOverloaded(Id)) &&
182 "This version of getName is for overloaded intrinsics only");
183 (void)EarlyModuleCheck;
184 assert((!EarlyModuleCheck || M ||
185 !any_of(OverloadTys, llvm::IsaPred<PointerType>)) &&
186 "Intrinsic overloading on pointer types need to provide a Module");
187 bool HasUnnamedType = false;
188 std::string Result(Intrinsic::getBaseName(id: Id));
189 for (Type *Ty : OverloadTys)
190 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
191 if (HasUnnamedType) {
192 assert(M && "unnamed types need a module");
193 if (!FT)
194 FT = Intrinsic::getType(Context&: M->getContext(), id: Id, OverloadTys);
195 else
196 assert(FT == Intrinsic::getType(M->getContext(), Id, OverloadTys) &&
197 "Provided FunctionType must match arguments");
198 return M->getUniqueIntrinsicName(BaseName: Result, Id, Proto: FT);
199 }
200 return Result;
201}
202
203std::string Intrinsic::getName(ID Id, ArrayRef<Type *> OverloadTys, Module *M,
204 FunctionType *FT) {
205 assert(M && "We need to have a Module");
206 return getIntrinsicNameImpl(Id, OverloadTys, M, FT, EarlyModuleCheck: true);
207}
208
209std::string Intrinsic::getNameNoUnnamedTypes(ID Id,
210 ArrayRef<Type *> OverloadTys) {
211 return getIntrinsicNameImpl(Id, OverloadTys, M: nullptr, FT: nullptr, EarlyModuleCheck: false);
212}
213
214/// IIT_Info - These are enumerators that describe the entries returned by the
215/// getIntrinsicInfoTableEntries function.
216///
217/// Defined in Intrinsics.td.
218enum IIT_Info {
219#define GET_INTRINSIC_IITINFO
220#include "llvm/IR/IntrinsicImpl.inc"
221};
222
223static_assert(IIT_Done == 0, "IIT_Done expected to be 0");
224
225static void
226DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
227 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
228 using namespace Intrinsic;
229
230 auto IsScalableVector = [&]() {
231 IIT_Info NextInfo = IIT_Info(Infos[NextElt]);
232 if (NextInfo != IIT_SCALABLE_VEC)
233 return false;
234 // Eat the IIT_SCALABLE_VEC token.
235 ++NextElt;
236 return true;
237 };
238
239 IIT_Info Info = IIT_Info(Infos[NextElt++]);
240
241 switch (Info) {
242 case IIT_Done:
243 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Void, Field: 0));
244 return;
245 case IIT_VARARG:
246 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VarArg, Field: 0));
247 return;
248 case IIT_MMX:
249 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::MMX, Field: 0));
250 return;
251 case IIT_AMX:
252 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AMX, Field: 0));
253 return;
254 case IIT_TOKEN:
255 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Token, Field: 0));
256 return;
257 case IIT_METADATA:
258 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Metadata, Field: 0));
259 return;
260 case IIT_F16:
261 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Half, Field: 0));
262 return;
263 case IIT_BF16:
264 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::BFloat, Field: 0));
265 return;
266 case IIT_F32:
267 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Float, Field: 0));
268 return;
269 case IIT_F64:
270 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Double, Field: 0));
271 return;
272 case IIT_F128:
273 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Quad, Field: 0));
274 return;
275 case IIT_PPCF128:
276 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::PPCQuad, Field: 0));
277 return;
278 case IIT_I1:
279 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 1));
280 return;
281 case IIT_I2:
282 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 2));
283 return;
284 case IIT_I4:
285 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 4));
286 return;
287 case IIT_AARCH64_SVCOUNT:
288 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AArch64Svcount, Field: 0));
289 return;
290 case IIT_I8:
291 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 8));
292 return;
293 case IIT_I16:
294 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 16));
295 return;
296 case IIT_I32:
297 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 32));
298 return;
299 case IIT_I64:
300 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 64));
301 return;
302 case IIT_I128:
303 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 128));
304 return;
305 case IIT_V1:
306 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1, IsScalable: IsScalableVector()));
307 DecodeIITType(NextElt, Infos, OutputTable);
308 return;
309 case IIT_V2:
310 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2, IsScalable: IsScalableVector()));
311 DecodeIITType(NextElt, Infos, OutputTable);
312 return;
313 case IIT_V3:
314 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 3, IsScalable: IsScalableVector()));
315 DecodeIITType(NextElt, Infos, OutputTable);
316 return;
317 case IIT_V4:
318 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4, IsScalable: IsScalableVector()));
319 DecodeIITType(NextElt, Infos, OutputTable);
320 return;
321 case IIT_V6:
322 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 6, IsScalable: IsScalableVector()));
323 DecodeIITType(NextElt, Infos, OutputTable);
324 return;
325 case IIT_V8:
326 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 8, IsScalable: IsScalableVector()));
327 DecodeIITType(NextElt, Infos, OutputTable);
328 return;
329 case IIT_V10:
330 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 10, IsScalable: IsScalableVector()));
331 DecodeIITType(NextElt, Infos, OutputTable);
332 return;
333 case IIT_V16:
334 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 16, IsScalable: IsScalableVector()));
335 DecodeIITType(NextElt, Infos, OutputTable);
336 return;
337 case IIT_V32:
338 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 32, IsScalable: IsScalableVector()));
339 DecodeIITType(NextElt, Infos, OutputTable);
340 return;
341 case IIT_V64:
342 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 64, IsScalable: IsScalableVector()));
343 DecodeIITType(NextElt, Infos, OutputTable);
344 return;
345 case IIT_V128:
346 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 128, IsScalable: IsScalableVector()));
347 DecodeIITType(NextElt, Infos, OutputTable);
348 return;
349 case IIT_V256:
350 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 256, IsScalable: IsScalableVector()));
351 DecodeIITType(NextElt, Infos, OutputTable);
352 return;
353 case IIT_V512:
354 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 512, IsScalable: IsScalableVector()));
355 DecodeIITType(NextElt, Infos, OutputTable);
356 return;
357 case IIT_V1024:
358 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1024, IsScalable: IsScalableVector()));
359 DecodeIITType(NextElt, Infos, OutputTable);
360 return;
361 case IIT_V2048:
362 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2048, IsScalable: IsScalableVector()));
363 DecodeIITType(NextElt, Infos, OutputTable);
364 return;
365 case IIT_V4096:
366 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4096, IsScalable: IsScalableVector()));
367 DecodeIITType(NextElt, Infos, OutputTable);
368 return;
369 case IIT_EXTERNREF:
370 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmExternref, Field: 0));
371 return;
372 case IIT_FUNCREF:
373 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmFuncref, Field: 0));
374 return;
375 case IIT_PTR:
376 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 0));
377 return;
378 case IIT_PTR_AS: // pointer with address space.
379 OutputTable.push_back(
380 Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: Infos[NextElt++]));
381 return;
382 case IIT_ANY: {
383 unsigned OverloadIndex = Infos[NextElt++];
384 unsigned ArgKindEnums = Infos[NextElt++];
385 unsigned Packed = (ArgKindEnums << 8) | OverloadIndex;
386 OutputTable.push_back(
387 Elt: IITDescriptor::get(K: IITDescriptor::Overloaded, Field: Packed));
388 return;
389 }
390 case IIT_MATCH: {
391 unsigned OverloadIndex = Infos[NextElt++];
392 OutputTable.push_back(
393 Elt: IITDescriptor::get(K: IITDescriptor::Match, Field: OverloadIndex));
394 return;
395 }
396 case IIT_EXTEND_ARG: {
397 unsigned OverloadIndex = Infos[NextElt++];
398 OutputTable.push_back(
399 Elt: IITDescriptor::get(K: IITDescriptor::Extend, Field: OverloadIndex));
400 return;
401 }
402 case IIT_TRUNC_ARG: {
403 unsigned OverloadIndex = Infos[NextElt++];
404 OutputTable.push_back(
405 Elt: IITDescriptor::get(K: IITDescriptor::Trunc, Field: OverloadIndex));
406 return;
407 }
408 case IIT_ONE_NTH_ELTS_VEC_ARG: {
409 unsigned short OverloadIndex = Infos[NextElt++];
410 unsigned short N = Infos[NextElt++];
411 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::OneNthEltsVec,
412 /*Hi=*/N, /*Lo=*/OverloadIndex));
413 return;
414 }
415 case IIT_SAME_VEC_WIDTH_ARG: {
416 unsigned OverloadIndex = Infos[NextElt++];
417 OutputTable.push_back(
418 Elt: IITDescriptor::get(K: IITDescriptor::SameVecWidth, Field: OverloadIndex));
419 // IIT_SAME_VEC_WIDTH_ARG entry is followed by the element type.
420 DecodeIITType(NextElt, Infos, OutputTable);
421 return;
422 }
423 case IIT_ANY_SAME_VEC_WIDTH_ARG: {
424 unsigned short OverloadIndex = Infos[NextElt++];
425 unsigned short RefOverloadIndex = Infos[NextElt++];
426 unsigned short ElementConstraint = Infos[NextElt++];
427 // Pack the element constraint into bits [4:7], leaving the low 4 bits for
428 // this type's overload index.
429 OutputTable.push_back(Elt: IITDescriptor::get(
430 K: IITDescriptor::AnySameVecWidth, /*Hi=*/RefOverloadIndex,
431 /*Lo=*/ElementConstraint << 4 | OverloadIndex));
432 return;
433 }
434 case IIT_STRUCT: {
435 unsigned StructElts = Infos[NextElt++] + 2;
436
437 OutputTable.push_back(
438 Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: StructElts));
439
440 for (unsigned i = 0; i != StructElts; ++i)
441 DecodeIITType(NextElt, Infos, OutputTable);
442 return;
443 }
444 case IIT_SUBDIVIDE2_ARG: {
445 unsigned OverloadIndex = Infos[NextElt++];
446 OutputTable.push_back(
447 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide2, Field: OverloadIndex));
448 return;
449 }
450 case IIT_SUBDIVIDE4_ARG: {
451 unsigned OverloadIndex = Infos[NextElt++];
452 OutputTable.push_back(
453 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide4, Field: OverloadIndex));
454 return;
455 }
456 case IIT_VEC_ELEMENT: {
457 unsigned OverloadIndex = Infos[NextElt++];
458 OutputTable.push_back(
459 Elt: IITDescriptor::get(K: IITDescriptor::VecElement, Field: OverloadIndex));
460 return;
461 }
462 case IIT_VEC_OF_BITCASTS_TO_INT: {
463 unsigned OverloadIndex = Infos[NextElt++];
464 OutputTable.push_back(
465 Elt: IITDescriptor::get(K: IITDescriptor::VecOfBitcastsToInt, Field: OverloadIndex));
466 return;
467 }
468 case IIT_SCALABLE_VEC:
469 break;
470 }
471 llvm_unreachable("unhandled");
472}
473
474#define GET_INTRINSIC_GENERATOR_GLOBAL
475#include "llvm/IR/IntrinsicImpl.inc"
476
477std::tuple<ArrayRef<Intrinsic::IITDescriptor>, unsigned, bool>
478Intrinsic::getIntrinsicInfoTableEntries(ID id,
479 SmallVectorImpl<IITDescriptor> &T) {
480 // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
481 // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
482 // IntrinsicEmitter.cpp.
483 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
484 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
485 // Mask with all bits 1 except the most significant bit.
486 constexpr unsigned Mask = (1U << MSBPosition) - 1;
487
488 FixedEncodingTy TableVal = IIT_Table[id - 1];
489
490 // Array to hold the inlined fixed encoding values expanded from nibbles to
491 // bytes. Its size can be be atmost FixedEncodingBits / 4 i.e., number
492 // of nibbles that can fit in `FixedEncodingTy` + 1 (the IIT_Done terminator
493 // that is not explicitly encoded). Note that if there are trailing 0 bytes
494 // in the encoding (for example, payload following one of the IIT tokens),
495 // the inlined encoding does not encode the actual size of the encoding, so
496 // we always assume its size of this maximum length possible, followed by the
497 // IIT_Done terminator token (whose value is 0).
498 unsigned char IITValues[FixedEncodingBits / 4 + 1] = {0};
499
500 ArrayRef<unsigned char> IITEntries;
501 unsigned NextElt = 0;
502 // Check to see if the intrinsic's type was inlined in the fixed encoding
503 // table.
504 if (TableVal >> MSBPosition) {
505 // This is an offset into the IIT_LongEncodingTable.
506 IITEntries = IIT_LongEncodingTable;
507
508 // Strip sentinel bit.
509 NextElt = TableVal & Mask;
510 } else {
511 // If the entry was encoded into a single word in the table itself, decode
512 // it from an array of nibbles to an array of bytes.
513 do {
514 IITValues[NextElt++] = TableVal & 0xF;
515 TableVal >>= 4;
516 } while (TableVal);
517
518 IITEntries = IITValues;
519 NextElt = 0;
520 }
521
522 // Okay, decode the table into the output vector of IITDescriptors.
523 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
524 unsigned NumArgs = 0;
525 while (IITEntries[NextElt] != IIT_Done) {
526 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
527 ++NumArgs;
528 }
529
530 ArrayRef<IITDescriptor> TableRef = T;
531
532 bool IsVarArg = false;
533 if (TableRef.back().Kind == Intrinsic::IITDescriptor::VarArg) {
534 IsVarArg = true;
535 TableRef.consume_back();
536 --NumArgs;
537 }
538 return {TableRef, NumArgs, IsVarArg};
539}
540
541static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
542 ArrayRef<Type *> OverloadTys,
543 LLVMContext &Context) {
544 using namespace Intrinsic;
545
546 IITDescriptor D = Infos.consume_front();
547
548 switch (D.Kind) {
549 case IITDescriptor::Void:
550 return Type::getVoidTy(C&: Context);
551 case IITDescriptor::MMX:
552 return llvm::FixedVectorType::get(ElementType: llvm::IntegerType::get(C&: Context, NumBits: 64), NumElts: 1);
553 case IITDescriptor::AMX:
554 return Type::getX86_AMXTy(C&: Context);
555 case IITDescriptor::Token:
556 return Type::getTokenTy(C&: Context);
557 case IITDescriptor::Metadata:
558 return Type::getMetadataTy(C&: Context);
559 case IITDescriptor::Half:
560 return Type::getHalfTy(C&: Context);
561 case IITDescriptor::BFloat:
562 return Type::getBFloatTy(C&: Context);
563 case IITDescriptor::Float:
564 return Type::getFloatTy(C&: Context);
565 case IITDescriptor::Double:
566 return Type::getDoubleTy(C&: Context);
567 case IITDescriptor::Quad:
568 return Type::getFP128Ty(C&: Context);
569 case IITDescriptor::PPCQuad:
570 return Type::getPPC_FP128Ty(C&: Context);
571 case IITDescriptor::AArch64Svcount:
572 return TargetExtType::get(Context, Name: "aarch64.svcount");
573 case IITDescriptor::WasmExternref:
574 return TargetExtType::get(Context, Name: "wasm.externref");
575 case IITDescriptor::WasmFuncref:
576 return TargetExtType::get(Context, Name: "wasm.funcref");
577 case IITDescriptor::Integer:
578 return IntegerType::get(C&: Context, NumBits: D.IntegerWidth);
579 case IITDescriptor::Vector:
580 return VectorType::get(ElementType: DecodeFixedType(Infos, OverloadTys, Context),
581 EC: D.VectorWidth);
582 case IITDescriptor::Pointer:
583 return PointerType::get(C&: Context, AddressSpace: D.PointerAddressSpace);
584 case IITDescriptor::Struct: {
585 SmallVector<Type *, 8> Elts;
586 for (unsigned i = 0, e = D.StructNumElements; i != e; ++i)
587 Elts.push_back(Elt: DecodeFixedType(Infos, OverloadTys, Context));
588 return StructType::get(Context, Elements: Elts);
589 }
590 // For any overload type or partially dependent type, substitute it with the
591 // corresponding concrete type from OverloadTys. Additionally, do the same
592 // for the fully dependent type that matches an overload type.
593 case IITDescriptor::Overloaded:
594 case IITDescriptor::AnySameVecWidth:
595 case IITDescriptor::Match:
596 return OverloadTys[D.getOverloadIndex()];
597 case IITDescriptor::Extend:
598 return OverloadTys[D.getOverloadIndex()]->getExtendedType();
599 case IITDescriptor::Trunc:
600 return OverloadTys[D.getOverloadIndex()]->getTruncatedType();
601 case IITDescriptor::Subdivide2:
602 case IITDescriptor::Subdivide4: {
603 Type *Ty = OverloadTys[D.getOverloadIndex()];
604 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
605 assert(VTy && "Expected overload type to be a Vector Type");
606 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
607 return VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
608 }
609 case IITDescriptor::OneNthEltsVec:
610 return VectorType::getOneNthElementsVectorType(
611 VTy: cast<VectorType>(Val: OverloadTys[D.getOverloadIndex()]),
612 Denominator: D.getVectorDivisor());
613 case IITDescriptor::SameVecWidth: {
614 Type *EltTy = DecodeFixedType(Infos, OverloadTys, Context);
615 Type *Ty = OverloadTys[D.getOverloadIndex()];
616 if (auto *VTy = dyn_cast<VectorType>(Val: Ty))
617 return VectorType::get(ElementType: EltTy, EC: VTy->getElementCount());
618 return EltTy;
619 }
620 case IITDescriptor::VecElement: {
621 Type *Ty = OverloadTys[D.getOverloadIndex()];
622 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
623 return VTy->getElementType();
624 llvm_unreachable("Expected overload type to be a Vector Type");
625 }
626 case IITDescriptor::VecOfBitcastsToInt: {
627 Type *Ty = OverloadTys[D.getOverloadIndex()];
628 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
629 assert(VTy && "Expected overload type to be a Vector Type");
630 return VectorType::getInteger(VTy);
631 }
632 case IITDescriptor::VarArg:
633 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
634 // should never see it here.
635 llvm_unreachable("IITDescriptor::VarArg not expected");
636 }
637 llvm_unreachable("unhandled");
638}
639
640FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
641 ArrayRef<Type *> OverloadTys) {
642 SmallVector<IITDescriptor, 8> Table;
643 auto [TableRef, _, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
644
645 Type *ResultTy = DecodeFixedType(Infos&: TableRef, OverloadTys, Context);
646
647 SmallVector<Type *, 8> ArgTys;
648 while (!TableRef.empty())
649 ArgTys.push_back(Elt: DecodeFixedType(Infos&: TableRef, OverloadTys, Context));
650 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: IsVarArg);
651}
652
653bool Intrinsic::isOverloaded(ID id) {
654#define GET_INTRINSIC_OVERLOAD_TABLE
655#include "llvm/IR/IntrinsicImpl.inc"
656}
657
658bool Intrinsic::isTriviallyScalarizable(ID id) {
659#define GET_INTRINSIC_SCALARIZABLE_TABLE
660#include "llvm/IR/IntrinsicImpl.inc"
661}
662
663bool Intrinsic::hasPrettyPrintedArgs(ID id){
664#define GET_INTRINSIC_PRETTY_PRINT_TABLE
665#include "llvm/IR/IntrinsicImpl.inc"
666}
667
668/// Table of per-target intrinsic name tables.
669#define GET_INTRINSIC_TARGET_DATA
670#include "llvm/IR/IntrinsicImpl.inc"
671
672bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
673 return IID > TargetInfos[0].Count;
674}
675
676/// Looks up Name in NameTable via binary search. NameTable must be sorted
677/// and all entries must start with "llvm.". If NameTable contains an exact
678/// match for Name or a prefix of Name followed by a dot, its index in
679/// NameTable is returned. Otherwise, -1 is returned.
680static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
681 StringRef Name, StringRef Target = "") {
682 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
683 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
684
685 // Do successive binary searches of the dotted name components. For
686 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
687 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
688 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
689 // size 1. During the search, we can skip the prefix that we already know is
690 // identical. By using strncmp we consider names with differing suffixes to
691 // be part of the equal range.
692 size_t CmpEnd = 4; // Skip the "llvm" component.
693 if (!Target.empty())
694 CmpEnd += 1 + Target.size(); // skip the .target component.
695
696 const unsigned *Low = NameOffsetTable.begin();
697 const unsigned *High = NameOffsetTable.end();
698 const unsigned *LastLow = Low;
699 while (CmpEnd < Name.size() && High - Low > 0) {
700 size_t CmpStart = CmpEnd;
701 CmpEnd = Name.find(C: '.', From: CmpStart + 1);
702 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
703 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
704 // `equal_range` requires the comparison to work with either side being an
705 // offset or the value. Detect which kind each side is to set up the
706 // compared strings.
707 const char *LHSStr;
708 if constexpr (std::is_integral_v<decltype(LHS)>)
709 LHSStr = IntrinsicNameTable.getCString(O: LHS);
710 else
711 LHSStr = LHS;
712
713 const char *RHSStr;
714 if constexpr (std::is_integral_v<decltype(RHS)>)
715 RHSStr = IntrinsicNameTable.getCString(O: RHS);
716 else
717 RHSStr = RHS;
718
719 return strncmp(s1: LHSStr + CmpStart, s2: RHSStr + CmpStart, n: CmpEnd - CmpStart) <
720 0;
721 };
722 LastLow = Low;
723 std::tie(args&: Low, args&: High) = std::equal_range(first: Low, last: High, val: Name.data(), comp: Cmp);
724 }
725 if (High - Low > 0)
726 LastLow = Low;
727
728 if (LastLow == NameOffsetTable.end())
729 return -1;
730 StringRef NameFound = IntrinsicNameTable[*LastLow];
731 if (Name == NameFound ||
732 (Name.starts_with(Prefix: NameFound) && Name[NameFound.size()] == '.'))
733 return LastLow - NameOffsetTable.begin();
734 return -1;
735}
736
737/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
738/// target as \c Name, or the generic table if \c Name is not target specific.
739///
740/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
741/// name.
742static std::pair<ArrayRef<unsigned>, StringRef>
743findTargetSubtable(StringRef Name) {
744 assert(Name.starts_with("llvm."));
745
746 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
747 // Drop "llvm." and take the first dotted component. That will be the target
748 // if this is target specific.
749 StringRef Target = Name.drop_front(N: 5).split(Separator: '.').first;
750 auto It = partition_point(
751 Range&: Targets, P: [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
752 // We've either found the target or just fall back to the generic set, which
753 // is always first.
754 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
755 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
756 TI.Name};
757}
758
759/// This does the actual lookup of an intrinsic ID which matches the given
760/// function name.
761Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
762 auto [NameOffsetTable, Target] = findTargetSubtable(Name);
763 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
764 if (Idx == -1)
765 return Intrinsic::not_intrinsic;
766
767 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
768 // an index into a sub-table.
769 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
770 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
771
772 // If the intrinsic is not overloaded, require an exact match. If it is
773 // overloaded, require either exact or prefix match.
774 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
775 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
776 bool IsExactMatch = Name.size() == MatchSize;
777 return IsExactMatch || Intrinsic::isOverloaded(id: ID) ? ID
778 : Intrinsic::not_intrinsic;
779}
780
781/// This defines the "Intrinsic::getAttributes(ID id)" method.
782#define GET_INTRINSIC_ATTRIBUTES
783#include "llvm/IR/IntrinsicImpl.inc"
784
785static Function *
786getOrInsertIntrinsicDeclarationImpl(Module *M, Intrinsic::ID id,
787 ArrayRef<Type *> OverloadTys,
788 FunctionType *FT) {
789 std::string Name = OverloadTys.empty()
790 ? Intrinsic::getName(id).str()
791 : Intrinsic::getName(Id: id, OverloadTys, M, FT);
792 Function *F = cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
793 if (F->getFunctionType() == FT)
794 return F;
795
796 // It's possible that a declaration for this intrinsic already exists with an
797 // incorrect signature, if the signature has changed, but this particular
798 // declaration has not been auto-upgraded yet. In that case, rename the
799 // invalid declaration and insert a new one with the correct signature. The
800 // invalid declaration will get upgraded later.
801 F->setName(F->getName() + ".invalid");
802 return cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
803}
804
805Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
806 ArrayRef<Type *> OverloadTys) {
807 // There can never be multiple globals with the same name of different types,
808 // because intrinsics must be a specific type.
809 FunctionType *FT = getType(Context&: M->getContext(), id, OverloadTys);
810 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT);
811}
812
813Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
814 ArrayRef<Type *> ArgTys) {
815 // If the intrinsic is not overloaded, use the non-overloaded version.
816 if (!Intrinsic::isOverloaded(id))
817 return getOrInsertDeclaration(M, id);
818
819 // Get the intrinsic signature metadata.
820 SmallVector<Intrinsic::IITDescriptor, 8> Table;
821 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
822 FunctionType *FTy = FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: IsVarArg);
823
824 // Automatically determine the overloaded types.
825 SmallVector<Type *, 4> OverloadTys;
826 [[maybe_unused]] bool IsValid = ::isSignatureValid(
827 FTy, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS&: nulls());
828 assert(IsValid && "intrinsic signature mismatch");
829 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT: FTy);
830}
831
832Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
833 return M->getFunction(Name: getName(id));
834}
835
836Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
837 ArrayRef<Type *> OverloadTys,
838 FunctionType *FT) {
839 return M->getFunction(Name: getName(Id: id, OverloadTys, M, FT));
840}
841
842// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
843#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
844#include "llvm/IR/IntrinsicImpl.inc"
845
846// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
847#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
848#include "llvm/IR/IntrinsicImpl.inc"
849
850bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
851 switch (QID) {
852#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
853 case Intrinsic::INTRINSIC:
854#include "llvm/IR/ConstrainedOps.def"
855#undef INSTRUCTION
856 return true;
857 default:
858 return false;
859 }
860}
861
862bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
863 switch (QID) {
864#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
865 case Intrinsic::INTRINSIC: \
866 return ROUND_MODE == 1;
867#include "llvm/IR/ConstrainedOps.def"
868#undef INSTRUCTION
869 default:
870 return false;
871 }
872}
873
874// This class represents a position in the intrinsic's type signature and is
875// used to generate error messages in `matchIntrinsicType`. The printed position
876// can be of the following forms:
877//
878// return
879// return struct element 3
880// return vector element
881// return struct element 3 vector element
882// argument 3
883// argument 3 vector element
884//
885// To support deferred checks also being able to generate these error messages
886// we need to encode the position compactly so that it can be stashed into
887// DeferredIntrinsicMatchInfo below (without materializing it into a string).
888// The class below serves that purpose.
889//
890namespace {
891struct MatchPosition {
892 uint16_t IsRet : 1;
893 uint16_t Num : 15; // Argument number (when IsRet = false).
894 struct Index {
895 uint16_t IsStruct : 1; // If true, this is a struct element with element
896 // index `Num`, else its a vector element.
897 uint16_t Num : 15; // Struct element index.
898 };
899 // We expect this to be just 2 levels deep, since nested structs are not
900 // supported.
901 static constexpr unsigned INDEX_TABLE_SIZE = 2;
902 Index Indices[INDEX_TABLE_SIZE];
903 uint16_t NumIndices = 0;
904
905 void pop_index() {
906 assert(NumIndices > 0 && "cannot pop from empty indices");
907 --NumIndices;
908 }
909
910 void push_struct_element(unsigned ElementNum) {
911 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
912 assert(isInt<15>(ElementNum) && "Element index overflow");
913 Indices[NumIndices].IsStruct = true;
914 Indices[NumIndices++].Num = ElementNum;
915 }
916
917 void push_vector_element() {
918 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
919 Indices[NumIndices].IsStruct = false;
920 Indices[NumIndices++].Num = 0;
921 }
922};
923} // namespace
924
925static raw_ostream &operator<<(raw_ostream &OS, const MatchPosition &Pos) {
926 OS << "intrinsic ";
927
928 if (Pos.IsRet)
929 OS << "return";
930 else
931 OS << "argument " << Pos.Num;
932
933 for (const MatchPosition::Index &Idx :
934 ArrayRef(Pos.Indices).take_front(N: Pos.NumIndices)) {
935 if (Idx.IsStruct)
936 OS << " struct element " << Idx.Num;
937 else
938 OS << " vector element";
939 }
940 return OS;
941}
942
943using DeferredIntrinsicMatchInfo =
944 std::tuple<Type *, ArrayRef<Intrinsic::IITDescriptor>, MatchPosition>;
945
946/// Check whether \p Ty satisfies the given element constraint \p EC and return
947/// a description of the expected element kind for error messages.
948static std::pair<bool, StringRef>
949checkElementConstraint(Type *Ty,
950 Intrinsic::IITDescriptor::AnyKindElementConstraint EC) {
951 using namespace Intrinsic;
952
953 Type *EltTy = Ty->getScalarType();
954 switch (EC) {
955 case IITDescriptor::EC_None:
956 return {true, "value"};
957 case IITDescriptor::EC_Integer:
958 return {EltTy->isIntegerTy(), "integer"};
959 case IITDescriptor::EC_Float:
960 return {EltTy->isFloatingPointTy(), "fp"};
961 case IITDescriptor::EC_Pointer:
962 return {EltTy->isPointerTy(), "pointer"};
963 }
964 llvm_unreachable("invalid element constraint");
965}
966
967static bool
968matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
969 MatchPosition Position, SmallVectorImpl<Type *> &OverloadTys,
970 SmallVectorImpl<DeferredIntrinsicMatchInfo> &DeferredChecks,
971 bool IsDeferredCheck, raw_ostream &OS) {
972 using namespace Intrinsic;
973
974 // If we ran out of descriptors, there are too many arguments or returns.
975 if (Infos.empty()) {
976 OS << Position << " too many "
977 << (Position.IsRet ? "returns" : "arguments");
978 return true;
979 }
980
981 // Do this before slicing off the 'front' part
982 auto InfosRef = Infos;
983 auto DeferCheck = [&DeferredChecks, &InfosRef, &Position](Type *T) {
984 DeferredChecks.emplace_back(Args&: T, Args&: InfosRef, Args&: Position);
985 return false;
986 };
987
988 IITDescriptor D = Infos.consume_front();
989
990 // Print error message when the (non-dependent) type for current position is
991 // invalid.
992 auto PrintMsg = [&OS, &Position,
993 Ty](bool IsValid, const Twine &Expected,
994 std::optional<unsigned> OIdx = std::nullopt) -> bool {
995 if (IsValid)
996 return false;
997 OS << Position << " type";
998 if (OIdx)
999 OS << " (overload type " << *OIdx << ")";
1000 OS << " expected " << Expected << ", but got " << *Ty;
1001 return true;
1002 };
1003
1004 // Print message when an overload type is invalid as a result of its use in
1005 // current dependent type. DependentQualifier describes the "function" applied
1006 // to the overload type to get the dependent type.
1007 auto PrintMsgInvalidOverloadTy =
1008 [&OS, &Position, &OverloadTys](const Twine &DependentQualifier,
1009 const Twine &Expected,
1010 unsigned OIdx) -> bool {
1011 OS << Position << " is " << DependentQualifier << " overload type " << OIdx
1012 << ", so overload type " << OIdx << " expected " << Expected
1013 << ", but got " << *OverloadTys[OIdx];
1014 return true;
1015 };
1016
1017 // Print message when a dependent type is invalid.
1018 auto PrintMsgInvalidDepType =
1019 [&OS, &Position, &OverloadTys,
1020 Ty](bool IsValid, const Twine &DependentQualifier, const Twine &Expected,
1021 unsigned OIdx) -> bool {
1022 if (IsValid)
1023 return false;
1024 bool IsMatching = DependentQualifier.isSingleStringRef() &&
1025 DependentQualifier.getSingleStringRef() == "matching";
1026 OS << Position << " type (" << DependentQualifier << " overload type "
1027 << OIdx << ") expected " << Expected;
1028 if (!IsMatching)
1029 OS << " (overload type " << OIdx << " is " << *OverloadTys[OIdx] << ")";
1030 OS << ", but got " << *Ty;
1031 return true;
1032 };
1033
1034 switch (D.Kind) {
1035 case IITDescriptor::Void:
1036 assert(Position.IsRet && Position.NumIndices == 0 &&
1037 "void descriptor expected only for return type");
1038 return PrintMsg(Ty->isVoidTy(), "void");
1039 case IITDescriptor::MMX: {
1040 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
1041 return PrintMsg(VT && VT->getNumElements() == 1 &&
1042 VT->getElementType()->isIntegerTy(BitWidth: 64),
1043 "x86_mmx (<1 x i64>)");
1044 }
1045 case IITDescriptor::AMX:
1046 return PrintMsg(Ty->isX86_AMXTy(), "x86_amx");
1047 case IITDescriptor::Token:
1048 return PrintMsg(Ty->isTokenTy(), "token");
1049 case IITDescriptor::Metadata:
1050 return PrintMsg(Ty->isMetadataTy(), "metadata");
1051 case IITDescriptor::Half:
1052 return PrintMsg(Ty->isHalfTy(), "half");
1053 case IITDescriptor::BFloat:
1054 return PrintMsg(Ty->isBFloatTy(), "bfloat");
1055 case IITDescriptor::Float:
1056 return PrintMsg(Ty->isFloatTy(), "float");
1057 case IITDescriptor::Double:
1058 return PrintMsg(Ty->isDoubleTy(), "double");
1059 case IITDescriptor::Quad:
1060 return PrintMsg(Ty->isFP128Ty(), "fp128");
1061 case IITDescriptor::PPCQuad:
1062 return PrintMsg(Ty->isPPC_FP128Ty(), "ppc_fp128");
1063 case IITDescriptor::Integer:
1064 return PrintMsg(Ty->isIntegerTy(BitWidth: D.IntegerWidth),
1065 "i" + Twine(D.IntegerWidth));
1066 case IITDescriptor::AArch64Svcount:
1067 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1068 cast<TargetExtType>(Val: Ty)->getName() == "aarch64.svcount",
1069 "aarch64.svcount");
1070 case IITDescriptor::WasmExternref:
1071 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1072 cast<TargetExtType>(Val: Ty)->getName() == "wasm.externref",
1073 "wasm.externref");
1074 case IITDescriptor::WasmFuncref:
1075 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1076 cast<TargetExtType>(Val: Ty)->getName() == "wasm.funcref",
1077 "wasm.funcref");
1078 case IITDescriptor::Vector: {
1079 VectorType *VT = dyn_cast<VectorType>(Val: Ty);
1080 StringRef Scalable = D.VectorWidth.isScalable() ? "vscale " : "";
1081 bool HasError =
1082 PrintMsg(VT && VT->getElementCount() == D.VectorWidth,
1083 Twine(Scalable) + "vector with " +
1084 Twine(D.VectorWidth.getKnownMinValue()) + " elements");
1085 if (HasError)
1086 return true;
1087 Position.push_vector_element();
1088 return matchIntrinsicType(Ty: VT->getElementType(), Infos, Position,
1089 OverloadTys, DeferredChecks, IsDeferredCheck, OS);
1090 }
1091 case IITDescriptor::Pointer: {
1092 PointerType *PT = dyn_cast<PointerType>(Val: Ty);
1093 unsigned AS = D.PointerAddressSpace;
1094 bool IsValid = PT && PT->getAddressSpace() == AS;
1095 if (AS == 0)
1096 return PrintMsg(IsValid, "ptr");
1097 return PrintMsg(IsValid, "ptr addrspace(" + Twine(AS) + ")");
1098 }
1099
1100 case IITDescriptor::Struct: {
1101 StructType *ST = dyn_cast<StructType>(Val: Ty);
1102 unsigned EC = D.StructNumElements;
1103 bool HasError = PrintMsg(
1104 ST && ST->isLiteral() && !ST->isPacked() && ST->getNumElements() == EC,
1105 "literal non-packed struct with " + Twine(EC) + " elements");
1106 if (HasError)
1107 return true;
1108
1109 for (const auto &[Idx, ETy] : llvm::enumerate(First: ST->elements())) {
1110 Position.push_struct_element(ElementNum: Idx);
1111 if (matchIntrinsicType(Ty: ETy, Infos, Position, OverloadTys, DeferredChecks,
1112 IsDeferredCheck, OS))
1113 return true;
1114 Position.pop_index();
1115 }
1116 return false;
1117 }
1118
1119 case IITDescriptor::Overloaded: {
1120 unsigned OIdx = D.getOverloadIndex();
1121 assert(OIdx == OverloadTys.size() && !IsDeferredCheck &&
1122 "Table consistency error");
1123 OverloadTys.push_back(Elt: Ty);
1124
1125 // Token has no mangling (see getMangledTypeStr), so it cannot be an
1126 // overload type; reject it with a signature error. Label is already
1127 // excluded from function signatures, so it never reaches here.
1128 if (Ty->isTokenTy())
1129 return PrintMsg(false, "any manglable type", OIdx);
1130
1131 IITDescriptor::AnyKindVectorConstraint VC;
1132 IITDescriptor::AnyKindElementConstraint EC;
1133 std::tie(args&: VC, args&: EC) = D.getOverloadConstraints();
1134
1135 bool IsValid = [&]() {
1136 switch (VC) {
1137 case IITDescriptor::VC_None:
1138 return true;
1139 case IITDescriptor::VC_Vector:
1140 return isa<VectorType>(Val: Ty);
1141 case IITDescriptor::VC_Scalar:
1142 return !isa<VectorType>(Val: Ty);
1143 }
1144 llvm_unreachable("invalid vector constraint");
1145 }();
1146
1147 auto [ElemValid, EK] = checkElementConstraint(Ty, EC);
1148 IsValid &= ElemValid;
1149
1150 if (IsValid)
1151 return false;
1152
1153 static constexpr StringLiteral VectorKinds[] = {
1154 "",
1155 "vector",
1156 "scalar",
1157 };
1158
1159 if (EC == IITDescriptor::EC_None) {
1160 // No constraint on element type.
1161 // Expected = any {vector | scalar} type.
1162 StringLiteral VK = ArrayRef(VectorKinds)[VC];
1163 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: VK), OIdx);
1164 }
1165
1166 switch (VC) {
1167 case IITDescriptor::VC_None:
1168 // Expected = any EK or EK vector.
1169 return PrintMsg(false, formatv(Fmt: "any {0} or {0} vector", Vals&: EK), OIdx);
1170 case IITDescriptor::VC_Vector:
1171 return PrintMsg(false, formatv(Fmt: "any {} vector", Vals&: EK), OIdx);
1172 case IITDescriptor::VC_Scalar:
1173 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: EK), OIdx);
1174 }
1175 llvm_unreachable("invalid vector constraint");
1176 }
1177
1178 case IITDescriptor::Match: {
1179 unsigned OIdx = D.getOverloadIndex();
1180 if (OIdx >= OverloadTys.size())
1181 return IsDeferredCheck || DeferCheck(Ty);
1182 return PrintMsgInvalidDepType(Ty == OverloadTys[OIdx], "matching",
1183 formatv(Fmt: "{}", Vals&: *OverloadTys[OIdx]), OIdx);
1184 }
1185
1186 case IITDescriptor::Extend:
1187 case IITDescriptor::Trunc: {
1188 unsigned OIdx = D.getOverloadIndex();
1189 // If this is a forward reference, defer the check for later.
1190 if (OIdx >= OverloadTys.size())
1191 return IsDeferredCheck || DeferCheck(Ty);
1192
1193 Type *OTy = OverloadTys[OIdx];
1194 bool IsExtend = D.Kind == IITDescriptor::Extend;
1195 StringRef Qualifier = IsExtend ? "extended" : "truncated";
1196 if (!OTy->isIntOrIntVectorTy())
1197 return PrintMsgInvalidOverloadTy(Qualifier, "int or vector of int", OIdx);
1198
1199 Type *NewTy = IsExtend ? OTy->getExtendedType() : OTy->getTruncatedType();
1200 return PrintMsgInvalidDepType(Ty == NewTy, Qualifier, formatv(Fmt: "{}", Vals&: *NewTy),
1201 OIdx);
1202 }
1203 case IITDescriptor::OneNthEltsVec: {
1204 unsigned OIdx = D.getOverloadIndex();
1205 unsigned Divisor = D.getVectorDivisor();
1206 // If this is a forward reference, defer the check for later.
1207 if (OIdx >= OverloadTys.size())
1208 return IsDeferredCheck || DeferCheck(Ty);
1209 Type *OTy = OverloadTys[OIdx];
1210 auto *OVecTy = dyn_cast<VectorType>(Val: OTy);
1211 auto Qualifier = formatv(Fmt: "1/nth (n={}) elements vector of", Vals&: Divisor);
1212 if (!OVecTy)
1213 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1214 if (!OVecTy->getElementCount().isKnownMultipleOf(RHS: Divisor))
1215 return PrintMsgInvalidOverloadTy(
1216 Qualifier, formatv(Fmt: "vector with multiple of {} elements", Vals&: Divisor),
1217 OIdx);
1218 Type *Expected = VectorType::getOneNthElementsVectorType(VTy: OVecTy, Denominator: Divisor);
1219 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1220 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1221 }
1222 case IITDescriptor::SameVecWidth: {
1223 unsigned OIdx = D.getOverloadIndex();
1224 if (OIdx >= OverloadTys.size()) {
1225 // Defer check and subsequent check for the vector element type.
1226 Infos.consume_front();
1227 return IsDeferredCheck || DeferCheck(Ty);
1228 }
1229 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1230 auto *ThisArgVecType = dyn_cast<VectorType>(Val: Ty);
1231 // Both must be vectors of the same number of elements or neither.
1232 StringRef Qualifier = "same vector width of";
1233 if (OVecTy && !ThisArgVecType)
1234 return PrintMsgInvalidDepType(false, Qualifier, "vector", OIdx);
1235 if (!OVecTy && ThisArgVecType)
1236 return PrintMsgInvalidDepType(false, Qualifier, "scalar", OIdx);
1237 Type *EltTy = Ty;
1238 if (ThisArgVecType) {
1239 ElementCount Expected = OVecTy->getElementCount();
1240 if (Expected != ThisArgVecType->getElementCount())
1241 return PrintMsgInvalidDepType(
1242 false, Qualifier, formatv(Fmt: "vector with {} elements", Vals&: Expected),
1243 OIdx);
1244 EltTy = ThisArgVecType->getElementType();
1245 Position.push_vector_element();
1246 }
1247 return matchIntrinsicType(Ty: EltTy, Infos, Position, OverloadTys,
1248 DeferredChecks, IsDeferredCheck, OS);
1249 }
1250 case IITDescriptor::AnySameVecWidth: {
1251 unsigned RefOverloadIndex = D.getRefOverloadIndex();
1252 if (RefOverloadIndex >= OverloadTys.size()) {
1253 if (IsDeferredCheck)
1254 return true;
1255 // If forward referencing, already add the type and defer the checks for
1256 // later.
1257 assert(D.getOverloadIndex() == OverloadTys.size() &&
1258 "Table consistency error");
1259 OverloadTys.push_back(Elt: Ty);
1260 return DeferCheck(Ty);
1261 }
1262
1263 if (!IsDeferredCheck) {
1264 assert(D.getOverloadIndex() == OverloadTys.size() &&
1265 "Table consistency error");
1266 OverloadTys.push_back(Elt: Ty);
1267 }
1268
1269 auto [ElemValid, Kind] =
1270 checkElementConstraint(Ty, EC: D.getElementConstraint());
1271
1272 auto *ReferenceType = dyn_cast<VectorType>(Val: OverloadTys[RefOverloadIndex]);
1273 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1274 StringRef Qualifier = "same vector width of";
1275 if (ReferenceType) {
1276 auto ExpectedCount = ReferenceType->getElementCount();
1277 auto Expected =
1278 formatv(Fmt: "{0} vector with {1} elements", Vals&: Kind, Vals&: ExpectedCount);
1279 if (!ThisArgVecTy)
1280 return PrintMsgInvalidDepType(false, Qualifier, Expected,
1281 RefOverloadIndex);
1282 bool IsValid =
1283 ThisArgVecTy->getElementCount() == ExpectedCount && ElemValid;
1284 return PrintMsgInvalidDepType(IsValid, Qualifier, Expected,
1285 RefOverloadIndex);
1286 }
1287
1288 auto Expected = formatv(Fmt: "scalar {}", Vals&: Kind);
1289 if (ThisArgVecTy)
1290 return PrintMsgInvalidDepType(false, Qualifier, Expected,
1291 RefOverloadIndex);
1292 return PrintMsgInvalidDepType(ElemValid, Qualifier, Expected,
1293 RefOverloadIndex);
1294 }
1295 case IITDescriptor::VecElement: {
1296 unsigned OIdx = D.getOverloadIndex();
1297 if (OIdx >= OverloadTys.size())
1298 return IsDeferredCheck || DeferCheck(Ty);
1299 StringRef Qualifier = "vector element of";
1300 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1301 if (!OVecTy)
1302 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1303 Type *Expected = OVecTy->getElementType();
1304 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1305 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1306 }
1307 case IITDescriptor::Subdivide2:
1308 case IITDescriptor::Subdivide4: {
1309 unsigned OIdx = D.getOverloadIndex();
1310 // If this is a forward reference, defer the check for later.
1311 if (OIdx >= OverloadTys.size())
1312 return IsDeferredCheck || DeferCheck(Ty);
1313
1314 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
1315 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1316 auto Qualifier =
1317 formatv(Fmt: "subdivided by {} vector of", Vals: SubDivs == 1 ? 2 : 4);
1318 if (!OVecTy)
1319 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1320
1321 // TODO: Verify that the element type of the overload type is subdivisible
1322 // by 2 or 4.
1323 Type *Expected = VectorType::getSubdividedVectorType(VTy: OVecTy, NumSubdivs: SubDivs);
1324 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1325 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1326 }
1327 case IITDescriptor::VecOfBitcastsToInt: {
1328 unsigned OIdx = D.getOverloadIndex();
1329 if (OIdx >= OverloadTys.size())
1330 return IsDeferredCheck || DeferCheck(Ty);
1331 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1332 StringRef Qualifier = "vector of bitcasts to int of";
1333 if (!OVecTy)
1334 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1335 Type *Expected = VectorType::getInteger(VTy: OVecTy);
1336 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1337 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1338 }
1339 case IITDescriptor::VarArg:
1340 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
1341 // should never see it here.
1342 llvm_unreachable("IITDescriptor::VarArg not expected");
1343 }
1344 llvm_unreachable("unhandled");
1345}
1346
1347/// Return true if the function type \p FTy is a valid type signature for the
1348/// type constraints specified in the .td file, represented by \p Infos and
1349/// \p IsVarArg. The overloaded types for the intrinsic are pushed to the
1350/// \p OverloadTys vector.
1351///
1352/// If \p NumMissingTrailingParams is non-zero, \p FTy may omit exactly that
1353/// many trailing parameters. Omitted parameters must have concrete integer
1354/// types and therefore cannot contribute an unresolved overload type.
1355///
1356/// If the type is not valid, returns false and prints an error message to
1357/// \p OS.
1358static bool isSignatureValid(FunctionType *FTy,
1359 ArrayRef<Intrinsic::IITDescriptor> &Infos,
1360 unsigned NumArgs, bool IsVarArg,
1361 SmallVectorImpl<Type *> &OverloadTys,
1362 raw_ostream &OS,
1363 unsigned NumMissingTrailingParams) {
1364 SmallVector<DeferredIntrinsicMatchInfo, 2> DeferredChecks;
1365
1366 assert(!Infos.empty() && "Table consistency error");
1367
1368 MatchPosition Pos;
1369 Pos.IsRet = true;
1370 Pos.Num = 0;
1371
1372 if (matchIntrinsicType(Ty: FTy->getReturnType(), Infos, Position: Pos, OverloadTys,
1373 DeferredChecks, IsDeferredCheck: false, OS))
1374 return false;
1375
1376 unsigned ProvidedArgs = FTy->getNumParams();
1377 if (ProvidedArgs + NumMissingTrailingParams != NumArgs) {
1378 OS << "intrinsic has incorrect number of args. Expected " << NumArgs
1379 << ", but got " << ProvidedArgs;
1380 return false;
1381 }
1382
1383 Pos.IsRet = false;
1384 for (const auto &[Idx, Ty] : llvm::enumerate(First: FTy->params())) {
1385 Pos.Num = Idx;
1386 if (matchIntrinsicType(Ty, Infos, Position: Pos, OverloadTys, DeferredChecks, IsDeferredCheck: false,
1387 OS))
1388 return false;
1389 }
1390
1391 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
1392 auto &[DefTy, DefInfos, DefPosition] = DeferredChecks[I];
1393 if (matchIntrinsicType(Ty: DefTy, Infos&: DefInfos, Position: DefPosition, OverloadTys,
1394 DeferredChecks, IsDeferredCheck: true, OS))
1395 return false;
1396 }
1397
1398 // Default arguments are materialized as ConstantInt values, requiring one
1399 // concrete integer descriptor per omitted parameter.
1400 if (NumMissingTrailingParams != 0 &&
1401 (Infos.size() != NumMissingTrailingParams ||
1402 llvm::any_of(Range&: Infos, P: [](Intrinsic::IITDescriptor D) {
1403 return D.Kind != Intrinsic::IITDescriptor::Integer;
1404 }))) {
1405 OS << "cannot omit trailing parameters that are overloaded or non-integer";
1406 return false;
1407 }
1408 Infos = Infos.drop_front(N: NumMissingTrailingParams);
1409
1410 if (!Infos.empty()) {
1411 OS << "intrinsic has too few arguments!";
1412 return false;
1413 }
1414
1415 if (FTy->isVarArg() != IsVarArg) {
1416 if (IsVarArg)
1417 OS << "intrinsic was not defined with variable arguments!";
1418 else
1419 OS << "intrinsic was defined with variable arguments!";
1420 return false;
1421 }
1422
1423 return true;
1424}
1425
1426bool Intrinsic::hasStructReturnType(ID id) {
1427 using namespace Intrinsic;
1428 SmallVector<IITDescriptor> Table;
1429 getIntrinsicInfoTableEntries(id, T&: Table);
1430 return !Table.empty() && Table[0].Kind == IITDescriptor::Struct;
1431}
1432
1433bool Intrinsic::isSignatureValid(Intrinsic::ID ID, FunctionType *FT,
1434 SmallVectorImpl<Type *> &OverloadTys,
1435 raw_ostream &OS) {
1436 return isSignatureValid(ID, FT, OverloadTys,
1437 /*NumMissingTrailingParams=*/0, OS);
1438}
1439
1440bool Intrinsic::isSignatureValid(Intrinsic::ID ID, FunctionType *FT,
1441 SmallVectorImpl<Type *> &OverloadTys,
1442 unsigned NumMissingTrailingParams,
1443 raw_ostream &OS) {
1444 if (!ID)
1445 return false;
1446
1447 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1448 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id: ID, T&: Table);
1449
1450 return ::isSignatureValid(FTy: FT, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS,
1451 NumMissingTrailingParams);
1452}
1453
1454bool Intrinsic::isSignatureValid(Function *F,
1455 SmallVectorImpl<Type *> &OverloadTys,
1456 raw_ostream &OS) {
1457 return isSignatureValid(ID: F->getIntrinsicID(), FT: F->getFunctionType(),
1458 OverloadTys, OS);
1459}
1460
1461std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
1462 SmallVector<Type *, 4> OverloadTys;
1463 if (!isSignatureValid(F, OverloadTys))
1464 return std::nullopt;
1465
1466 Intrinsic::ID ID = F->getIntrinsicID();
1467 StringRef Name = F->getName();
1468 std::string WantedName =
1469 Intrinsic::getName(Id: ID, OverloadTys, M: F->getParent(), FT: F->getFunctionType());
1470 if (Name == WantedName)
1471 return std::nullopt;
1472
1473 Function *NewDecl = [&] {
1474 if (auto *ExistingGV = F->getParent()->getNamedValue(Name: WantedName)) {
1475 if (auto *ExistingF = dyn_cast<Function>(Val: ExistingGV))
1476 if (ExistingF->getFunctionType() == F->getFunctionType())
1477 return ExistingF;
1478
1479 // The name already exists, but is not a function or has the wrong
1480 // prototype. Make place for the new one by renaming the old version.
1481 // Either this old version will be removed later on or the module is
1482 // invalid and we'll get an error.
1483 ExistingGV->setName(WantedName + ".renamed");
1484 }
1485 return Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys);
1486 }();
1487
1488 NewDecl->setCallingConv(F->getCallingConv());
1489 assert(NewDecl->getFunctionType() == F->getFunctionType() &&
1490 "Shouldn't change the signature");
1491 return NewDecl;
1492}
1493
1494struct InterleaveIntrinsic {
1495 Intrinsic::ID Interleave, Deinterleave;
1496};
1497
1498static InterleaveIntrinsic InterleaveIntrinsics[] = {
1499 {.Interleave: Intrinsic::vector_interleave2, .Deinterleave: Intrinsic::vector_deinterleave2},
1500 {.Interleave: Intrinsic::vector_interleave3, .Deinterleave: Intrinsic::vector_deinterleave3},
1501 {.Interleave: Intrinsic::vector_interleave4, .Deinterleave: Intrinsic::vector_deinterleave4},
1502 {.Interleave: Intrinsic::vector_interleave5, .Deinterleave: Intrinsic::vector_deinterleave5},
1503 {.Interleave: Intrinsic::vector_interleave6, .Deinterleave: Intrinsic::vector_deinterleave6},
1504 {.Interleave: Intrinsic::vector_interleave7, .Deinterleave: Intrinsic::vector_deinterleave7},
1505 {.Interleave: Intrinsic::vector_interleave8, .Deinterleave: Intrinsic::vector_deinterleave8},
1506};
1507
1508Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
1509 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1510 return InterleaveIntrinsics[Factor - 2].Interleave;
1511}
1512
1513Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
1514 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1515 return InterleaveIntrinsics[Factor - 2].Deinterleave;
1516}
1517
1518LLVM_ABI void Intrinsic::printFPClassMask(raw_ostream &OS,
1519 const Constant *ImmArgVal) {
1520 uint64_t Val = cast<ConstantInt>(Val: ImmArgVal)->getZExtValue();
1521 OS << static_cast<FPClassTest>(Val);
1522}
1523
1524#define GET_INTRINSIC_IMMARG_RANGE_SET_CHECKS
1525#include "llvm/IR/IntrinsicImpl.inc"
1526
1527#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1528#include "llvm/IR/IntrinsicImpl.inc"
1529
1530// Emit the default-argument values table and lookup function
1531// (Intrinsic::getAllDefaultArgValues).
1532#define GET_INTRINSIC_DEFAULT_ARG_VALUES
1533#include "llvm/IR/IntrinsicImpl.inc"
1534