1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions required for supporting intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Intrinsics.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringTable.h"
16#include "llvm/IR/ConstantRange.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsPowerPC.h"
26#include "llvm/IR/IntrinsicsR600.h"
27#include "llvm/IR/IntrinsicsRISCV.h"
28#include "llvm/IR/IntrinsicsS390.h"
29#include "llvm/IR/IntrinsicsSPIRV.h"
30#include "llvm/IR/IntrinsicsVE.h"
31#include "llvm/IR/IntrinsicsX86.h"
32#include "llvm/IR/IntrinsicsXCore.h"
33#include "llvm/IR/Module.h"
34#include "llvm/IR/NVVMIntrinsicUtils.h"
35#include "llvm/IR/Type.h"
36#include "llvm/Support/FormatVariadic.h"
37#include "llvm/Support/MathExtras.h"
38
39using namespace llvm;
40
41// Forward declaration of static functions.
42static bool isSignatureValid(FunctionType *FTy,
43 ArrayRef<Intrinsic::IITDescriptor> &Infos,
44 unsigned NumArgs, bool IsVarArg,
45 SmallVectorImpl<Type *> &OverloadTys,
46 raw_ostream &OS);
47
48/// Table of string intrinsic names indexed by enum value.
49#define GET_INTRINSIC_NAME_TABLE
50#include "llvm/IR/IntrinsicImpl.inc"
51
52/// Table of required target features indexed by enum value.
53#define GET_INTRINSIC_TARGET_FEATURES_TABLE
54#include "llvm/IR/IntrinsicImpl.inc"
55
56StringRef Intrinsic::getBaseName(ID id) {
57 assert(id < num_intrinsics && "Invalid intrinsic ID!");
58 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
59}
60
61StringRef Intrinsic::getRequiredTargetFeatures(ID id) {
62 assert(id < num_intrinsics && "invalid intrinsic ID!");
63 return IntrinsicTargetFeaturesTable[IntrinsicTargetFeaturesOffsetTable[id]];
64}
65
66StringRef Intrinsic::getName(ID id) {
67 assert(id < num_intrinsics && "Invalid intrinsic ID!");
68 assert(!Intrinsic::isOverloaded(id) &&
69 "This version of getName does not support overloading");
70 return getBaseName(id);
71}
72
73/// Returns a stable mangling for the type specified for use in the name
74/// mangling scheme used by 'any' types in intrinsic signatures. The mangling
75/// of named types is simply their name. Manglings for unnamed types consist
76/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
77/// combined with the mangling of their component types. A vararg function
78/// type will have a suffix of 'vararg'. Since function types can contain
79/// other function types, we close a function type mangling with suffix 'f'
80/// which can't be confused with it's prefix. This ensures we don't have
81/// collisions between two unrelated function types. Otherwise, you might
82/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)
83/// The HasUnnamedType boolean is set if an unnamed type was encountered,
84/// indicating that extra care must be taken to ensure a unique name.
85static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
86 std::string Result;
87 if (PointerType *PTyp = dyn_cast<PointerType>(Val: Ty)) {
88 Result += "p" + utostr(X: PTyp->getAddressSpace());
89 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Val: Ty)) {
90 Result += "a" + utostr(X: ATyp->getNumElements()) +
91 getMangledTypeStr(Ty: ATyp->getElementType(), HasUnnamedType);
92 } else if (StructType *STyp = dyn_cast<StructType>(Val: Ty)) {
93 if (!STyp->isLiteral()) {
94 Result += "s_";
95 if (STyp->hasName())
96 Result += STyp->getName();
97 else
98 HasUnnamedType = true;
99 } else {
100 Result += "sl_";
101 for (auto *Elem : STyp->elements())
102 Result += getMangledTypeStr(Ty: Elem, HasUnnamedType);
103 }
104 // Ensure nested structs are distinguishable.
105 Result += "s";
106 } else if (FunctionType *FT = dyn_cast<FunctionType>(Val: Ty)) {
107 Result += "f_" + getMangledTypeStr(Ty: FT->getReturnType(), HasUnnamedType);
108 for (size_t i = 0; i < FT->getNumParams(); i++)
109 Result += getMangledTypeStr(Ty: FT->getParamType(i), HasUnnamedType);
110 if (FT->isVarArg())
111 Result += "vararg";
112 // Ensure nested function types are distinguishable.
113 Result += "f";
114 } else if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
115 ElementCount EC = VTy->getElementCount();
116 if (EC.isScalable())
117 Result += "nx";
118 Result += "v" + utostr(X: EC.getKnownMinValue()) +
119 getMangledTypeStr(Ty: VTy->getElementType(), HasUnnamedType);
120 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Val: Ty)) {
121 Result += "t";
122 Result += TETy->getName();
123 for (Type *ParamTy : TETy->type_params())
124 Result += "_" + getMangledTypeStr(Ty: ParamTy, HasUnnamedType);
125 for (unsigned IntParam : TETy->int_params())
126 Result += "_" + utostr(X: IntParam);
127 // Ensure nested target extension types are distinguishable.
128 Result += "t";
129 } else if (Ty) {
130 switch (Ty->getTypeID()) {
131 default:
132 llvm_unreachable("Unhandled type");
133 case Type::VoidTyID:
134 Result += "isVoid";
135 break;
136 case Type::MetadataTyID:
137 Result += "Metadata";
138 break;
139 case Type::HalfTyID:
140 Result += "f16";
141 break;
142 case Type::BFloatTyID:
143 Result += "bf16";
144 break;
145 case Type::FloatTyID:
146 Result += "f32";
147 break;
148 case Type::DoubleTyID:
149 Result += "f64";
150 break;
151 case Type::X86_FP80TyID:
152 Result += "f80";
153 break;
154 case Type::FP128TyID:
155 Result += "f128";
156 break;
157 case Type::PPC_FP128TyID:
158 Result += "ppcf128";
159 break;
160 case Type::X86_AMXTyID:
161 Result += "x86amx";
162 break;
163 case Type::IntegerTyID:
164 Result += "i" + utostr(X: cast<IntegerType>(Val: Ty)->getBitWidth());
165 break;
166 case Type::ByteTyID:
167 Result += "b" + utostr(X: cast<ByteType>(Val: Ty)->getBitWidth());
168 break;
169 }
170 }
171 return Result;
172}
173
174static std::string getIntrinsicNameImpl(Intrinsic::ID Id,
175 ArrayRef<Type *> OverloadTys, Module *M,
176 FunctionType *FT,
177 bool EarlyModuleCheck) {
178
179 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
180 assert((OverloadTys.empty() || Intrinsic::isOverloaded(Id)) &&
181 "This version of getName is for overloaded intrinsics only");
182 (void)EarlyModuleCheck;
183 assert((!EarlyModuleCheck || M ||
184 !any_of(OverloadTys, llvm::IsaPred<PointerType>)) &&
185 "Intrinsic overloading on pointer types need to provide a Module");
186 bool HasUnnamedType = false;
187 std::string Result(Intrinsic::getBaseName(id: Id));
188 for (Type *Ty : OverloadTys)
189 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
190 if (HasUnnamedType) {
191 assert(M && "unnamed types need a module");
192 if (!FT)
193 FT = Intrinsic::getType(Context&: M->getContext(), id: Id, OverloadTys);
194 else
195 assert(FT == Intrinsic::getType(M->getContext(), Id, OverloadTys) &&
196 "Provided FunctionType must match arguments");
197 return M->getUniqueIntrinsicName(BaseName: Result, Id, Proto: FT);
198 }
199 return Result;
200}
201
202std::string Intrinsic::getName(ID Id, ArrayRef<Type *> OverloadTys, Module *M,
203 FunctionType *FT) {
204 assert(M && "We need to have a Module");
205 return getIntrinsicNameImpl(Id, OverloadTys, M, FT, EarlyModuleCheck: true);
206}
207
208std::string Intrinsic::getNameNoUnnamedTypes(ID Id,
209 ArrayRef<Type *> OverloadTys) {
210 return getIntrinsicNameImpl(Id, OverloadTys, M: nullptr, FT: nullptr, EarlyModuleCheck: false);
211}
212
213/// IIT_Info - These are enumerators that describe the entries returned by the
214/// getIntrinsicInfoTableEntries function.
215///
216/// Defined in Intrinsics.td.
217enum IIT_Info {
218#define GET_INTRINSIC_IITINFO
219#include "llvm/IR/IntrinsicImpl.inc"
220};
221
222static_assert(IIT_Done == 0, "IIT_Done expected to be 0");
223
224static void
225DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
226 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
227 using namespace Intrinsic;
228
229 auto IsScalableVector = [&]() {
230 IIT_Info NextInfo = IIT_Info(Infos[NextElt]);
231 if (NextInfo != IIT_SCALABLE_VEC)
232 return false;
233 // Eat the IIT_SCALABLE_VEC token.
234 ++NextElt;
235 return true;
236 };
237
238 IIT_Info Info = IIT_Info(Infos[NextElt++]);
239
240 switch (Info) {
241 case IIT_Done:
242 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Void, Field: 0));
243 return;
244 case IIT_VARARG:
245 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VarArg, Field: 0));
246 return;
247 case IIT_MMX:
248 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::MMX, Field: 0));
249 return;
250 case IIT_AMX:
251 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AMX, Field: 0));
252 return;
253 case IIT_TOKEN:
254 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Token, Field: 0));
255 return;
256 case IIT_METADATA:
257 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Metadata, Field: 0));
258 return;
259 case IIT_F16:
260 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Half, Field: 0));
261 return;
262 case IIT_BF16:
263 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::BFloat, Field: 0));
264 return;
265 case IIT_F32:
266 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Float, Field: 0));
267 return;
268 case IIT_F64:
269 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Double, Field: 0));
270 return;
271 case IIT_F128:
272 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Quad, Field: 0));
273 return;
274 case IIT_PPCF128:
275 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::PPCQuad, Field: 0));
276 return;
277 case IIT_I1:
278 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 1));
279 return;
280 case IIT_I2:
281 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 2));
282 return;
283 case IIT_I4:
284 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 4));
285 return;
286 case IIT_AARCH64_SVCOUNT:
287 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AArch64Svcount, Field: 0));
288 return;
289 case IIT_I8:
290 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 8));
291 return;
292 case IIT_I16:
293 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 16));
294 return;
295 case IIT_I32:
296 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 32));
297 return;
298 case IIT_I64:
299 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 64));
300 return;
301 case IIT_I128:
302 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 128));
303 return;
304 case IIT_V1:
305 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1, IsScalable: IsScalableVector()));
306 DecodeIITType(NextElt, Infos, OutputTable);
307 return;
308 case IIT_V2:
309 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2, IsScalable: IsScalableVector()));
310 DecodeIITType(NextElt, Infos, OutputTable);
311 return;
312 case IIT_V3:
313 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 3, IsScalable: IsScalableVector()));
314 DecodeIITType(NextElt, Infos, OutputTable);
315 return;
316 case IIT_V4:
317 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4, IsScalable: IsScalableVector()));
318 DecodeIITType(NextElt, Infos, OutputTable);
319 return;
320 case IIT_V6:
321 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 6, IsScalable: IsScalableVector()));
322 DecodeIITType(NextElt, Infos, OutputTable);
323 return;
324 case IIT_V8:
325 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 8, IsScalable: IsScalableVector()));
326 DecodeIITType(NextElt, Infos, OutputTable);
327 return;
328 case IIT_V10:
329 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 10, IsScalable: IsScalableVector()));
330 DecodeIITType(NextElt, Infos, OutputTable);
331 return;
332 case IIT_V16:
333 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 16, IsScalable: IsScalableVector()));
334 DecodeIITType(NextElt, Infos, OutputTable);
335 return;
336 case IIT_V32:
337 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 32, IsScalable: IsScalableVector()));
338 DecodeIITType(NextElt, Infos, OutputTable);
339 return;
340 case IIT_V64:
341 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 64, IsScalable: IsScalableVector()));
342 DecodeIITType(NextElt, Infos, OutputTable);
343 return;
344 case IIT_V128:
345 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 128, IsScalable: IsScalableVector()));
346 DecodeIITType(NextElt, Infos, OutputTable);
347 return;
348 case IIT_V256:
349 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 256, IsScalable: IsScalableVector()));
350 DecodeIITType(NextElt, Infos, OutputTable);
351 return;
352 case IIT_V512:
353 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 512, IsScalable: IsScalableVector()));
354 DecodeIITType(NextElt, Infos, OutputTable);
355 return;
356 case IIT_V1024:
357 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1024, IsScalable: IsScalableVector()));
358 DecodeIITType(NextElt, Infos, OutputTable);
359 return;
360 case IIT_V2048:
361 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2048, IsScalable: IsScalableVector()));
362 DecodeIITType(NextElt, Infos, OutputTable);
363 return;
364 case IIT_V4096:
365 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4096, IsScalable: IsScalableVector()));
366 DecodeIITType(NextElt, Infos, OutputTable);
367 return;
368 case IIT_EXTERNREF:
369 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmExternref, Field: 0));
370 return;
371 case IIT_FUNCREF:
372 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmFuncref, Field: 0));
373 return;
374 case IIT_PTR:
375 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 0));
376 return;
377 case IIT_PTR_AS: // pointer with address space.
378 OutputTable.push_back(
379 Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: Infos[NextElt++]));
380 return;
381 case IIT_ANY: {
382 unsigned OverloadIndex = Infos[NextElt++];
383 unsigned ArgKindEnums = Infos[NextElt++];
384 unsigned Packed = (ArgKindEnums << 8) | OverloadIndex;
385 OutputTable.push_back(
386 Elt: IITDescriptor::get(K: IITDescriptor::Overloaded, Field: Packed));
387 return;
388 }
389 case IIT_MATCH: {
390 unsigned OverloadIndex = Infos[NextElt++];
391 OutputTable.push_back(
392 Elt: IITDescriptor::get(K: IITDescriptor::Match, Field: OverloadIndex));
393 return;
394 }
395 case IIT_EXTEND_ARG: {
396 unsigned OverloadIndex = Infos[NextElt++];
397 OutputTable.push_back(
398 Elt: IITDescriptor::get(K: IITDescriptor::Extend, Field: OverloadIndex));
399 return;
400 }
401 case IIT_TRUNC_ARG: {
402 unsigned OverloadIndex = Infos[NextElt++];
403 OutputTable.push_back(
404 Elt: IITDescriptor::get(K: IITDescriptor::Trunc, Field: OverloadIndex));
405 return;
406 }
407 case IIT_ONE_NTH_ELTS_VEC_ARG: {
408 unsigned short OverloadIndex = Infos[NextElt++];
409 unsigned short N = Infos[NextElt++];
410 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::OneNthEltsVec,
411 /*Hi=*/N, /*Lo=*/OverloadIndex));
412 return;
413 }
414 case IIT_SAME_VEC_WIDTH_ARG: {
415 unsigned OverloadIndex = Infos[NextElt++];
416 OutputTable.push_back(
417 Elt: IITDescriptor::get(K: IITDescriptor::SameVecWidth, Field: OverloadIndex));
418 // IIT_SAME_VEC_WIDTH_ARG entry is followed by the element type.
419 DecodeIITType(NextElt, Infos, OutputTable);
420 return;
421 }
422 case IIT_VEC_OF_ANYPTRS_TO_ELT: {
423 unsigned short OverloadIndex = Infos[NextElt++];
424 unsigned short RefOverloadIndex = Infos[NextElt++];
425 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VecOfAnyPtrsToElt,
426 /*Hi=*/RefOverloadIndex,
427 /*Lo=*/OverloadIndex));
428 return;
429 }
430 case IIT_STRUCT: {
431 unsigned StructElts = Infos[NextElt++] + 2;
432
433 OutputTable.push_back(
434 Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: StructElts));
435
436 for (unsigned i = 0; i != StructElts; ++i)
437 DecodeIITType(NextElt, Infos, OutputTable);
438 return;
439 }
440 case IIT_SUBDIVIDE2_ARG: {
441 unsigned OverloadIndex = Infos[NextElt++];
442 OutputTable.push_back(
443 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide2, Field: OverloadIndex));
444 return;
445 }
446 case IIT_SUBDIVIDE4_ARG: {
447 unsigned OverloadIndex = Infos[NextElt++];
448 OutputTable.push_back(
449 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide4, Field: OverloadIndex));
450 return;
451 }
452 case IIT_VEC_ELEMENT: {
453 unsigned OverloadIndex = Infos[NextElt++];
454 OutputTable.push_back(
455 Elt: IITDescriptor::get(K: IITDescriptor::VecElement, Field: OverloadIndex));
456 return;
457 }
458 case IIT_VEC_OF_BITCASTS_TO_INT: {
459 unsigned OverloadIndex = Infos[NextElt++];
460 OutputTable.push_back(
461 Elt: IITDescriptor::get(K: IITDescriptor::VecOfBitcastsToInt, Field: OverloadIndex));
462 return;
463 }
464 case IIT_SCALABLE_VEC:
465 break;
466 }
467 llvm_unreachable("unhandled");
468}
469
470#define GET_INTRINSIC_GENERATOR_GLOBAL
471#include "llvm/IR/IntrinsicImpl.inc"
472
473std::tuple<ArrayRef<Intrinsic::IITDescriptor>, unsigned, bool>
474Intrinsic::getIntrinsicInfoTableEntries(ID id,
475 SmallVectorImpl<IITDescriptor> &T) {
476 // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
477 // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
478 // IntrinsicEmitter.cpp.
479 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
480 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
481 // Mask with all bits 1 except the most significant bit.
482 constexpr unsigned Mask = (1U << MSBPosition) - 1;
483
484 FixedEncodingTy TableVal = IIT_Table[id - 1];
485
486 // Array to hold the inlined fixed encoding values expanded from nibbles to
487 // bytes. Its size can be be atmost FixedEncodingBits / 4 i.e., number
488 // of nibbles that can fit in `FixedEncodingTy` + 1 (the IIT_Done terminator
489 // that is not explicitly encoded). Note that if there are trailing 0 bytes
490 // in the encoding (for example, payload following one of the IIT tokens),
491 // the inlined encoding does not encode the actual size of the encoding, so
492 // we always assume its size of this maximum length possible, followed by the
493 // IIT_Done terminator token (whose value is 0).
494 unsigned char IITValues[FixedEncodingBits / 4 + 1] = {0};
495
496 ArrayRef<unsigned char> IITEntries;
497 unsigned NextElt = 0;
498 // Check to see if the intrinsic's type was inlined in the fixed encoding
499 // table.
500 if (TableVal >> MSBPosition) {
501 // This is an offset into the IIT_LongEncodingTable.
502 IITEntries = IIT_LongEncodingTable;
503
504 // Strip sentinel bit.
505 NextElt = TableVal & Mask;
506 } else {
507 // If the entry was encoded into a single word in the table itself, decode
508 // it from an array of nibbles to an array of bytes.
509 do {
510 IITValues[NextElt++] = TableVal & 0xF;
511 TableVal >>= 4;
512 } while (TableVal);
513
514 IITEntries = IITValues;
515 NextElt = 0;
516 }
517
518 // Okay, decode the table into the output vector of IITDescriptors.
519 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
520 unsigned NumArgs = 0;
521 while (IITEntries[NextElt] != IIT_Done) {
522 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
523 ++NumArgs;
524 }
525
526 ArrayRef<IITDescriptor> TableRef = T;
527
528 bool IsVarArg = false;
529 if (TableRef.back().Kind == Intrinsic::IITDescriptor::VarArg) {
530 IsVarArg = true;
531 TableRef.consume_back();
532 --NumArgs;
533 }
534 return {TableRef, NumArgs, IsVarArg};
535}
536
537static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
538 ArrayRef<Type *> OverloadTys,
539 LLVMContext &Context) {
540 using namespace Intrinsic;
541
542 IITDescriptor D = Infos.consume_front();
543
544 switch (D.Kind) {
545 case IITDescriptor::Void:
546 return Type::getVoidTy(C&: Context);
547 case IITDescriptor::MMX:
548 return llvm::FixedVectorType::get(ElementType: llvm::IntegerType::get(C&: Context, NumBits: 64), NumElts: 1);
549 case IITDescriptor::AMX:
550 return Type::getX86_AMXTy(C&: Context);
551 case IITDescriptor::Token:
552 return Type::getTokenTy(C&: Context);
553 case IITDescriptor::Metadata:
554 return Type::getMetadataTy(C&: Context);
555 case IITDescriptor::Half:
556 return Type::getHalfTy(C&: Context);
557 case IITDescriptor::BFloat:
558 return Type::getBFloatTy(C&: Context);
559 case IITDescriptor::Float:
560 return Type::getFloatTy(C&: Context);
561 case IITDescriptor::Double:
562 return Type::getDoubleTy(C&: Context);
563 case IITDescriptor::Quad:
564 return Type::getFP128Ty(C&: Context);
565 case IITDescriptor::PPCQuad:
566 return Type::getPPC_FP128Ty(C&: Context);
567 case IITDescriptor::AArch64Svcount:
568 return TargetExtType::get(Context, Name: "aarch64.svcount");
569 case IITDescriptor::WasmExternref:
570 return TargetExtType::get(Context, Name: "wasm.externref");
571 case IITDescriptor::WasmFuncref:
572 return TargetExtType::get(Context, Name: "wasm.funcref");
573 case IITDescriptor::Integer:
574 return IntegerType::get(C&: Context, NumBits: D.IntegerWidth);
575 case IITDescriptor::Vector:
576 return VectorType::get(ElementType: DecodeFixedType(Infos, OverloadTys, Context),
577 EC: D.VectorWidth);
578 case IITDescriptor::Pointer:
579 return PointerType::get(C&: Context, AddressSpace: D.PointerAddressSpace);
580 case IITDescriptor::Struct: {
581 SmallVector<Type *, 8> Elts;
582 for (unsigned i = 0, e = D.StructNumElements; i != e; ++i)
583 Elts.push_back(Elt: DecodeFixedType(Infos, OverloadTys, Context));
584 return StructType::get(Context, Elements: Elts);
585 }
586 // For any overload type or partially dependent type, substitute it with the
587 // corresponding concrete type from OverloadTys. Additionally, do the same
588 // for the fully dependent type that matches an overload type.
589 case IITDescriptor::Overloaded:
590 case IITDescriptor::VecOfAnyPtrsToElt:
591 case IITDescriptor::Match:
592 return OverloadTys[D.getOverloadIndex()];
593 case IITDescriptor::Extend:
594 return OverloadTys[D.getOverloadIndex()]->getExtendedType();
595 case IITDescriptor::Trunc:
596 return OverloadTys[D.getOverloadIndex()]->getTruncatedType();
597 case IITDescriptor::Subdivide2:
598 case IITDescriptor::Subdivide4: {
599 Type *Ty = OverloadTys[D.getOverloadIndex()];
600 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
601 assert(VTy && "Expected overload type to be a Vector Type");
602 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
603 return VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
604 }
605 case IITDescriptor::OneNthEltsVec:
606 return VectorType::getOneNthElementsVectorType(
607 VTy: cast<VectorType>(Val: OverloadTys[D.getOverloadIndex()]),
608 Denominator: D.getVectorDivisor());
609 case IITDescriptor::SameVecWidth: {
610 Type *EltTy = DecodeFixedType(Infos, OverloadTys, Context);
611 Type *Ty = OverloadTys[D.getOverloadIndex()];
612 if (auto *VTy = dyn_cast<VectorType>(Val: Ty))
613 return VectorType::get(ElementType: EltTy, EC: VTy->getElementCount());
614 return EltTy;
615 }
616 case IITDescriptor::VecElement: {
617 Type *Ty = OverloadTys[D.getOverloadIndex()];
618 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
619 return VTy->getElementType();
620 llvm_unreachable("Expected overload type to be a Vector Type");
621 }
622 case IITDescriptor::VecOfBitcastsToInt: {
623 Type *Ty = OverloadTys[D.getOverloadIndex()];
624 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
625 assert(VTy && "Expected overload type to be a Vector Type");
626 return VectorType::getInteger(VTy);
627 }
628 case IITDescriptor::VarArg:
629 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
630 // should never see it here.
631 llvm_unreachable("IITDescriptor::VarArg not expected");
632 }
633 llvm_unreachable("unhandled");
634}
635
636FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
637 ArrayRef<Type *> OverloadTys) {
638 SmallVector<IITDescriptor, 8> Table;
639 auto [TableRef, _, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
640
641 Type *ResultTy = DecodeFixedType(Infos&: TableRef, OverloadTys, Context);
642
643 SmallVector<Type *, 8> ArgTys;
644 while (!TableRef.empty())
645 ArgTys.push_back(Elt: DecodeFixedType(Infos&: TableRef, OverloadTys, Context));
646 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: IsVarArg);
647}
648
649bool Intrinsic::isOverloaded(ID id) {
650#define GET_INTRINSIC_OVERLOAD_TABLE
651#include "llvm/IR/IntrinsicImpl.inc"
652}
653
654bool Intrinsic::isTriviallyScalarizable(ID id) {
655#define GET_INTRINSIC_SCALARIZABLE_TABLE
656#include "llvm/IR/IntrinsicImpl.inc"
657}
658
659bool Intrinsic::hasPrettyPrintedArgs(ID id){
660#define GET_INTRINSIC_PRETTY_PRINT_TABLE
661#include "llvm/IR/IntrinsicImpl.inc"
662}
663
664/// Table of per-target intrinsic name tables.
665#define GET_INTRINSIC_TARGET_DATA
666#include "llvm/IR/IntrinsicImpl.inc"
667
668bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
669 return IID > TargetInfos[0].Count;
670}
671
672/// Looks up Name in NameTable via binary search. NameTable must be sorted
673/// and all entries must start with "llvm.". If NameTable contains an exact
674/// match for Name or a prefix of Name followed by a dot, its index in
675/// NameTable is returned. Otherwise, -1 is returned.
676static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
677 StringRef Name, StringRef Target = "") {
678 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
679 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
680
681 // Do successive binary searches of the dotted name components. For
682 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
683 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
684 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
685 // size 1. During the search, we can skip the prefix that we already know is
686 // identical. By using strncmp we consider names with differing suffixes to
687 // be part of the equal range.
688 size_t CmpEnd = 4; // Skip the "llvm" component.
689 if (!Target.empty())
690 CmpEnd += 1 + Target.size(); // skip the .target component.
691
692 const unsigned *Low = NameOffsetTable.begin();
693 const unsigned *High = NameOffsetTable.end();
694 const unsigned *LastLow = Low;
695 while (CmpEnd < Name.size() && High - Low > 0) {
696 size_t CmpStart = CmpEnd;
697 CmpEnd = Name.find(C: '.', From: CmpStart + 1);
698 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
699 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
700 // `equal_range` requires the comparison to work with either side being an
701 // offset or the value. Detect which kind each side is to set up the
702 // compared strings.
703 const char *LHSStr;
704 if constexpr (std::is_integral_v<decltype(LHS)>)
705 LHSStr = IntrinsicNameTable.getCString(O: LHS);
706 else
707 LHSStr = LHS;
708
709 const char *RHSStr;
710 if constexpr (std::is_integral_v<decltype(RHS)>)
711 RHSStr = IntrinsicNameTable.getCString(O: RHS);
712 else
713 RHSStr = RHS;
714
715 return strncmp(s1: LHSStr + CmpStart, s2: RHSStr + CmpStart, n: CmpEnd - CmpStart) <
716 0;
717 };
718 LastLow = Low;
719 std::tie(args&: Low, args&: High) = std::equal_range(first: Low, last: High, val: Name.data(), comp: Cmp);
720 }
721 if (High - Low > 0)
722 LastLow = Low;
723
724 if (LastLow == NameOffsetTable.end())
725 return -1;
726 StringRef NameFound = IntrinsicNameTable[*LastLow];
727 if (Name == NameFound ||
728 (Name.starts_with(Prefix: NameFound) && Name[NameFound.size()] == '.'))
729 return LastLow - NameOffsetTable.begin();
730 return -1;
731}
732
733/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
734/// target as \c Name, or the generic table if \c Name is not target specific.
735///
736/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
737/// name.
738static std::pair<ArrayRef<unsigned>, StringRef>
739findTargetSubtable(StringRef Name) {
740 assert(Name.starts_with("llvm."));
741
742 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
743 // Drop "llvm." and take the first dotted component. That will be the target
744 // if this is target specific.
745 StringRef Target = Name.drop_front(N: 5).split(Separator: '.').first;
746 auto It = partition_point(
747 Range&: Targets, P: [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
748 // We've either found the target or just fall back to the generic set, which
749 // is always first.
750 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
751 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
752 TI.Name};
753}
754
755/// This does the actual lookup of an intrinsic ID which matches the given
756/// function name.
757Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
758 auto [NameOffsetTable, Target] = findTargetSubtable(Name);
759 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
760 if (Idx == -1)
761 return Intrinsic::not_intrinsic;
762
763 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
764 // an index into a sub-table.
765 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
766 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
767
768 // If the intrinsic is not overloaded, require an exact match. If it is
769 // overloaded, require either exact or prefix match.
770 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
771 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
772 bool IsExactMatch = Name.size() == MatchSize;
773 return IsExactMatch || Intrinsic::isOverloaded(id: ID) ? ID
774 : Intrinsic::not_intrinsic;
775}
776
777/// This defines the "Intrinsic::getAttributes(ID id)" method.
778#define GET_INTRINSIC_ATTRIBUTES
779#include "llvm/IR/IntrinsicImpl.inc"
780
781static Function *
782getOrInsertIntrinsicDeclarationImpl(Module *M, Intrinsic::ID id,
783 ArrayRef<Type *> OverloadTys,
784 FunctionType *FT) {
785 std::string Name = OverloadTys.empty()
786 ? Intrinsic::getName(id).str()
787 : Intrinsic::getName(Id: id, OverloadTys, M, FT);
788 Function *F = cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
789 if (F->getFunctionType() == FT)
790 return F;
791
792 // It's possible that a declaration for this intrinsic already exists with an
793 // incorrect signature, if the signature has changed, but this particular
794 // declaration has not been auto-upgraded yet. In that case, rename the
795 // invalid declaration and insert a new one with the correct signature. The
796 // invalid declaration will get upgraded later.
797 F->setName(F->getName() + ".invalid");
798 return cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
799}
800
801Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
802 ArrayRef<Type *> OverloadTys) {
803 // There can never be multiple globals with the same name of different types,
804 // because intrinsics must be a specific type.
805 FunctionType *FT = getType(Context&: M->getContext(), id, OverloadTys);
806 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT);
807}
808
809Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
810 ArrayRef<Type *> ArgTys) {
811 // If the intrinsic is not overloaded, use the non-overloaded version.
812 if (!Intrinsic::isOverloaded(id))
813 return getOrInsertDeclaration(M, id);
814
815 // Get the intrinsic signature metadata.
816 SmallVector<Intrinsic::IITDescriptor, 8> Table;
817 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
818 FunctionType *FTy = FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: IsVarArg);
819
820 // Automatically determine the overloaded types.
821 SmallVector<Type *, 4> OverloadTys;
822 [[maybe_unused]] bool IsValid = ::isSignatureValid(
823 FTy, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS&: nulls());
824 assert(IsValid && "intrinsic signature mismatch");
825 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT: FTy);
826}
827
828Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
829 return M->getFunction(Name: getName(id));
830}
831
832Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
833 ArrayRef<Type *> OverloadTys,
834 FunctionType *FT) {
835 return M->getFunction(Name: getName(Id: id, OverloadTys, M, FT));
836}
837
838// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
839#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
840#include "llvm/IR/IntrinsicImpl.inc"
841
842// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
843#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
844#include "llvm/IR/IntrinsicImpl.inc"
845
846bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
847 switch (QID) {
848#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
849 case Intrinsic::INTRINSIC:
850#include "llvm/IR/ConstrainedOps.def"
851#undef INSTRUCTION
852 return true;
853 default:
854 return false;
855 }
856}
857
858bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
859 switch (QID) {
860#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
861 case Intrinsic::INTRINSIC: \
862 return ROUND_MODE == 1;
863#include "llvm/IR/ConstrainedOps.def"
864#undef INSTRUCTION
865 default:
866 return false;
867 }
868}
869
870// This class represents a position in the intrinsic's type signature and is
871// used to generate error messages in `matchIntrinsicType`. The printed position
872// can be of the following forms:
873//
874// return
875// return struct element 3
876// return vector element
877// return struct element 3 vector element
878// argument 3
879// argument 3 vector element
880//
881// To support deferred checks also being able to generate these error messages
882// we need to encode the position compactly so that it can be stashed into
883// DeferredIntrinsicMatchInfo below (without materializing it into a string).
884// The class below serves that purpose.
885//
886namespace {
887struct MatchPosition {
888 uint16_t IsRet : 1;
889 uint16_t Num : 15; // Argument number (when IsRet = false).
890 struct Index {
891 uint16_t IsStruct : 1; // If true, this is a struct element with element
892 // index `Num`, else its a vector element.
893 uint16_t Num : 15; // Struct element index.
894 };
895 // We expect this to be just 2 levels deep, since nested structs are not
896 // supported.
897 static constexpr unsigned INDEX_TABLE_SIZE = 2;
898 Index Indices[INDEX_TABLE_SIZE];
899 uint16_t NumIndices = 0;
900
901 void pop_index() {
902 assert(NumIndices > 0 && "cannot pop from empty indices");
903 --NumIndices;
904 }
905
906 void push_struct_element(unsigned ElementNum) {
907 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
908 assert(isInt<15>(ElementNum) && "Element index overflow");
909 Indices[NumIndices].IsStruct = true;
910 Indices[NumIndices++].Num = ElementNum;
911 }
912
913 void push_vector_element() {
914 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
915 Indices[NumIndices].IsStruct = false;
916 Indices[NumIndices++].Num = 0;
917 }
918};
919} // namespace
920
921static raw_ostream &operator<<(raw_ostream &OS, const MatchPosition &Pos) {
922 OS << "intrinsic ";
923
924 if (Pos.IsRet)
925 OS << "return";
926 else
927 OS << "argument " << Pos.Num;
928
929 for (const MatchPosition::Index &Idx :
930 ArrayRef(Pos.Indices).take_front(N: Pos.NumIndices)) {
931 if (Idx.IsStruct)
932 OS << " struct element " << Idx.Num;
933 else
934 OS << " vector element";
935 }
936 return OS;
937}
938
939using DeferredIntrinsicMatchInfo =
940 std::tuple<Type *, ArrayRef<Intrinsic::IITDescriptor>, MatchPosition>;
941
942static bool
943matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
944 MatchPosition Position, SmallVectorImpl<Type *> &OverloadTys,
945 SmallVectorImpl<DeferredIntrinsicMatchInfo> &DeferredChecks,
946 bool IsDeferredCheck, raw_ostream &OS) {
947 using namespace Intrinsic;
948
949 // If we ran out of descriptors, there are too many arguments or returns.
950 if (Infos.empty()) {
951 OS << Position << " too many "
952 << (Position.IsRet ? "returns" : "arguments");
953 return true;
954 }
955
956 // Do this before slicing off the 'front' part
957 auto InfosRef = Infos;
958 auto DeferCheck = [&DeferredChecks, &InfosRef, &Position](Type *T) {
959 DeferredChecks.emplace_back(Args&: T, Args&: InfosRef, Args&: Position);
960 return false;
961 };
962
963 IITDescriptor D = Infos.consume_front();
964
965 // Print error message when the (non-dependent) type for current position is
966 // invalid.
967 auto PrintMsg = [&OS, &Position,
968 Ty](bool IsValid, const Twine &Expected,
969 std::optional<unsigned> OIdx = std::nullopt) -> bool {
970 if (IsValid)
971 return false;
972 OS << Position << " type";
973 if (OIdx)
974 OS << " (overload type " << *OIdx << ")";
975 OS << " expected " << Expected << ", but got " << *Ty;
976 return true;
977 };
978
979 // Print message when an overload type is invalid as a result of its use in
980 // current dependent type. DependentQualifier describes the "function" applied
981 // to the overload type to get the dependent type.
982 auto PrintMsgInvalidOverloadTy =
983 [&OS, &Position, &OverloadTys](const Twine &DependentQualifier,
984 const Twine &Expected,
985 unsigned OIdx) -> bool {
986 OS << Position << " is " << DependentQualifier << " overload type " << OIdx
987 << ", so overload type " << OIdx << " expected " << Expected
988 << ", but got " << *OverloadTys[OIdx];
989 return true;
990 };
991
992 // Print message when a dependent type is invalid.
993 auto PrintMsgInvalidDepType =
994 [&OS, &Position, &OverloadTys,
995 Ty](bool IsValid, const Twine &DependentQualifier, const Twine &Expected,
996 unsigned OIdx) -> bool {
997 if (IsValid)
998 return false;
999 bool IsMatching = DependentQualifier.isSingleStringRef() &&
1000 DependentQualifier.getSingleStringRef() == "matching";
1001 OS << Position << " type (" << DependentQualifier << " overload type "
1002 << OIdx << ") expected " << Expected;
1003 if (!IsMatching)
1004 OS << " (overload type " << OIdx << " is " << *OverloadTys[OIdx] << ")";
1005 OS << ", but got " << *Ty;
1006 return true;
1007 };
1008
1009 switch (D.Kind) {
1010 case IITDescriptor::Void:
1011 assert(Position.IsRet && Position.NumIndices == 0 &&
1012 "void descriptor expected only for return type");
1013 return PrintMsg(Ty->isVoidTy(), "void");
1014 case IITDescriptor::MMX: {
1015 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
1016 return PrintMsg(VT && VT->getNumElements() == 1 &&
1017 VT->getElementType()->isIntegerTy(BitWidth: 64),
1018 "x86_mmx (<1 x i64>)");
1019 }
1020 case IITDescriptor::AMX:
1021 return PrintMsg(Ty->isX86_AMXTy(), "x86_amx");
1022 case IITDescriptor::Token:
1023 return PrintMsg(Ty->isTokenTy(), "token");
1024 case IITDescriptor::Metadata:
1025 return PrintMsg(Ty->isMetadataTy(), "metadata");
1026 case IITDescriptor::Half:
1027 return PrintMsg(Ty->isHalfTy(), "half");
1028 case IITDescriptor::BFloat:
1029 return PrintMsg(Ty->isBFloatTy(), "bfloat");
1030 case IITDescriptor::Float:
1031 return PrintMsg(Ty->isFloatTy(), "float");
1032 case IITDescriptor::Double:
1033 return PrintMsg(Ty->isDoubleTy(), "double");
1034 case IITDescriptor::Quad:
1035 return PrintMsg(Ty->isFP128Ty(), "fp128");
1036 case IITDescriptor::PPCQuad:
1037 return PrintMsg(Ty->isPPC_FP128Ty(), "ppc_fp128");
1038 case IITDescriptor::Integer:
1039 return PrintMsg(Ty->isIntegerTy(BitWidth: D.IntegerWidth),
1040 "i" + Twine(D.IntegerWidth));
1041 case IITDescriptor::AArch64Svcount:
1042 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1043 cast<TargetExtType>(Val: Ty)->getName() == "aarch64.svcount",
1044 "aarch64.svcount");
1045 case IITDescriptor::WasmExternref:
1046 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1047 cast<TargetExtType>(Val: Ty)->getName() == "wasm.externref",
1048 "wasm.externref");
1049 case IITDescriptor::WasmFuncref:
1050 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1051 cast<TargetExtType>(Val: Ty)->getName() == "wasm.funcref",
1052 "wasm.funcref");
1053 case IITDescriptor::Vector: {
1054 VectorType *VT = dyn_cast<VectorType>(Val: Ty);
1055 StringRef Scalable = D.VectorWidth.isScalable() ? "vscale " : "";
1056 bool HasError =
1057 PrintMsg(VT && VT->getElementCount() == D.VectorWidth,
1058 Twine(Scalable) + "vector with " +
1059 Twine(D.VectorWidth.getKnownMinValue()) + " elements");
1060 if (HasError)
1061 return true;
1062 Position.push_vector_element();
1063 return matchIntrinsicType(Ty: VT->getElementType(), Infos, Position,
1064 OverloadTys, DeferredChecks, IsDeferredCheck, OS);
1065 }
1066 case IITDescriptor::Pointer: {
1067 PointerType *PT = dyn_cast<PointerType>(Val: Ty);
1068 unsigned AS = D.PointerAddressSpace;
1069 bool IsValid = PT && PT->getAddressSpace() == AS;
1070 if (AS == 0)
1071 return PrintMsg(IsValid, "ptr");
1072 return PrintMsg(IsValid, "ptr addrspace(" + Twine(AS) + ")");
1073 }
1074
1075 case IITDescriptor::Struct: {
1076 StructType *ST = dyn_cast<StructType>(Val: Ty);
1077 unsigned EC = D.StructNumElements;
1078 bool HasError = PrintMsg(
1079 ST && ST->isLiteral() && !ST->isPacked() && ST->getNumElements() == EC,
1080 "literal non-packed struct with " + Twine(EC) + " elements");
1081 if (HasError)
1082 return true;
1083
1084 for (const auto &[Idx, ETy] : llvm::enumerate(First: ST->elements())) {
1085 Position.push_struct_element(ElementNum: Idx);
1086 if (matchIntrinsicType(Ty: ETy, Infos, Position, OverloadTys, DeferredChecks,
1087 IsDeferredCheck, OS))
1088 return true;
1089 Position.pop_index();
1090 }
1091 return false;
1092 }
1093
1094 case IITDescriptor::Overloaded: {
1095 unsigned OIdx = D.getOverloadIndex();
1096 assert(OIdx == OverloadTys.size() && !IsDeferredCheck &&
1097 "Table consistency error");
1098 OverloadTys.push_back(Elt: Ty);
1099
1100 // Token has no mangling (see getMangledTypeStr), so it cannot be an
1101 // overload type; reject it with a signature error. Label is already
1102 // excluded from function signatures, so it never reaches here.
1103 if (Ty->isTokenTy())
1104 return PrintMsg(false, "any manglable type", OIdx);
1105
1106 IITDescriptor::AnyKindVectorConstraint VC;
1107 IITDescriptor::AnyKindElementConstraint EC;
1108 std::tie(args&: VC, args&: EC) = D.getOverloadConstraints();
1109
1110 bool IsValid = [&]() {
1111 switch (VC) {
1112 case IITDescriptor::VC_None:
1113 return true;
1114 case IITDescriptor::VC_Vector:
1115 return isa<VectorType>(Val: Ty);
1116 case IITDescriptor::VC_Scalar:
1117 return !isa<VectorType>(Val: Ty);
1118 }
1119 llvm_unreachable("invalid vector constraint");
1120 }();
1121
1122 IsValid &= [&]() {
1123 Type *ETy = Ty->getScalarType();
1124 switch (EC) {
1125 case IITDescriptor::EC_None:
1126 return true;
1127 case IITDescriptor::EC_Integer:
1128 return ETy->isIntegerTy();
1129 case IITDescriptor::EC_Float:
1130 return ETy->isFloatingPointTy();
1131 case IITDescriptor::EC_Pointer:
1132 return ETy->isPointerTy();
1133 }
1134 llvm_unreachable("invalid element constraint");
1135 }();
1136
1137 if (IsValid)
1138 return false;
1139
1140 static constexpr StringLiteral VectorKinds[] = {
1141 "",
1142 "vector",
1143 "scalar",
1144 };
1145 static constexpr StringLiteral ElementKinds[] = {
1146 "",
1147 "integer",
1148 "fp",
1149 "pointer",
1150 };
1151
1152 if (EC == IITDescriptor::EC_None) {
1153 // No constraint on element type.
1154 // Expected = any {vector | scalar} type.
1155 StringLiteral VK = ArrayRef(VectorKinds)[VC];
1156 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: VK), OIdx);
1157 }
1158
1159 StringLiteral EK = ArrayRef(ElementKinds)[EC];
1160 switch (VC) {
1161 case IITDescriptor::VC_None:
1162 // Expected = any EK or EK vector.
1163 return PrintMsg(false, formatv(Fmt: "any {0} or {0} vector", Vals&: EK), OIdx);
1164 case IITDescriptor::VC_Vector:
1165 return PrintMsg(false, formatv(Fmt: "any {} vector", Vals&: EK), OIdx);
1166 case IITDescriptor::VC_Scalar:
1167 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: EK), OIdx);
1168 }
1169 llvm_unreachable("invalid vector constraint");
1170 }
1171
1172 case IITDescriptor::Match: {
1173 unsigned OIdx = D.getOverloadIndex();
1174 if (OIdx >= OverloadTys.size())
1175 return IsDeferredCheck || DeferCheck(Ty);
1176 return PrintMsgInvalidDepType(Ty == OverloadTys[OIdx], "matching",
1177 formatv(Fmt: "{}", Vals&: *OverloadTys[OIdx]), OIdx);
1178 }
1179
1180 case IITDescriptor::Extend:
1181 case IITDescriptor::Trunc: {
1182 unsigned OIdx = D.getOverloadIndex();
1183 // If this is a forward reference, defer the check for later.
1184 if (OIdx >= OverloadTys.size())
1185 return IsDeferredCheck || DeferCheck(Ty);
1186
1187 Type *OTy = OverloadTys[OIdx];
1188 bool IsExtend = D.Kind == IITDescriptor::Extend;
1189 StringRef Qualifier = IsExtend ? "extended" : "truncated";
1190 if (!OTy->isIntOrIntVectorTy())
1191 return PrintMsgInvalidOverloadTy(Qualifier, "int or vector of int", OIdx);
1192
1193 Type *NewTy = IsExtend ? OTy->getExtendedType() : OTy->getTruncatedType();
1194 return PrintMsgInvalidDepType(Ty == NewTy, Qualifier, formatv(Fmt: "{}", Vals&: *NewTy),
1195 OIdx);
1196 }
1197 case IITDescriptor::OneNthEltsVec: {
1198 unsigned OIdx = D.getOverloadIndex();
1199 unsigned Divisor = D.getVectorDivisor();
1200 // If this is a forward reference, defer the check for later.
1201 if (OIdx >= OverloadTys.size())
1202 return IsDeferredCheck || DeferCheck(Ty);
1203 Type *OTy = OverloadTys[OIdx];
1204 auto *OVecTy = dyn_cast<VectorType>(Val: OTy);
1205 auto Qualifier = formatv(Fmt: "1/nth (n={}) elements vector of", Vals&: Divisor);
1206 if (!OVecTy)
1207 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1208 if (!OVecTy->getElementCount().isKnownMultipleOf(RHS: Divisor))
1209 return PrintMsgInvalidOverloadTy(
1210 Qualifier, formatv(Fmt: "vector with multiple of {} elements", Vals&: Divisor),
1211 OIdx);
1212 Type *Expected = VectorType::getOneNthElementsVectorType(VTy: OVecTy, Denominator: Divisor);
1213 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1214 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1215 }
1216 case IITDescriptor::SameVecWidth: {
1217 unsigned OIdx = D.getOverloadIndex();
1218 if (OIdx >= OverloadTys.size()) {
1219 // Defer check and subsequent check for the vector element type.
1220 Infos.consume_front();
1221 return IsDeferredCheck || DeferCheck(Ty);
1222 }
1223 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1224 auto *ThisArgVecType = dyn_cast<VectorType>(Val: Ty);
1225 // Both must be vectors of the same number of elements or neither.
1226 StringRef Qualifier = "same vector width of";
1227 if (OVecTy && !ThisArgVecType)
1228 return PrintMsgInvalidDepType(false, Qualifier, "vector", OIdx);
1229 if (!OVecTy && ThisArgVecType)
1230 return PrintMsgInvalidDepType(false, Qualifier, "scalar", OIdx);
1231 Type *EltTy = Ty;
1232 if (ThisArgVecType) {
1233 ElementCount Expected = OVecTy->getElementCount();
1234 if (Expected != ThisArgVecType->getElementCount())
1235 return PrintMsgInvalidDepType(
1236 false, Qualifier, formatv(Fmt: "vector with {} elements", Vals&: Expected),
1237 OIdx);
1238 EltTy = ThisArgVecType->getElementType();
1239 Position.push_vector_element();
1240 }
1241 return matchIntrinsicType(Ty: EltTy, Infos, Position, OverloadTys,
1242 DeferredChecks, IsDeferredCheck, OS);
1243 }
1244 case IITDescriptor::VecOfAnyPtrsToElt: {
1245 unsigned RefOverloadIndex = D.getRefOverloadIndex();
1246 if (RefOverloadIndex >= OverloadTys.size()) {
1247 if (IsDeferredCheck)
1248 return true;
1249 // If forward referencing, already add the pointer-vector type and
1250 // defer the checks for later.
1251 assert(D.getOverloadIndex() == OverloadTys.size() &&
1252 "Table consistency error");
1253 OverloadTys.push_back(Elt: Ty);
1254 return DeferCheck(Ty);
1255 }
1256
1257 if (!IsDeferredCheck) {
1258 assert(D.getOverloadIndex() == OverloadTys.size() &&
1259 "Table consistency error");
1260 OverloadTys.push_back(Elt: Ty);
1261 }
1262
1263 // Verify the overloaded type "matches" the Ref type.
1264 // i.e. Ty is a vector with the same width as Ref and composed of pointers.
1265
1266 StringRef Qualifier = "vector of pointers to elements of";
1267 auto *ReferenceType = dyn_cast<VectorType>(Val: OverloadTys[RefOverloadIndex]);
1268 if (!ReferenceType)
1269 return PrintMsgInvalidOverloadTy(Qualifier, "vector", RefOverloadIndex);
1270
1271 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1272 if (!ThisArgVecTy)
1273 return PrintMsgInvalidDepType(false, Qualifier, "vector",
1274 RefOverloadIndex);
1275
1276 auto ExpectedCount = ReferenceType->getElementCount();
1277 auto Expected =
1278 formatv(Fmt: "vector of pointers with {} elements", Vals&: ExpectedCount);
1279 bool IsValid = ThisArgVecTy->getElementCount() == ExpectedCount &&
1280 ThisArgVecTy->getElementType()->isPointerTy();
1281 return PrintMsgInvalidDepType(IsValid, Qualifier, Expected,
1282 RefOverloadIndex);
1283 }
1284 case IITDescriptor::VecElement: {
1285 unsigned OIdx = D.getOverloadIndex();
1286 if (OIdx >= OverloadTys.size())
1287 return IsDeferredCheck || DeferCheck(Ty);
1288 StringRef Qualifier = "vector element of";
1289 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1290 if (!OVecTy)
1291 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1292 Type *Expected = OVecTy->getElementType();
1293 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1294 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1295 }
1296 case IITDescriptor::Subdivide2:
1297 case IITDescriptor::Subdivide4: {
1298 unsigned OIdx = D.getOverloadIndex();
1299 // If this is a forward reference, defer the check for later.
1300 if (OIdx >= OverloadTys.size())
1301 return IsDeferredCheck || DeferCheck(Ty);
1302
1303 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
1304 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1305 auto Qualifier =
1306 formatv(Fmt: "subdivided by {} vector of", Vals: SubDivs == 1 ? 2 : 4);
1307 if (!OVecTy)
1308 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1309
1310 // TODO: Verify that the element type of the overload type is subdivisible
1311 // by 2 or 4.
1312 Type *Expected = VectorType::getSubdividedVectorType(VTy: OVecTy, NumSubdivs: SubDivs);
1313 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1314 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1315 }
1316 case IITDescriptor::VecOfBitcastsToInt: {
1317 unsigned OIdx = D.getOverloadIndex();
1318 if (OIdx >= OverloadTys.size())
1319 return IsDeferredCheck || DeferCheck(Ty);
1320 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1321 StringRef Qualifier = "vector of bitcasts to int of";
1322 if (!OVecTy)
1323 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1324 Type *Expected = VectorType::getInteger(VTy: OVecTy);
1325 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1326 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1327 }
1328 case IITDescriptor::VarArg:
1329 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
1330 // should never see it here.
1331 llvm_unreachable("IITDescriptor::VarArg not expected");
1332 }
1333 llvm_unreachable("unhandled");
1334}
1335
1336/// Return true if the function type \p FTy is a valid type signature for the
1337/// type constraints specified in the .td file, represented by \p Infos and
1338/// \p IsVarArg. The overloaded types for the intrinsic are pushed to the
1339/// \p OverloadTys vector.
1340///
1341/// If the type is not valid, returns false and prints an error message to
1342/// \p OS.
1343static bool isSignatureValid(FunctionType *FTy,
1344 ArrayRef<Intrinsic::IITDescriptor> &Infos,
1345 unsigned NumArgs, bool IsVarArg,
1346 SmallVectorImpl<Type *> &OverloadTys,
1347 raw_ostream &OS) {
1348 SmallVector<DeferredIntrinsicMatchInfo, 2> DeferredChecks;
1349
1350 assert(!Infos.empty() && "Table consistency error");
1351
1352 MatchPosition Pos;
1353 Pos.IsRet = true;
1354 Pos.Num = 0;
1355
1356 if (matchIntrinsicType(Ty: FTy->getReturnType(), Infos, Position: Pos, OverloadTys,
1357 DeferredChecks, IsDeferredCheck: false, OS))
1358 return false;
1359
1360 if (FTy->getNumParams() != NumArgs) {
1361 OS << "intrinsic has incorrect number of args. Expected " << NumArgs
1362 << ", but got " << FTy->getNumParams();
1363 return false;
1364 }
1365
1366 Pos.IsRet = false;
1367 for (const auto &[Idx, Ty] : llvm::enumerate(First: FTy->params())) {
1368 Pos.Num = Idx;
1369 if (matchIntrinsicType(Ty, Infos, Position: Pos, OverloadTys, DeferredChecks, IsDeferredCheck: false,
1370 OS))
1371 return false;
1372 }
1373
1374 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
1375 auto &[DefTy, DefInfos, DefPosition] = DeferredChecks[I];
1376 if (matchIntrinsicType(Ty: DefTy, Infos&: DefInfos, Position: DefPosition, OverloadTys,
1377 DeferredChecks, IsDeferredCheck: true, OS))
1378 return false;
1379 }
1380
1381 if (!Infos.empty()) {
1382 OS << "intrinsic has too few arguments!";
1383 return false;
1384 }
1385
1386 if (FTy->isVarArg() != IsVarArg) {
1387 if (IsVarArg)
1388 OS << "intrinsic was not defined with variable arguments!";
1389 else
1390 OS << "intrinsic was defined with variable arguments!";
1391 return false;
1392 }
1393
1394 return true;
1395}
1396
1397bool Intrinsic::hasStructReturnType(ID id) {
1398 using namespace Intrinsic;
1399 SmallVector<IITDescriptor> Table;
1400 getIntrinsicInfoTableEntries(id, T&: Table);
1401 return !Table.empty() && Table[0].Kind == IITDescriptor::Struct;
1402}
1403
1404bool Intrinsic::isSignatureValid(Intrinsic::ID ID, FunctionType *FT,
1405 SmallVectorImpl<Type *> &OverloadTys,
1406 raw_ostream &OS) {
1407 if (!ID)
1408 return false;
1409
1410 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1411 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id: ID, T&: Table);
1412
1413 return ::isSignatureValid(FTy: FT, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS);
1414}
1415
1416bool Intrinsic::isSignatureValid(Function *F,
1417 SmallVectorImpl<Type *> &OverloadTys,
1418 raw_ostream &OS) {
1419 return isSignatureValid(ID: F->getIntrinsicID(), FT: F->getFunctionType(),
1420 OverloadTys, OS);
1421}
1422
1423std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
1424 SmallVector<Type *, 4> OverloadTys;
1425 if (!isSignatureValid(F, OverloadTys))
1426 return std::nullopt;
1427
1428 Intrinsic::ID ID = F->getIntrinsicID();
1429 StringRef Name = F->getName();
1430 std::string WantedName =
1431 Intrinsic::getName(Id: ID, OverloadTys, M: F->getParent(), FT: F->getFunctionType());
1432 if (Name == WantedName)
1433 return std::nullopt;
1434
1435 Function *NewDecl = [&] {
1436 if (auto *ExistingGV = F->getParent()->getNamedValue(Name: WantedName)) {
1437 if (auto *ExistingF = dyn_cast<Function>(Val: ExistingGV))
1438 if (ExistingF->getFunctionType() == F->getFunctionType())
1439 return ExistingF;
1440
1441 // The name already exists, but is not a function or has the wrong
1442 // prototype. Make place for the new one by renaming the old version.
1443 // Either this old version will be removed later on or the module is
1444 // invalid and we'll get an error.
1445 ExistingGV->setName(WantedName + ".renamed");
1446 }
1447 return Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys);
1448 }();
1449
1450 NewDecl->setCallingConv(F->getCallingConv());
1451 assert(NewDecl->getFunctionType() == F->getFunctionType() &&
1452 "Shouldn't change the signature");
1453 return NewDecl;
1454}
1455
1456struct InterleaveIntrinsic {
1457 Intrinsic::ID Interleave, Deinterleave;
1458};
1459
1460static InterleaveIntrinsic InterleaveIntrinsics[] = {
1461 {.Interleave: Intrinsic::vector_interleave2, .Deinterleave: Intrinsic::vector_deinterleave2},
1462 {.Interleave: Intrinsic::vector_interleave3, .Deinterleave: Intrinsic::vector_deinterleave3},
1463 {.Interleave: Intrinsic::vector_interleave4, .Deinterleave: Intrinsic::vector_deinterleave4},
1464 {.Interleave: Intrinsic::vector_interleave5, .Deinterleave: Intrinsic::vector_deinterleave5},
1465 {.Interleave: Intrinsic::vector_interleave6, .Deinterleave: Intrinsic::vector_deinterleave6},
1466 {.Interleave: Intrinsic::vector_interleave7, .Deinterleave: Intrinsic::vector_deinterleave7},
1467 {.Interleave: Intrinsic::vector_interleave8, .Deinterleave: Intrinsic::vector_deinterleave8},
1468};
1469
1470Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
1471 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1472 return InterleaveIntrinsics[Factor - 2].Interleave;
1473}
1474
1475Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
1476 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1477 return InterleaveIntrinsics[Factor - 2].Deinterleave;
1478}
1479
1480LLVM_ABI void Intrinsic::printFPClassMask(raw_ostream &OS,
1481 const Constant *ImmArgVal) {
1482 uint64_t Val = cast<ConstantInt>(Val: ImmArgVal)->getZExtValue();
1483 OS << static_cast<FPClassTest>(Val);
1484}
1485
1486#define GET_INTRINSIC_IMMARG_RANGE_SET_CHECKS
1487#include "llvm/IR/IntrinsicImpl.inc"
1488
1489#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1490#include "llvm/IR/IntrinsicImpl.inc"
1491
1492// Emit the default-argument values table and lookup function
1493// (Intrinsic::getAllDefaultArgValues).
1494#define GET_INTRINSIC_DEFAULT_ARG_VALUES
1495#include "llvm/IR/IntrinsicImpl.inc"
1496