1//===-- Intrinsics.cpp - Intrinsic Function Handling ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions required for supporting intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Intrinsics.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringTable.h"
16#include "llvm/IR/ConstantRange.h"
17#include "llvm/IR/Function.h"
18#include "llvm/IR/IntrinsicsAArch64.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/IR/IntrinsicsARM.h"
21#include "llvm/IR/IntrinsicsBPF.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
23#include "llvm/IR/IntrinsicsLoongArch.h"
24#include "llvm/IR/IntrinsicsMips.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/IR/IntrinsicsPowerPC.h"
27#include "llvm/IR/IntrinsicsR600.h"
28#include "llvm/IR/IntrinsicsRISCV.h"
29#include "llvm/IR/IntrinsicsS390.h"
30#include "llvm/IR/IntrinsicsSPIRV.h"
31#include "llvm/IR/IntrinsicsVE.h"
32#include "llvm/IR/IntrinsicsX86.h"
33#include "llvm/IR/IntrinsicsXCore.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NVVMIntrinsicUtils.h"
36#include "llvm/IR/Type.h"
37#include "llvm/Support/FormatVariadic.h"
38#include "llvm/Support/MathExtras.h"
39
40using namespace llvm;
41
42// Forward declaration of static functions.
43static bool isSignatureValid(FunctionType *FTy,
44 ArrayRef<Intrinsic::IITDescriptor> &Infos,
45 unsigned NumArgs, bool IsVarArg,
46 SmallVectorImpl<Type *> &OverloadTys,
47 raw_ostream &OS);
48
49/// Table of string intrinsic names indexed by enum value.
50#define GET_INTRINSIC_NAME_TABLE
51#include "llvm/IR/IntrinsicImpl.inc"
52
53StringRef Intrinsic::getBaseName(ID id) {
54 assert(id < num_intrinsics && "Invalid intrinsic ID!");
55 return IntrinsicNameTable[IntrinsicNameOffsetTable[id]];
56}
57
58StringRef Intrinsic::getName(ID id) {
59 assert(id < num_intrinsics && "Invalid intrinsic ID!");
60 assert(!Intrinsic::isOverloaded(id) &&
61 "This version of getName does not support overloading");
62 return getBaseName(id);
63}
64
65/// Returns a stable mangling for the type specified for use in the name
66/// mangling scheme used by 'any' types in intrinsic signatures. The mangling
67/// of named types is simply their name. Manglings for unnamed types consist
68/// of a prefix ('p' for pointers, 'a' for arrays, 'f_' for functions)
69/// combined with the mangling of their component types. A vararg function
70/// type will have a suffix of 'vararg'. Since function types can contain
71/// other function types, we close a function type mangling with suffix 'f'
72/// which can't be confused with it's prefix. This ensures we don't have
73/// collisions between two unrelated function types. Otherwise, you might
74/// parse ffXX as f(fXX) or f(fX)X. (X is a placeholder for any other type.)
75/// The HasUnnamedType boolean is set if an unnamed type was encountered,
76/// indicating that extra care must be taken to ensure a unique name.
77static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
78 std::string Result;
79 if (PointerType *PTyp = dyn_cast<PointerType>(Val: Ty)) {
80 Result += "p" + utostr(X: PTyp->getAddressSpace());
81 } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Val: Ty)) {
82 Result += "a" + utostr(X: ATyp->getNumElements()) +
83 getMangledTypeStr(Ty: ATyp->getElementType(), HasUnnamedType);
84 } else if (StructType *STyp = dyn_cast<StructType>(Val: Ty)) {
85 if (!STyp->isLiteral()) {
86 Result += "s_";
87 if (STyp->hasName())
88 Result += STyp->getName();
89 else
90 HasUnnamedType = true;
91 } else {
92 Result += "sl_";
93 for (auto *Elem : STyp->elements())
94 Result += getMangledTypeStr(Ty: Elem, HasUnnamedType);
95 }
96 // Ensure nested structs are distinguishable.
97 Result += "s";
98 } else if (FunctionType *FT = dyn_cast<FunctionType>(Val: Ty)) {
99 Result += "f_" + getMangledTypeStr(Ty: FT->getReturnType(), HasUnnamedType);
100 for (size_t i = 0; i < FT->getNumParams(); i++)
101 Result += getMangledTypeStr(Ty: FT->getParamType(i), HasUnnamedType);
102 if (FT->isVarArg())
103 Result += "vararg";
104 // Ensure nested function types are distinguishable.
105 Result += "f";
106 } else if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
107 ElementCount EC = VTy->getElementCount();
108 if (EC.isScalable())
109 Result += "nx";
110 Result += "v" + utostr(X: EC.getKnownMinValue()) +
111 getMangledTypeStr(Ty: VTy->getElementType(), HasUnnamedType);
112 } else if (TargetExtType *TETy = dyn_cast<TargetExtType>(Val: Ty)) {
113 Result += "t";
114 Result += TETy->getName();
115 for (Type *ParamTy : TETy->type_params())
116 Result += "_" + getMangledTypeStr(Ty: ParamTy, HasUnnamedType);
117 for (unsigned IntParam : TETy->int_params())
118 Result += "_" + utostr(X: IntParam);
119 // Ensure nested target extension types are distinguishable.
120 Result += "t";
121 } else if (Ty) {
122 switch (Ty->getTypeID()) {
123 default:
124 llvm_unreachable("Unhandled type");
125 case Type::VoidTyID:
126 Result += "isVoid";
127 break;
128 case Type::MetadataTyID:
129 Result += "Metadata";
130 break;
131 case Type::HalfTyID:
132 Result += "f16";
133 break;
134 case Type::BFloatTyID:
135 Result += "bf16";
136 break;
137 case Type::FloatTyID:
138 Result += "f32";
139 break;
140 case Type::DoubleTyID:
141 Result += "f64";
142 break;
143 case Type::X86_FP80TyID:
144 Result += "f80";
145 break;
146 case Type::FP128TyID:
147 Result += "f128";
148 break;
149 case Type::PPC_FP128TyID:
150 Result += "ppcf128";
151 break;
152 case Type::X86_AMXTyID:
153 Result += "x86amx";
154 break;
155 case Type::IntegerTyID:
156 Result += "i" + utostr(X: cast<IntegerType>(Val: Ty)->getBitWidth());
157 break;
158 case Type::ByteTyID:
159 Result += "b" + utostr(X: cast<ByteType>(Val: Ty)->getBitWidth());
160 break;
161 }
162 }
163 return Result;
164}
165
166static std::string getIntrinsicNameImpl(Intrinsic::ID Id,
167 ArrayRef<Type *> OverloadTys, Module *M,
168 FunctionType *FT,
169 bool EarlyModuleCheck) {
170
171 assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
172 assert((OverloadTys.empty() || Intrinsic::isOverloaded(Id)) &&
173 "This version of getName is for overloaded intrinsics only");
174 (void)EarlyModuleCheck;
175 assert((!EarlyModuleCheck || M ||
176 !any_of(OverloadTys, llvm::IsaPred<PointerType>)) &&
177 "Intrinsic overloading on pointer types need to provide a Module");
178 bool HasUnnamedType = false;
179 std::string Result(Intrinsic::getBaseName(id: Id));
180 for (Type *Ty : OverloadTys)
181 Result += "." + getMangledTypeStr(Ty, HasUnnamedType);
182 if (HasUnnamedType) {
183 assert(M && "unnamed types need a module");
184 if (!FT)
185 FT = Intrinsic::getType(Context&: M->getContext(), id: Id, OverloadTys);
186 else
187 assert(FT == Intrinsic::getType(M->getContext(), Id, OverloadTys) &&
188 "Provided FunctionType must match arguments");
189 return M->getUniqueIntrinsicName(BaseName: Result, Id, Proto: FT);
190 }
191 return Result;
192}
193
194std::string Intrinsic::getName(ID Id, ArrayRef<Type *> OverloadTys, Module *M,
195 FunctionType *FT) {
196 assert(M && "We need to have a Module");
197 return getIntrinsicNameImpl(Id, OverloadTys, M, FT, EarlyModuleCheck: true);
198}
199
200std::string Intrinsic::getNameNoUnnamedTypes(ID Id,
201 ArrayRef<Type *> OverloadTys) {
202 return getIntrinsicNameImpl(Id, OverloadTys, M: nullptr, FT: nullptr, EarlyModuleCheck: false);
203}
204
205/// IIT_Info - These are enumerators that describe the entries returned by the
206/// getIntrinsicInfoTableEntries function.
207///
208/// Defined in Intrinsics.td.
209enum IIT_Info {
210#define GET_INTRINSIC_IITINFO
211#include "llvm/IR/IntrinsicImpl.inc"
212};
213
214static_assert(IIT_Done == 0, "IIT_Done expected to be 0");
215
216static void
217DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
218 SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
219 using namespace Intrinsic;
220
221 auto IsScalableVector = [&]() {
222 IIT_Info NextInfo = IIT_Info(Infos[NextElt]);
223 if (NextInfo != IIT_SCALABLE_VEC)
224 return false;
225 // Eat the IIT_SCALABLE_VEC token.
226 ++NextElt;
227 return true;
228 };
229
230 IIT_Info Info = IIT_Info(Infos[NextElt++]);
231
232 switch (Info) {
233 case IIT_Done:
234 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Void, Field: 0));
235 return;
236 case IIT_VARARG:
237 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VarArg, Field: 0));
238 return;
239 case IIT_MMX:
240 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::MMX, Field: 0));
241 return;
242 case IIT_AMX:
243 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AMX, Field: 0));
244 return;
245 case IIT_TOKEN:
246 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Token, Field: 0));
247 return;
248 case IIT_METADATA:
249 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Metadata, Field: 0));
250 return;
251 case IIT_F16:
252 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Half, Field: 0));
253 return;
254 case IIT_BF16:
255 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::BFloat, Field: 0));
256 return;
257 case IIT_F32:
258 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Float, Field: 0));
259 return;
260 case IIT_F64:
261 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Double, Field: 0));
262 return;
263 case IIT_F128:
264 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Quad, Field: 0));
265 return;
266 case IIT_PPCF128:
267 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::PPCQuad, Field: 0));
268 return;
269 case IIT_I1:
270 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 1));
271 return;
272 case IIT_I2:
273 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 2));
274 return;
275 case IIT_I4:
276 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 4));
277 return;
278 case IIT_AARCH64_SVCOUNT:
279 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::AArch64Svcount, Field: 0));
280 return;
281 case IIT_I8:
282 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 8));
283 return;
284 case IIT_I16:
285 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 16));
286 return;
287 case IIT_I32:
288 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 32));
289 return;
290 case IIT_I64:
291 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 64));
292 return;
293 case IIT_I128:
294 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Integer, Field: 128));
295 return;
296 case IIT_V1:
297 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1, IsScalable: IsScalableVector()));
298 DecodeIITType(NextElt, Infos, OutputTable);
299 return;
300 case IIT_V2:
301 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2, IsScalable: IsScalableVector()));
302 DecodeIITType(NextElt, Infos, OutputTable);
303 return;
304 case IIT_V3:
305 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 3, IsScalable: IsScalableVector()));
306 DecodeIITType(NextElt, Infos, OutputTable);
307 return;
308 case IIT_V4:
309 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4, IsScalable: IsScalableVector()));
310 DecodeIITType(NextElt, Infos, OutputTable);
311 return;
312 case IIT_V6:
313 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 6, IsScalable: IsScalableVector()));
314 DecodeIITType(NextElt, Infos, OutputTable);
315 return;
316 case IIT_V8:
317 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 8, IsScalable: IsScalableVector()));
318 DecodeIITType(NextElt, Infos, OutputTable);
319 return;
320 case IIT_V10:
321 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 10, IsScalable: IsScalableVector()));
322 DecodeIITType(NextElt, Infos, OutputTable);
323 return;
324 case IIT_V16:
325 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 16, IsScalable: IsScalableVector()));
326 DecodeIITType(NextElt, Infos, OutputTable);
327 return;
328 case IIT_V32:
329 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 32, IsScalable: IsScalableVector()));
330 DecodeIITType(NextElt, Infos, OutputTable);
331 return;
332 case IIT_V64:
333 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 64, IsScalable: IsScalableVector()));
334 DecodeIITType(NextElt, Infos, OutputTable);
335 return;
336 case IIT_V128:
337 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 128, IsScalable: IsScalableVector()));
338 DecodeIITType(NextElt, Infos, OutputTable);
339 return;
340 case IIT_V256:
341 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 256, IsScalable: IsScalableVector()));
342 DecodeIITType(NextElt, Infos, OutputTable);
343 return;
344 case IIT_V512:
345 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 512, IsScalable: IsScalableVector()));
346 DecodeIITType(NextElt, Infos, OutputTable);
347 return;
348 case IIT_V1024:
349 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 1024, IsScalable: IsScalableVector()));
350 DecodeIITType(NextElt, Infos, OutputTable);
351 return;
352 case IIT_V2048:
353 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 2048, IsScalable: IsScalableVector()));
354 DecodeIITType(NextElt, Infos, OutputTable);
355 return;
356 case IIT_V4096:
357 OutputTable.push_back(Elt: IITDescriptor::getVector(Width: 4096, IsScalable: IsScalableVector()));
358 DecodeIITType(NextElt, Infos, OutputTable);
359 return;
360 case IIT_EXTERNREF:
361 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmExternref, Field: 0));
362 return;
363 case IIT_FUNCREF:
364 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::WasmFuncref, Field: 0));
365 return;
366 case IIT_PTR:
367 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: 0));
368 return;
369 case IIT_PTR_AS: // pointer with address space.
370 OutputTable.push_back(
371 Elt: IITDescriptor::get(K: IITDescriptor::Pointer, Field: Infos[NextElt++]));
372 return;
373 case IIT_ANY: {
374 unsigned OverloadIndex = Infos[NextElt++];
375 unsigned ArgKindEnums = Infos[NextElt++];
376 unsigned Packed = (ArgKindEnums << 8) | OverloadIndex;
377 OutputTable.push_back(
378 Elt: IITDescriptor::get(K: IITDescriptor::Overloaded, Field: Packed));
379 return;
380 }
381 case IIT_MATCH: {
382 unsigned OverloadIndex = Infos[NextElt++];
383 OutputTable.push_back(
384 Elt: IITDescriptor::get(K: IITDescriptor::Match, Field: OverloadIndex));
385 return;
386 }
387 case IIT_EXTEND_ARG: {
388 unsigned OverloadIndex = Infos[NextElt++];
389 OutputTable.push_back(
390 Elt: IITDescriptor::get(K: IITDescriptor::Extend, Field: OverloadIndex));
391 return;
392 }
393 case IIT_TRUNC_ARG: {
394 unsigned OverloadIndex = Infos[NextElt++];
395 OutputTable.push_back(
396 Elt: IITDescriptor::get(K: IITDescriptor::Trunc, Field: OverloadIndex));
397 return;
398 }
399 case IIT_ONE_NTH_ELTS_VEC_ARG: {
400 unsigned short OverloadIndex = Infos[NextElt++];
401 unsigned short N = Infos[NextElt++];
402 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::OneNthEltsVec,
403 /*Hi=*/N, /*Lo=*/OverloadIndex));
404 return;
405 }
406 case IIT_SAME_VEC_WIDTH_ARG: {
407 unsigned OverloadIndex = Infos[NextElt++];
408 OutputTable.push_back(
409 Elt: IITDescriptor::get(K: IITDescriptor::SameVecWidth, Field: OverloadIndex));
410 // IIT_SAME_VEC_WIDTH_ARG entry is followed by the element type.
411 DecodeIITType(NextElt, Infos, OutputTable);
412 return;
413 }
414 case IIT_VEC_OF_ANYPTRS_TO_ELT: {
415 unsigned short OverloadIndex = Infos[NextElt++];
416 unsigned short RefOverloadIndex = Infos[NextElt++];
417 OutputTable.push_back(Elt: IITDescriptor::get(K: IITDescriptor::VecOfAnyPtrsToElt,
418 /*Hi=*/RefOverloadIndex,
419 /*Lo=*/OverloadIndex));
420 return;
421 }
422 case IIT_STRUCT: {
423 unsigned StructElts = Infos[NextElt++] + 2;
424
425 OutputTable.push_back(
426 Elt: IITDescriptor::get(K: IITDescriptor::Struct, Field: StructElts));
427
428 for (unsigned i = 0; i != StructElts; ++i)
429 DecodeIITType(NextElt, Infos, OutputTable);
430 return;
431 }
432 case IIT_SUBDIVIDE2_ARG: {
433 unsigned OverloadIndex = Infos[NextElt++];
434 OutputTable.push_back(
435 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide2, Field: OverloadIndex));
436 return;
437 }
438 case IIT_SUBDIVIDE4_ARG: {
439 unsigned OverloadIndex = Infos[NextElt++];
440 OutputTable.push_back(
441 Elt: IITDescriptor::get(K: IITDescriptor::Subdivide4, Field: OverloadIndex));
442 return;
443 }
444 case IIT_VEC_ELEMENT: {
445 unsigned OverloadIndex = Infos[NextElt++];
446 OutputTable.push_back(
447 Elt: IITDescriptor::get(K: IITDescriptor::VecElement, Field: OverloadIndex));
448 return;
449 }
450 case IIT_VEC_OF_BITCASTS_TO_INT: {
451 unsigned OverloadIndex = Infos[NextElt++];
452 OutputTable.push_back(
453 Elt: IITDescriptor::get(K: IITDescriptor::VecOfBitcastsToInt, Field: OverloadIndex));
454 return;
455 }
456 case IIT_SCALABLE_VEC:
457 break;
458 }
459 llvm_unreachable("unhandled");
460}
461
462#define GET_INTRINSIC_GENERATOR_GLOBAL
463#include "llvm/IR/IntrinsicImpl.inc"
464
465std::tuple<ArrayRef<Intrinsic::IITDescriptor>, unsigned, bool>
466Intrinsic::getIntrinsicInfoTableEntries(ID id,
467 SmallVectorImpl<IITDescriptor> &T) {
468 // Note that `FixedEncodingTy` is defined in IntrinsicImpl.inc and can be
469 // uint16_t or uint32_t based on the the value of `Use16BitFixedEncoding` in
470 // IntrinsicEmitter.cpp.
471 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
472 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
473 // Mask with all bits 1 except the most significant bit.
474 constexpr unsigned Mask = (1U << MSBPosition) - 1;
475
476 FixedEncodingTy TableVal = IIT_Table[id - 1];
477
478 // Array to hold the inlined fixed encoding values expanded from nibbles to
479 // bytes. Its size can be be atmost FixedEncodingBits / 4 i.e., number
480 // of nibbles that can fit in `FixedEncodingTy` + 1 (the IIT_Done terminator
481 // that is not explicitly encoded). Note that if there are trailing 0 bytes
482 // in the encoding (for example, payload following one of the IIT tokens),
483 // the inlined encoding does not encode the actual size of the encoding, so
484 // we always assume its size of this maximum length possible, followed by the
485 // IIT_Done terminator token (whose value is 0).
486 unsigned char IITValues[FixedEncodingBits / 4 + 1] = {0};
487
488 ArrayRef<unsigned char> IITEntries;
489 unsigned NextElt = 0;
490 // Check to see if the intrinsic's type was inlined in the fixed encoding
491 // table.
492 if (TableVal >> MSBPosition) {
493 // This is an offset into the IIT_LongEncodingTable.
494 IITEntries = IIT_LongEncodingTable;
495
496 // Strip sentinel bit.
497 NextElt = TableVal & Mask;
498 } else {
499 // If the entry was encoded into a single word in the table itself, decode
500 // it from an array of nibbles to an array of bytes.
501 do {
502 IITValues[NextElt++] = TableVal & 0xF;
503 TableVal >>= 4;
504 } while (TableVal);
505
506 IITEntries = IITValues;
507 NextElt = 0;
508 }
509
510 // Okay, decode the table into the output vector of IITDescriptors.
511 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
512 unsigned NumArgs = 0;
513 while (IITEntries[NextElt] != IIT_Done) {
514 DecodeIITType(NextElt, Infos: IITEntries, OutputTable&: T);
515 ++NumArgs;
516 }
517
518 ArrayRef<IITDescriptor> TableRef = T;
519
520 bool IsVarArg = false;
521 if (TableRef.back().Kind == Intrinsic::IITDescriptor::VarArg) {
522 IsVarArg = true;
523 TableRef.consume_back();
524 --NumArgs;
525 }
526 return {TableRef, NumArgs, IsVarArg};
527}
528
529static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
530 ArrayRef<Type *> OverloadTys,
531 LLVMContext &Context) {
532 using namespace Intrinsic;
533
534 IITDescriptor D = Infos.consume_front();
535
536 switch (D.Kind) {
537 case IITDescriptor::Void:
538 return Type::getVoidTy(C&: Context);
539 case IITDescriptor::MMX:
540 return llvm::FixedVectorType::get(ElementType: llvm::IntegerType::get(C&: Context, NumBits: 64), NumElts: 1);
541 case IITDescriptor::AMX:
542 return Type::getX86_AMXTy(C&: Context);
543 case IITDescriptor::Token:
544 return Type::getTokenTy(C&: Context);
545 case IITDescriptor::Metadata:
546 return Type::getMetadataTy(C&: Context);
547 case IITDescriptor::Half:
548 return Type::getHalfTy(C&: Context);
549 case IITDescriptor::BFloat:
550 return Type::getBFloatTy(C&: Context);
551 case IITDescriptor::Float:
552 return Type::getFloatTy(C&: Context);
553 case IITDescriptor::Double:
554 return Type::getDoubleTy(C&: Context);
555 case IITDescriptor::Quad:
556 return Type::getFP128Ty(C&: Context);
557 case IITDescriptor::PPCQuad:
558 return Type::getPPC_FP128Ty(C&: Context);
559 case IITDescriptor::AArch64Svcount:
560 return TargetExtType::get(Context, Name: "aarch64.svcount");
561 case IITDescriptor::WasmExternref:
562 return TargetExtType::get(Context, Name: "wasm.externref");
563 case IITDescriptor::WasmFuncref:
564 return TargetExtType::get(Context, Name: "wasm.funcref");
565 case IITDescriptor::Integer:
566 return IntegerType::get(C&: Context, NumBits: D.IntegerWidth);
567 case IITDescriptor::Vector:
568 return VectorType::get(ElementType: DecodeFixedType(Infos, OverloadTys, Context),
569 EC: D.VectorWidth);
570 case IITDescriptor::Pointer:
571 return PointerType::get(C&: Context, AddressSpace: D.PointerAddressSpace);
572 case IITDescriptor::Struct: {
573 SmallVector<Type *, 8> Elts;
574 for (unsigned i = 0, e = D.StructNumElements; i != e; ++i)
575 Elts.push_back(Elt: DecodeFixedType(Infos, OverloadTys, Context));
576 return StructType::get(Context, Elements: Elts);
577 }
578 // For any overload type or partially dependent type, substitute it with the
579 // corresponding concrete type from OverloadTys. Additionally, do the same
580 // for the fully dependent type that matches an overload type.
581 case IITDescriptor::Overloaded:
582 case IITDescriptor::VecOfAnyPtrsToElt:
583 case IITDescriptor::Match:
584 return OverloadTys[D.getOverloadIndex()];
585 case IITDescriptor::Extend:
586 return OverloadTys[D.getOverloadIndex()]->getExtendedType();
587 case IITDescriptor::Trunc:
588 return OverloadTys[D.getOverloadIndex()]->getTruncatedType();
589 case IITDescriptor::Subdivide2:
590 case IITDescriptor::Subdivide4: {
591 Type *Ty = OverloadTys[D.getOverloadIndex()];
592 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
593 assert(VTy && "Expected overload type to be a Vector Type");
594 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
595 return VectorType::getSubdividedVectorType(VTy, NumSubdivs: SubDivs);
596 }
597 case IITDescriptor::OneNthEltsVec:
598 return VectorType::getOneNthElementsVectorType(
599 VTy: cast<VectorType>(Val: OverloadTys[D.getOverloadIndex()]),
600 Denominator: D.getVectorDivisor());
601 case IITDescriptor::SameVecWidth: {
602 Type *EltTy = DecodeFixedType(Infos, OverloadTys, Context);
603 Type *Ty = OverloadTys[D.getOverloadIndex()];
604 if (auto *VTy = dyn_cast<VectorType>(Val: Ty))
605 return VectorType::get(ElementType: EltTy, EC: VTy->getElementCount());
606 return EltTy;
607 }
608 case IITDescriptor::VecElement: {
609 Type *Ty = OverloadTys[D.getOverloadIndex()];
610 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty))
611 return VTy->getElementType();
612 llvm_unreachable("Expected overload type to be a Vector Type");
613 }
614 case IITDescriptor::VecOfBitcastsToInt: {
615 Type *Ty = OverloadTys[D.getOverloadIndex()];
616 VectorType *VTy = dyn_cast<VectorType>(Val: Ty);
617 assert(VTy && "Expected overload type to be a Vector Type");
618 return VectorType::getInteger(VTy);
619 }
620 case IITDescriptor::VarArg:
621 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
622 // should never see it here.
623 llvm_unreachable("IITDescriptor::VarArg not expected");
624 }
625 llvm_unreachable("unhandled");
626}
627
628FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
629 ArrayRef<Type *> OverloadTys) {
630 SmallVector<IITDescriptor, 8> Table;
631 auto [TableRef, _, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
632
633 Type *ResultTy = DecodeFixedType(Infos&: TableRef, OverloadTys, Context);
634
635 SmallVector<Type *, 8> ArgTys;
636 while (!TableRef.empty())
637 ArgTys.push_back(Elt: DecodeFixedType(Infos&: TableRef, OverloadTys, Context));
638 return FunctionType::get(Result: ResultTy, Params: ArgTys, isVarArg: IsVarArg);
639}
640
641bool Intrinsic::isOverloaded(ID id) {
642#define GET_INTRINSIC_OVERLOAD_TABLE
643#include "llvm/IR/IntrinsicImpl.inc"
644}
645
646bool Intrinsic::isTriviallyScalarizable(ID id) {
647#define GET_INTRINSIC_SCALARIZABLE_TABLE
648#include "llvm/IR/IntrinsicImpl.inc"
649}
650
651bool Intrinsic::hasPrettyPrintedArgs(ID id){
652#define GET_INTRINSIC_PRETTY_PRINT_TABLE
653#include "llvm/IR/IntrinsicImpl.inc"
654}
655
656/// Table of per-target intrinsic name tables.
657#define GET_INTRINSIC_TARGET_DATA
658#include "llvm/IR/IntrinsicImpl.inc"
659
660bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
661 return IID > TargetInfos[0].Count;
662}
663
664/// Looks up Name in NameTable via binary search. NameTable must be sorted
665/// and all entries must start with "llvm.". If NameTable contains an exact
666/// match for Name or a prefix of Name followed by a dot, its index in
667/// NameTable is returned. Otherwise, -1 is returned.
668static int lookupLLVMIntrinsicByName(ArrayRef<unsigned> NameOffsetTable,
669 StringRef Name, StringRef Target = "") {
670 assert(Name.starts_with("llvm.") && "Unexpected intrinsic prefix");
671 assert(Name.drop_front(5).starts_with(Target) && "Unexpected target");
672
673 // Do successive binary searches of the dotted name components. For
674 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
675 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
676 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
677 // size 1. During the search, we can skip the prefix that we already know is
678 // identical. By using strncmp we consider names with differing suffixes to
679 // be part of the equal range.
680 size_t CmpEnd = 4; // Skip the "llvm" component.
681 if (!Target.empty())
682 CmpEnd += 1 + Target.size(); // skip the .target component.
683
684 const unsigned *Low = NameOffsetTable.begin();
685 const unsigned *High = NameOffsetTable.end();
686 const unsigned *LastLow = Low;
687 while (CmpEnd < Name.size() && High - Low > 0) {
688 size_t CmpStart = CmpEnd;
689 CmpEnd = Name.find(C: '.', From: CmpStart + 1);
690 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
691 auto Cmp = [CmpStart, CmpEnd](auto LHS, auto RHS) {
692 // `equal_range` requires the comparison to work with either side being an
693 // offset or the value. Detect which kind each side is to set up the
694 // compared strings.
695 const char *LHSStr;
696 if constexpr (std::is_integral_v<decltype(LHS)>)
697 LHSStr = IntrinsicNameTable.getCString(O: LHS);
698 else
699 LHSStr = LHS;
700
701 const char *RHSStr;
702 if constexpr (std::is_integral_v<decltype(RHS)>)
703 RHSStr = IntrinsicNameTable.getCString(O: RHS);
704 else
705 RHSStr = RHS;
706
707 return strncmp(s1: LHSStr + CmpStart, s2: RHSStr + CmpStart, n: CmpEnd - CmpStart) <
708 0;
709 };
710 LastLow = Low;
711 std::tie(args&: Low, args&: High) = std::equal_range(first: Low, last: High, val: Name.data(), comp: Cmp);
712 }
713 if (High - Low > 0)
714 LastLow = Low;
715
716 if (LastLow == NameOffsetTable.end())
717 return -1;
718 StringRef NameFound = IntrinsicNameTable[*LastLow];
719 if (Name == NameFound ||
720 (Name.starts_with(Prefix: NameFound) && Name[NameFound.size()] == '.'))
721 return LastLow - NameOffsetTable.begin();
722 return -1;
723}
724
725/// Find the segment of \c IntrinsicNameOffsetTable for intrinsics with the same
726/// target as \c Name, or the generic table if \c Name is not target specific.
727///
728/// Returns the relevant slice of \c IntrinsicNameOffsetTable and the target
729/// name.
730static std::pair<ArrayRef<unsigned>, StringRef>
731findTargetSubtable(StringRef Name) {
732 assert(Name.starts_with("llvm."));
733
734 ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos);
735 // Drop "llvm." and take the first dotted component. That will be the target
736 // if this is target specific.
737 StringRef Target = Name.drop_front(N: 5).split(Separator: '.').first;
738 auto It = partition_point(
739 Range&: Targets, P: [=](const IntrinsicTargetInfo &TI) { return TI.Name < Target; });
740 // We've either found the target or just fall back to the generic set, which
741 // is always first.
742 const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
743 return {ArrayRef(&IntrinsicNameOffsetTable[1] + TI.Offset, TI.Count),
744 TI.Name};
745}
746
747/// This does the actual lookup of an intrinsic ID which matches the given
748/// function name.
749Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
750 auto [NameOffsetTable, Target] = findTargetSubtable(Name);
751 int Idx = lookupLLVMIntrinsicByName(NameOffsetTable, Name, Target);
752 if (Idx == -1)
753 return Intrinsic::not_intrinsic;
754
755 // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
756 // an index into a sub-table.
757 int Adjust = NameOffsetTable.data() - IntrinsicNameOffsetTable;
758 Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
759
760 // If the intrinsic is not overloaded, require an exact match. If it is
761 // overloaded, require either exact or prefix match.
762 const auto MatchSize = IntrinsicNameTable[NameOffsetTable[Idx]].size();
763 assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
764 bool IsExactMatch = Name.size() == MatchSize;
765 return IsExactMatch || Intrinsic::isOverloaded(id: ID) ? ID
766 : Intrinsic::not_intrinsic;
767}
768
769/// This defines the "Intrinsic::getAttributes(ID id)" method.
770#define GET_INTRINSIC_ATTRIBUTES
771#include "llvm/IR/IntrinsicImpl.inc"
772
773static Function *
774getOrInsertIntrinsicDeclarationImpl(Module *M, Intrinsic::ID id,
775 ArrayRef<Type *> OverloadTys,
776 FunctionType *FT) {
777 std::string Name = OverloadTys.empty()
778 ? Intrinsic::getName(id).str()
779 : Intrinsic::getName(Id: id, OverloadTys, M, FT);
780 Function *F = cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
781 if (F->getFunctionType() == FT)
782 return F;
783
784 // It's possible that a declaration for this intrinsic already exists with an
785 // incorrect signature, if the signature has changed, but this particular
786 // declaration has not been auto-upgraded yet. In that case, rename the
787 // invalid declaration and insert a new one with the correct signature. The
788 // invalid declaration will get upgraded later.
789 F->setName(F->getName() + ".invalid");
790 return cast<Function>(Val: M->getOrInsertFunction(Name, T: FT).getCallee());
791}
792
793Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id,
794 ArrayRef<Type *> OverloadTys) {
795 // There can never be multiple globals with the same name of different types,
796 // because intrinsics must be a specific type.
797 FunctionType *FT = getType(Context&: M->getContext(), id, OverloadTys);
798 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT);
799}
800
801Function *Intrinsic::getOrInsertDeclaration(Module *M, ID id, Type *RetTy,
802 ArrayRef<Type *> ArgTys) {
803 // If the intrinsic is not overloaded, use the non-overloaded version.
804 if (!Intrinsic::isOverloaded(id))
805 return getOrInsertDeclaration(M, id);
806
807 // Get the intrinsic signature metadata.
808 SmallVector<Intrinsic::IITDescriptor, 8> Table;
809 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id, T&: Table);
810 FunctionType *FTy = FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: IsVarArg);
811
812 // Automatically determine the overloaded types.
813 SmallVector<Type *, 4> OverloadTys;
814 [[maybe_unused]] bool IsValid = ::isSignatureValid(
815 FTy, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS&: nulls());
816 assert(IsValid && "intrinsic signature mismatch");
817 return getOrInsertIntrinsicDeclarationImpl(M, id, OverloadTys, FT: FTy);
818}
819
820Function *Intrinsic::getDeclarationIfExists(const Module *M, ID id) {
821 return M->getFunction(Name: getName(id));
822}
823
824Function *Intrinsic::getDeclarationIfExists(Module *M, ID id,
825 ArrayRef<Type *> OverloadTys,
826 FunctionType *FT) {
827 return M->getFunction(Name: getName(Id: id, OverloadTys, M, FT));
828}
829
830// This defines the "Intrinsic::getIntrinsicForClangBuiltin()" method.
831#define GET_LLVM_INTRINSIC_FOR_CLANG_BUILTIN
832#include "llvm/IR/IntrinsicImpl.inc"
833
834// This defines the "Intrinsic::getIntrinsicForMSBuiltin()" method.
835#define GET_LLVM_INTRINSIC_FOR_MS_BUILTIN
836#include "llvm/IR/IntrinsicImpl.inc"
837
838bool Intrinsic::isConstrainedFPIntrinsic(ID QID) {
839 switch (QID) {
840#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
841 case Intrinsic::INTRINSIC:
842#include "llvm/IR/ConstrainedOps.def"
843#undef INSTRUCTION
844 return true;
845 default:
846 return false;
847 }
848}
849
850bool Intrinsic::hasConstrainedFPRoundingModeOperand(Intrinsic::ID QID) {
851 switch (QID) {
852#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
853 case Intrinsic::INTRINSIC: \
854 return ROUND_MODE == 1;
855#include "llvm/IR/ConstrainedOps.def"
856#undef INSTRUCTION
857 default:
858 return false;
859 }
860}
861
862// This class represents a position in the intrinsic's type signature and is
863// used to generate error messages in `matchIntrinsicType`. The printed position
864// can be of the following forms:
865//
866// return
867// return struct element 3
868// return vector element
869// return struct element 3 vector element
870// argument 3
871// argument 3 vector element
872//
873// To support deferred checks also being able to generate these error messages
874// we need to encode the position compactly so that it can be stashed into
875// DeferredIntrinsicMatchInfo below (without materializing it into a string).
876// The class below serves that purpose.
877//
878namespace {
879struct MatchPosition {
880 uint16_t IsRet : 1;
881 uint16_t Num : 15; // Argument number (when IsRet = false).
882 struct Index {
883 uint16_t IsStruct : 1; // If true, this is a struct element with element
884 // index `Num`, else its a vector element.
885 uint16_t Num : 15; // Struct element index.
886 };
887 // We expect this to be just 2 levels deep, since nested structs are not
888 // supported.
889 static constexpr unsigned INDEX_TABLE_SIZE = 2;
890 Index Indices[INDEX_TABLE_SIZE];
891 uint16_t NumIndices = 0;
892
893 void pop_index() {
894 assert(NumIndices > 0 && "cannot pop from empty indices");
895 --NumIndices;
896 }
897
898 void push_struct_element(unsigned ElementNum) {
899 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
900 assert(isInt<15>(ElementNum) && "Element index overflow");
901 Indices[NumIndices].IsStruct = true;
902 Indices[NumIndices++].Num = ElementNum;
903 }
904
905 void push_vector_element() {
906 assert(NumIndices < INDEX_TABLE_SIZE && "index table overflow");
907 Indices[NumIndices].IsStruct = false;
908 Indices[NumIndices++].Num = 0;
909 }
910};
911} // namespace
912
913static raw_ostream &operator<<(raw_ostream &OS, const MatchPosition &Pos) {
914 OS << "intrinsic ";
915
916 if (Pos.IsRet)
917 OS << "return";
918 else
919 OS << "argument " << Pos.Num;
920
921 for (const MatchPosition::Index &Idx :
922 ArrayRef(Pos.Indices).take_front(N: Pos.NumIndices)) {
923 if (Idx.IsStruct)
924 OS << " struct element " << Idx.Num;
925 else
926 OS << " vector element";
927 }
928 return OS;
929}
930
931using DeferredIntrinsicMatchInfo =
932 std::tuple<Type *, ArrayRef<Intrinsic::IITDescriptor>, MatchPosition>;
933
934static bool
935matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
936 MatchPosition Position, SmallVectorImpl<Type *> &OverloadTys,
937 SmallVectorImpl<DeferredIntrinsicMatchInfo> &DeferredChecks,
938 bool IsDeferredCheck, raw_ostream &OS) {
939 using namespace Intrinsic;
940
941 // If we ran out of descriptors, there are too many arguments or returns.
942 if (Infos.empty()) {
943 OS << Position << " too many "
944 << (Position.IsRet ? "returns" : "arguments");
945 return true;
946 }
947
948 // Do this before slicing off the 'front' part
949 auto InfosRef = Infos;
950 auto DeferCheck = [&DeferredChecks, &InfosRef, &Position](Type *T) {
951 DeferredChecks.emplace_back(Args&: T, Args&: InfosRef, Args&: Position);
952 return false;
953 };
954
955 IITDescriptor D = Infos.consume_front();
956
957 // Print error message when the (non-dependent) type for current position is
958 // invalid.
959 auto PrintMsg = [&OS, &Position,
960 Ty](bool IsValid, const Twine &Expected,
961 std::optional<unsigned> OIdx = std::nullopt) -> bool {
962 if (IsValid)
963 return false;
964 OS << Position << " type";
965 if (OIdx)
966 OS << " (overload type " << *OIdx << ")";
967 OS << " expected " << Expected << ", but got " << *Ty;
968 return true;
969 };
970
971 // Print message when an overload type is invalid as a result of its use in
972 // current dependent type. DependentQualifier describes the "function" applied
973 // to the overload type to get the dependent type.
974 auto PrintMsgInvalidOverloadTy =
975 [&OS, &Position, &OverloadTys](const Twine &DependentQualifier,
976 const Twine &Expected,
977 unsigned OIdx) -> bool {
978 OS << Position << " is " << DependentQualifier << " overload type " << OIdx
979 << ", so overload type " << OIdx << " expected " << Expected
980 << ", but got " << *OverloadTys[OIdx];
981 return true;
982 };
983
984 // Print message when a dependent type is invalid.
985 auto PrintMsgInvalidDepType =
986 [&OS, &Position, &OverloadTys,
987 Ty](bool IsValid, const Twine &DependentQualifier, const Twine &Expected,
988 unsigned OIdx) -> bool {
989 if (IsValid)
990 return false;
991 bool IsMatching = DependentQualifier.isSingleStringRef() &&
992 DependentQualifier.getSingleStringRef() == "matching";
993 OS << Position << " type (" << DependentQualifier << " overload type "
994 << OIdx << ") expected " << Expected;
995 if (!IsMatching)
996 OS << " (overload type " << OIdx << " is " << *OverloadTys[OIdx] << ")";
997 OS << ", but got " << *Ty;
998 return true;
999 };
1000
1001 switch (D.Kind) {
1002 case IITDescriptor::Void:
1003 assert(Position.IsRet && Position.NumIndices == 0 &&
1004 "void descriptor expected only for return type");
1005 return PrintMsg(Ty->isVoidTy(), "void");
1006 case IITDescriptor::MMX: {
1007 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
1008 return PrintMsg(VT && VT->getNumElements() == 1 &&
1009 VT->getElementType()->isIntegerTy(BitWidth: 64),
1010 "x86_mmx (<1 x i64>)");
1011 }
1012 case IITDescriptor::AMX:
1013 return PrintMsg(Ty->isX86_AMXTy(), "x86_amx");
1014 case IITDescriptor::Token:
1015 return PrintMsg(Ty->isTokenTy(), "token");
1016 case IITDescriptor::Metadata:
1017 return PrintMsg(Ty->isMetadataTy(), "metadata");
1018 case IITDescriptor::Half:
1019 return PrintMsg(Ty->isHalfTy(), "half");
1020 case IITDescriptor::BFloat:
1021 return PrintMsg(Ty->isBFloatTy(), "bfloat");
1022 case IITDescriptor::Float:
1023 return PrintMsg(Ty->isFloatTy(), "float");
1024 case IITDescriptor::Double:
1025 return PrintMsg(Ty->isDoubleTy(), "double");
1026 case IITDescriptor::Quad:
1027 return PrintMsg(Ty->isFP128Ty(), "fp128");
1028 case IITDescriptor::PPCQuad:
1029 return PrintMsg(Ty->isPPC_FP128Ty(), "ppc_fp128");
1030 case IITDescriptor::Integer:
1031 return PrintMsg(Ty->isIntegerTy(BitWidth: D.IntegerWidth),
1032 "i" + Twine(D.IntegerWidth));
1033 case IITDescriptor::AArch64Svcount:
1034 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1035 cast<TargetExtType>(Val: Ty)->getName() == "aarch64.svcount",
1036 "aarch64.svcount");
1037 case IITDescriptor::WasmExternref:
1038 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1039 cast<TargetExtType>(Val: Ty)->getName() == "wasm.externref",
1040 "wasm.externref");
1041 case IITDescriptor::WasmFuncref:
1042 return PrintMsg(isa<TargetExtType>(Val: Ty) &&
1043 cast<TargetExtType>(Val: Ty)->getName() == "wasm.funcref",
1044 "wasm.funcref");
1045 case IITDescriptor::Vector: {
1046 VectorType *VT = dyn_cast<VectorType>(Val: Ty);
1047 StringRef Scalable = D.VectorWidth.isScalable() ? "vscale " : "";
1048 bool HasError =
1049 PrintMsg(VT && VT->getElementCount() == D.VectorWidth,
1050 Twine(Scalable) + "vector with " +
1051 Twine(D.VectorWidth.getKnownMinValue()) + " elements");
1052 if (HasError)
1053 return true;
1054 Position.push_vector_element();
1055 return matchIntrinsicType(Ty: VT->getElementType(), Infos, Position,
1056 OverloadTys, DeferredChecks, IsDeferredCheck, OS);
1057 }
1058 case IITDescriptor::Pointer: {
1059 PointerType *PT = dyn_cast<PointerType>(Val: Ty);
1060 unsigned AS = D.PointerAddressSpace;
1061 bool IsValid = PT && PT->getAddressSpace() == AS;
1062 if (AS == 0)
1063 return PrintMsg(IsValid, "ptr");
1064 return PrintMsg(IsValid, "ptr addrspace(" + Twine(AS) + ")");
1065 }
1066
1067 case IITDescriptor::Struct: {
1068 StructType *ST = dyn_cast<StructType>(Val: Ty);
1069 unsigned EC = D.StructNumElements;
1070 bool HasError = PrintMsg(
1071 ST && ST->isLiteral() && !ST->isPacked() && ST->getNumElements() == EC,
1072 "literal non-packed struct with " + Twine(EC) + " elements");
1073 if (HasError)
1074 return true;
1075
1076 for (const auto &[Idx, ETy] : llvm::enumerate(First: ST->elements())) {
1077 Position.push_struct_element(ElementNum: Idx);
1078 if (matchIntrinsicType(Ty: ETy, Infos, Position, OverloadTys, DeferredChecks,
1079 IsDeferredCheck, OS))
1080 return true;
1081 Position.pop_index();
1082 }
1083 return false;
1084 }
1085
1086 case IITDescriptor::Overloaded: {
1087 unsigned OIdx = D.getOverloadIndex();
1088 assert(OIdx == OverloadTys.size() && !IsDeferredCheck &&
1089 "Table consistency error");
1090 OverloadTys.push_back(Elt: Ty);
1091
1092 IITDescriptor::AnyKindVectorConstraint VC;
1093 IITDescriptor::AnyKindElementConstraint EC;
1094 std::tie(args&: VC, args&: EC) = D.getOverloadConstraints();
1095
1096 bool IsValid = [&]() {
1097 switch (VC) {
1098 case IITDescriptor::VC_None:
1099 return true;
1100 case IITDescriptor::VC_Vector:
1101 return isa<VectorType>(Val: Ty);
1102 case IITDescriptor::VC_Scalar:
1103 return !isa<VectorType>(Val: Ty);
1104 }
1105 llvm_unreachable("invalid vector constraint");
1106 }();
1107
1108 IsValid &= [&]() {
1109 Type *ETy = Ty->getScalarType();
1110 switch (EC) {
1111 case IITDescriptor::EC_None:
1112 return true;
1113 case IITDescriptor::EC_Integer:
1114 return ETy->isIntegerTy();
1115 case IITDescriptor::EC_Float:
1116 return ETy->isFloatingPointTy();
1117 case IITDescriptor::EC_Pointer:
1118 return ETy->isPointerTy();
1119 }
1120 llvm_unreachable("invalid element constraint");
1121 }();
1122
1123 if (IsValid)
1124 return false;
1125
1126 static constexpr StringLiteral VectorKinds[] = {
1127 "",
1128 "vector",
1129 "scalar",
1130 };
1131 static constexpr StringLiteral ElementKinds[] = {
1132 "",
1133 "integer",
1134 "fp",
1135 "pointer",
1136 };
1137
1138 if (EC == IITDescriptor::EC_None) {
1139 // No constraint on element type.
1140 // Expected = any {vector | scalar} type.
1141 StringLiteral VK = ArrayRef(VectorKinds)[VC];
1142 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: VK), OIdx);
1143 }
1144
1145 StringLiteral EK = ArrayRef(ElementKinds)[EC];
1146 switch (VC) {
1147 case IITDescriptor::VC_None:
1148 // Expected = any EK or EK vector.
1149 return PrintMsg(false, formatv(Fmt: "any {0} or {0} vector", Vals&: EK), OIdx);
1150 case IITDescriptor::VC_Vector:
1151 return PrintMsg(false, formatv(Fmt: "any {} vector", Vals&: EK), OIdx);
1152 case IITDescriptor::VC_Scalar:
1153 return PrintMsg(false, formatv(Fmt: "any {} type", Vals&: EK), OIdx);
1154 }
1155 llvm_unreachable("invalid vector constraint");
1156 }
1157
1158 case IITDescriptor::Match: {
1159 unsigned OIdx = D.getOverloadIndex();
1160 if (OIdx >= OverloadTys.size())
1161 return IsDeferredCheck || DeferCheck(Ty);
1162 return PrintMsgInvalidDepType(Ty == OverloadTys[OIdx], "matching",
1163 formatv(Fmt: "{}", Vals&: *OverloadTys[OIdx]), OIdx);
1164 }
1165
1166 case IITDescriptor::Extend:
1167 case IITDescriptor::Trunc: {
1168 unsigned OIdx = D.getOverloadIndex();
1169 // If this is a forward reference, defer the check for later.
1170 if (OIdx >= OverloadTys.size())
1171 return IsDeferredCheck || DeferCheck(Ty);
1172
1173 Type *OTy = OverloadTys[OIdx];
1174 bool IsExtend = D.Kind == IITDescriptor::Extend;
1175 StringRef Qualifier = IsExtend ? "extended" : "truncated";
1176 if (!OTy->isIntOrIntVectorTy())
1177 return PrintMsgInvalidOverloadTy(Qualifier, "int or vector of int", OIdx);
1178
1179 Type *NewTy = IsExtend ? OTy->getExtendedType() : OTy->getTruncatedType();
1180 return PrintMsgInvalidDepType(Ty == NewTy, Qualifier, formatv(Fmt: "{}", Vals&: *NewTy),
1181 OIdx);
1182 }
1183 case IITDescriptor::OneNthEltsVec: {
1184 unsigned OIdx = D.getOverloadIndex();
1185 unsigned Divisor = D.getVectorDivisor();
1186 // If this is a forward reference, defer the check for later.
1187 if (OIdx >= OverloadTys.size())
1188 return IsDeferredCheck || DeferCheck(Ty);
1189 Type *OTy = OverloadTys[OIdx];
1190 auto *OVecTy = dyn_cast<VectorType>(Val: OTy);
1191 auto Qualifier = formatv(Fmt: "1/nth (n={}) elements vector of", Vals&: Divisor);
1192 if (!OVecTy)
1193 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1194 if (!OVecTy->getElementCount().isKnownMultipleOf(RHS: Divisor))
1195 return PrintMsgInvalidOverloadTy(
1196 Qualifier, formatv(Fmt: "vector with multiple of {} elements", Vals&: Divisor),
1197 OIdx);
1198 Type *Expected = VectorType::getOneNthElementsVectorType(VTy: OVecTy, Denominator: Divisor);
1199 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1200 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1201 }
1202 case IITDescriptor::SameVecWidth: {
1203 unsigned OIdx = D.getOverloadIndex();
1204 if (OIdx >= OverloadTys.size()) {
1205 // Defer check and subsequent check for the vector element type.
1206 Infos.consume_front();
1207 return IsDeferredCheck || DeferCheck(Ty);
1208 }
1209 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1210 auto *ThisArgVecType = dyn_cast<VectorType>(Val: Ty);
1211 // Both must be vectors of the same number of elements or neither.
1212 StringRef Qualifier = "same vector width of";
1213 if (OVecTy && !ThisArgVecType)
1214 return PrintMsgInvalidDepType(false, Qualifier, "vector", OIdx);
1215 if (!OVecTy && ThisArgVecType)
1216 return PrintMsgInvalidDepType(false, Qualifier, "scalar", OIdx);
1217 Type *EltTy = Ty;
1218 if (ThisArgVecType) {
1219 ElementCount Expected = OVecTy->getElementCount();
1220 if (Expected != ThisArgVecType->getElementCount())
1221 return PrintMsgInvalidDepType(
1222 false, Qualifier, formatv(Fmt: "vector with {} elements", Vals&: Expected),
1223 OIdx);
1224 EltTy = ThisArgVecType->getElementType();
1225 Position.push_vector_element();
1226 }
1227 return matchIntrinsicType(Ty: EltTy, Infos, Position, OverloadTys,
1228 DeferredChecks, IsDeferredCheck, OS);
1229 }
1230 case IITDescriptor::VecOfAnyPtrsToElt: {
1231 unsigned RefOverloadIndex = D.getRefOverloadIndex();
1232 if (RefOverloadIndex >= OverloadTys.size()) {
1233 if (IsDeferredCheck)
1234 return true;
1235 // If forward referencing, already add the pointer-vector type and
1236 // defer the checks for later.
1237 assert(D.getOverloadIndex() == OverloadTys.size() &&
1238 "Table consistency error");
1239 OverloadTys.push_back(Elt: Ty);
1240 return DeferCheck(Ty);
1241 }
1242
1243 if (!IsDeferredCheck) {
1244 assert(D.getOverloadIndex() == OverloadTys.size() &&
1245 "Table consistency error");
1246 OverloadTys.push_back(Elt: Ty);
1247 }
1248
1249 // Verify the overloaded type "matches" the Ref type.
1250 // i.e. Ty is a vector with the same width as Ref and composed of pointers.
1251
1252 StringRef Qualifier = "vector of pointers to elements of";
1253 auto *ReferenceType = dyn_cast<VectorType>(Val: OverloadTys[RefOverloadIndex]);
1254 if (!ReferenceType)
1255 return PrintMsgInvalidOverloadTy(Qualifier, "vector", RefOverloadIndex);
1256
1257 auto *ThisArgVecTy = dyn_cast<VectorType>(Val: Ty);
1258 if (!ThisArgVecTy)
1259 return PrintMsgInvalidDepType(false, Qualifier, "vector",
1260 RefOverloadIndex);
1261
1262 auto ExpectedCount = ReferenceType->getElementCount();
1263 auto Expected =
1264 formatv(Fmt: "vector of pointers with {} elements", Vals&: ExpectedCount);
1265 bool IsValid = ThisArgVecTy->getElementCount() == ExpectedCount &&
1266 ThisArgVecTy->getElementType()->isPointerTy();
1267 return PrintMsgInvalidDepType(IsValid, Qualifier, Expected,
1268 RefOverloadIndex);
1269 }
1270 case IITDescriptor::VecElement: {
1271 unsigned OIdx = D.getOverloadIndex();
1272 if (OIdx >= OverloadTys.size())
1273 return IsDeferredCheck || DeferCheck(Ty);
1274 StringRef Qualifier = "vector element of";
1275 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1276 if (!OVecTy)
1277 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1278 Type *Expected = OVecTy->getElementType();
1279 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1280 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1281 }
1282 case IITDescriptor::Subdivide2:
1283 case IITDescriptor::Subdivide4: {
1284 unsigned OIdx = D.getOverloadIndex();
1285 // If this is a forward reference, defer the check for later.
1286 if (OIdx >= OverloadTys.size())
1287 return IsDeferredCheck || DeferCheck(Ty);
1288
1289 int SubDivs = D.Kind == IITDescriptor::Subdivide2 ? 1 : 2;
1290 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1291 auto Qualifier =
1292 formatv(Fmt: "subdivided by {} vector of", Vals: SubDivs == 1 ? 2 : 4);
1293 if (!OVecTy)
1294 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1295
1296 // TODO: Verify that the element type of the overload type is subdivisible
1297 // by 2 or 4.
1298 Type *Expected = VectorType::getSubdividedVectorType(VTy: OVecTy, NumSubdivs: SubDivs);
1299 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1300 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1301 }
1302 case IITDescriptor::VecOfBitcastsToInt: {
1303 unsigned OIdx = D.getOverloadIndex();
1304 if (OIdx >= OverloadTys.size())
1305 return IsDeferredCheck || DeferCheck(Ty);
1306 auto *OVecTy = dyn_cast<VectorType>(Val: OverloadTys[OIdx]);
1307 StringRef Qualifier = "vector of bitcasts to int of";
1308 if (!OVecTy)
1309 return PrintMsgInvalidOverloadTy(Qualifier, "vector", OIdx);
1310 Type *Expected = VectorType::getInteger(VTy: OVecTy);
1311 return PrintMsgInvalidDepType(Expected == Ty, Qualifier,
1312 formatv(Fmt: "{}", Vals&: *Expected), OIdx);
1313 }
1314 case IITDescriptor::VarArg:
1315 // VarArg token should be consumed by `getIntrinsicInfoTableEntries`, so we
1316 // should never see it here.
1317 llvm_unreachable("IITDescriptor::VarArg not expected");
1318 }
1319 llvm_unreachable("unhandled");
1320}
1321
1322/// Return true if the function type \p FTy is a valid type signature for the
1323/// type constraints specified in the .td file, represented by \p Infos and
1324/// \p IsVarArg. The overloaded types for the intrinsic are pushed to the
1325/// \p OverloadTys vector.
1326///
1327/// If the type is not valid, returns false and prints an error message to
1328/// \p OS.
1329static bool isSignatureValid(FunctionType *FTy,
1330 ArrayRef<Intrinsic::IITDescriptor> &Infos,
1331 unsigned NumArgs, bool IsVarArg,
1332 SmallVectorImpl<Type *> &OverloadTys,
1333 raw_ostream &OS) {
1334 SmallVector<DeferredIntrinsicMatchInfo, 2> DeferredChecks;
1335
1336 assert(!Infos.empty() && "Table consistency error");
1337
1338 MatchPosition Pos;
1339 Pos.IsRet = true;
1340 Pos.Num = 0;
1341
1342 if (matchIntrinsicType(Ty: FTy->getReturnType(), Infos, Position: Pos, OverloadTys,
1343 DeferredChecks, IsDeferredCheck: false, OS))
1344 return false;
1345
1346 if (FTy->getNumParams() != NumArgs) {
1347 OS << "intrinsic has incorrect number of args. Expected " << NumArgs
1348 << ", but got " << FTy->getNumParams();
1349 return false;
1350 }
1351
1352 Pos.IsRet = false;
1353 for (const auto &[Idx, Ty] : llvm::enumerate(First: FTy->params())) {
1354 Pos.Num = Idx;
1355 if (matchIntrinsicType(Ty, Infos, Position: Pos, OverloadTys, DeferredChecks, IsDeferredCheck: false,
1356 OS))
1357 return false;
1358 }
1359
1360 for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) {
1361 auto &[DefTy, DefInfos, DefPosition] = DeferredChecks[I];
1362 if (matchIntrinsicType(Ty: DefTy, Infos&: DefInfos, Position: DefPosition, OverloadTys,
1363 DeferredChecks, IsDeferredCheck: true, OS))
1364 return false;
1365 }
1366
1367 if (!Infos.empty()) {
1368 OS << "intrinsic has too few arguments!";
1369 return false;
1370 }
1371
1372 if (FTy->isVarArg() != IsVarArg) {
1373 if (IsVarArg)
1374 OS << "intrinsic was not defined with variable arguments!";
1375 else
1376 OS << "intrinsic was defined with variable arguments!";
1377 return false;
1378 }
1379
1380 return true;
1381}
1382
1383bool Intrinsic::hasStructReturnType(ID id) {
1384 using namespace Intrinsic;
1385 SmallVector<IITDescriptor> Table;
1386 getIntrinsicInfoTableEntries(id, T&: Table);
1387 return !Table.empty() && Table[0].Kind == IITDescriptor::Struct;
1388}
1389
1390bool Intrinsic::isSignatureValid(Intrinsic::ID ID, FunctionType *FT,
1391 SmallVectorImpl<Type *> &OverloadTys,
1392 raw_ostream &OS) {
1393 if (!ID)
1394 return false;
1395
1396 SmallVector<Intrinsic::IITDescriptor, 8> Table;
1397 auto [TableRef, NumArgs, IsVarArg] = getIntrinsicInfoTableEntries(id: ID, T&: Table);
1398
1399 return ::isSignatureValid(FTy: FT, Infos&: TableRef, NumArgs, IsVarArg, OverloadTys, OS);
1400}
1401
1402bool Intrinsic::isSignatureValid(Function *F,
1403 SmallVectorImpl<Type *> &OverloadTys,
1404 raw_ostream &OS) {
1405 return isSignatureValid(ID: F->getIntrinsicID(), FT: F->getFunctionType(),
1406 OverloadTys, OS);
1407}
1408
1409std::optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) {
1410 SmallVector<Type *, 4> OverloadTys;
1411 if (!isSignatureValid(F, OverloadTys))
1412 return std::nullopt;
1413
1414 Intrinsic::ID ID = F->getIntrinsicID();
1415 StringRef Name = F->getName();
1416 std::string WantedName =
1417 Intrinsic::getName(Id: ID, OverloadTys, M: F->getParent(), FT: F->getFunctionType());
1418 if (Name == WantedName)
1419 return std::nullopt;
1420
1421 Function *NewDecl = [&] {
1422 if (auto *ExistingGV = F->getParent()->getNamedValue(Name: WantedName)) {
1423 if (auto *ExistingF = dyn_cast<Function>(Val: ExistingGV))
1424 if (ExistingF->getFunctionType() == F->getFunctionType())
1425 return ExistingF;
1426
1427 // The name already exists, but is not a function or has the wrong
1428 // prototype. Make place for the new one by renaming the old version.
1429 // Either this old version will be removed later on or the module is
1430 // invalid and we'll get an error.
1431 ExistingGV->setName(WantedName + ".renamed");
1432 }
1433 return Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, OverloadTys);
1434 }();
1435
1436 NewDecl->setCallingConv(F->getCallingConv());
1437 assert(NewDecl->getFunctionType() == F->getFunctionType() &&
1438 "Shouldn't change the signature");
1439 return NewDecl;
1440}
1441
1442struct InterleaveIntrinsic {
1443 Intrinsic::ID Interleave, Deinterleave;
1444};
1445
1446static InterleaveIntrinsic InterleaveIntrinsics[] = {
1447 {.Interleave: Intrinsic::vector_interleave2, .Deinterleave: Intrinsic::vector_deinterleave2},
1448 {.Interleave: Intrinsic::vector_interleave3, .Deinterleave: Intrinsic::vector_deinterleave3},
1449 {.Interleave: Intrinsic::vector_interleave4, .Deinterleave: Intrinsic::vector_deinterleave4},
1450 {.Interleave: Intrinsic::vector_interleave5, .Deinterleave: Intrinsic::vector_deinterleave5},
1451 {.Interleave: Intrinsic::vector_interleave6, .Deinterleave: Intrinsic::vector_deinterleave6},
1452 {.Interleave: Intrinsic::vector_interleave7, .Deinterleave: Intrinsic::vector_deinterleave7},
1453 {.Interleave: Intrinsic::vector_interleave8, .Deinterleave: Intrinsic::vector_deinterleave8},
1454};
1455
1456Intrinsic::ID Intrinsic::getInterleaveIntrinsicID(unsigned Factor) {
1457 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1458 return InterleaveIntrinsics[Factor - 2].Interleave;
1459}
1460
1461Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
1462 assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
1463 return InterleaveIntrinsics[Factor - 2].Deinterleave;
1464}
1465
1466#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
1467#include "llvm/IR/IntrinsicImpl.inc"
1468