1//===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend emits information about intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenIntrinsics.h"
14#include "SequenceToOffsetTable.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Support/CommandLine.h"
20#include "llvm/Support/ErrorHandling.h"
21#include "llvm/Support/FormatVariadic.h"
22#include "llvm/Support/ModRef.h"
23#include "llvm/Support/SourceMgr.h"
24#include "llvm/Support/raw_ostream.h"
25#include "llvm/TableGen/Error.h"
26#include "llvm/TableGen/Record.h"
27#include "llvm/TableGen/StringToOffsetTable.h"
28#include "llvm/TableGen/TableGenBackend.h"
29#include <algorithm>
30#include <array>
31#include <cassert>
32#include <cctype>
33#include <map>
34#include <optional>
35#include <string>
36#include <utility>
37#include <vector>
38using namespace llvm;
39
40static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
41static cl::opt<std::string>
42 IntrinsicPrefix("intrinsic-prefix",
43 cl::desc("Generate intrinsics with this target prefix"),
44 cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat));
45
46namespace {
47class IntrinsicEmitter {
48 const RecordKeeper &Records;
49
50public:
51 IntrinsicEmitter(const RecordKeeper &R) : Records(R) {}
52
53 void run(raw_ostream &OS, bool Enums);
54
55 void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
56 void EmitArgKind(raw_ostream &OS);
57 void EmitIITInfo(raw_ostream &OS);
58 void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
59 void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints,
60 raw_ostream &OS);
61 void EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable &Ints,
62 raw_ostream &OS);
63 void EmitIntrinsicToPrettyPrintTable(const CodeGenIntrinsicTable &Ints,
64 raw_ostream &OS);
65 void EmitIntrinsicBitTable(
66 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,
67 StringRef TableName, StringRef Comment,
68 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty);
69 void EmitGenerator(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
70 void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
71 void EmitPrettyPrintArguments(const CodeGenIntrinsicTable &Ints,
72 raw_ostream &OS);
73 void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints,
74 bool IsClang, raw_ostream &OS);
75};
76
77// Helper class to use with `TableGen::Emitter::OptClass`.
78template <bool Enums> class IntrinsicEmitterOpt : public IntrinsicEmitter {
79public:
80 IntrinsicEmitterOpt(const RecordKeeper &R) : IntrinsicEmitter(R) {}
81 void run(raw_ostream &OS) { IntrinsicEmitter::run(OS, Enums); }
82};
83
84} // End anonymous namespace
85
86//===----------------------------------------------------------------------===//
87// IntrinsicEmitter Implementation
88//===----------------------------------------------------------------------===//
89
90void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {
91 emitSourceFileHeader(Desc: "Intrinsic Function Source Fragment", OS);
92
93 CodeGenIntrinsicTable Ints(Records);
94
95 if (Enums) {
96 // Emit the enum information.
97 EmitEnumInfo(Ints, OS);
98
99 // Emit ArgKind for Intrinsics.h.
100 EmitArgKind(OS);
101 } else {
102 // Emit IIT_Info constants.
103 EmitIITInfo(OS);
104
105 // Emit the target metadata.
106 EmitTargetInfo(Ints, OS);
107
108 // Emit the intrinsic ID -> name table.
109 EmitIntrinsicToNameTable(Ints, OS);
110
111 // Emit the intrinsic ID -> overload table.
112 EmitIntrinsicToOverloadTable(Ints, OS);
113
114 // Emit the intrinsic declaration generator.
115 EmitGenerator(Ints, OS);
116
117 // Emit the intrinsic parameter attributes.
118 EmitAttributes(Ints, OS);
119
120 // Emit the intrinsic ID -> pretty print table.
121 EmitIntrinsicToPrettyPrintTable(Ints, OS);
122
123 // Emit Pretty Print attribute.
124 EmitPrettyPrintArguments(Ints, OS);
125
126 // Emit code to translate Clang builtins into LLVM intrinsics.
127 EmitIntrinsicToBuiltinMap(Ints, IsClang: true, OS);
128
129 // Emit code to translate MS builtins into LLVM intrinsics.
130 EmitIntrinsicToBuiltinMap(Ints, IsClang: false, OS);
131 }
132}
133
134void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
135 raw_ostream &OS) {
136 // Find the TargetSet for which to generate enums. There will be an initial
137 // set with an empty target prefix which will include target independent
138 // intrinsics like dbg.value.
139 using TargetSet = CodeGenIntrinsicTable::TargetSet;
140 const TargetSet *Set = nullptr;
141 for (const auto &Target : Ints.getTargets()) {
142 if (Target.Name == IntrinsicPrefix) {
143 Set = &Target;
144 break;
145 }
146 }
147 if (!Set) {
148 // The first entry is for target independent intrinsics, so drop it.
149 auto KnowTargets = Ints.getTargets().drop_front();
150 PrintFatalError(PrintMsg: [KnowTargets](raw_ostream &OS) {
151 OS << "tried to generate intrinsics for unknown target "
152 << IntrinsicPrefix << "\nKnown targets are: ";
153 interleaveComma(c: KnowTargets, os&: OS,
154 each_fn: [&OS](const TargetSet &Target) { OS << Target.Name; });
155 OS << '\n';
156 });
157 }
158
159 // Generate a complete header for target specific intrinsics.
160 if (IntrinsicPrefix.empty()) {
161 OS << "#ifdef GET_INTRINSIC_ENUM_VALUES\n";
162 } else {
163 std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
164 OS << formatv(Fmt: "#ifndef LLVM_IR_INTRINSIC_{}_ENUMS_H\n", Vals&: UpperPrefix);
165 OS << formatv(Fmt: "#define LLVM_IR_INTRINSIC_{}_ENUMS_H\n", Vals&: UpperPrefix);
166 OS << "namespace llvm::Intrinsic {\n";
167 OS << formatv(Fmt: "enum {}Intrinsics : unsigned {{\n", Vals&: UpperPrefix);
168 }
169
170 OS << "// Enum values for intrinsics.\n";
171 bool First = true;
172 for (const CodeGenIntrinsic &Int : Ints[*Set]) {
173 OS << " " << Int.EnumName;
174
175 // Assign a value to the first intrinsic in this target set so that all
176 // intrinsic ids are distinct.
177 if (First) {
178 OS << " = " << Set->Offset + 1;
179 First = false;
180 }
181
182 OS << ", ";
183 if (Int.EnumName.size() < 40)
184 OS.indent(NumSpaces: 40 - Int.EnumName.size());
185 OS << formatv(
186 Fmt: " // {} ({})\n", Vals: Int.Name,
187 Vals: SrcMgr.getFormattedLocationNoOffset(Loc: Int.TheDef->getLoc().front()));
188 }
189
190 // Emit num_intrinsics into the target neutral enum.
191 if (IntrinsicPrefix.empty()) {
192 OS << formatv(Fmt: " num_intrinsics = {}\n", Vals: Ints.size() + 1);
193 OS << "#endif\n\n";
194 } else {
195 OS << R"(}; // enum
196} // namespace llvm::Intrinsic
197#endif
198
199)";
200 }
201}
202
203void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {
204 if (!IntrinsicPrefix.empty())
205 return;
206 OS << "// llvm::Intrinsic::IITDescriptor::ArgKind.\n";
207 OS << "#ifdef GET_INTRINSIC_ARGKIND\n";
208 if (const auto RecArgKind = Records.getDef(Name: "ArgKind")) {
209 for (const auto &RV : RecArgKind->getValues())
210 OS << " AK_" << RV.getName() << " = " << *RV.getValue() << ",\n";
211 } else {
212 OS << "#error \"ArgKind is not defined\"\n";
213 }
214 OS << "#endif\n\n";
215}
216
217void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {
218 OS << "#ifdef GET_INTRINSIC_IITINFO\n";
219 std::array<StringRef, 256> RecsByNumber;
220 auto IIT_Base = Records.getAllDerivedDefinitionsIfDefined(ClassName: "IIT_Base");
221 for (const Record *Rec : IIT_Base) {
222 auto Number = Rec->getValueAsInt(FieldName: "Number");
223 assert(0 <= Number && Number < (int)RecsByNumber.size() &&
224 "IIT_Info.Number should be uint8_t");
225 assert(RecsByNumber[Number].empty() && "Duplicate IIT_Info.Number");
226 RecsByNumber[Number] = Rec->getName();
227 }
228 if (IIT_Base.size() > 0) {
229 for (unsigned I = 0, E = RecsByNumber.size(); I < E; ++I)
230 if (!RecsByNumber[I].empty())
231 OS << " " << RecsByNumber[I] << " = " << I << ",\n";
232 } else {
233 OS << "#error \"class IIT_Base is not defined\"\n";
234 }
235 OS << "#endif\n\n";
236}
237
238void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
239 raw_ostream &OS) {
240 OS << R"(// Target mapping.
241#ifdef GET_INTRINSIC_TARGET_DATA
242struct IntrinsicTargetInfo {
243 StringLiteral Name;
244 size_t Offset;
245 size_t Count;
246};
247static constexpr IntrinsicTargetInfo TargetInfos[] = {
248)";
249 for (const auto [Name, Offset, Count] : Ints.getTargets())
250 OS << formatv(Fmt: " {{\"{}\", {}, {}},\n", Vals: Name, Vals: Offset, Vals: Count);
251 OS << R"(};
252#endif
253
254)";
255}
256
257/// Helper function to emit a bit table for intrinsic properties.
258/// This is used for both overload and pretty print bit tables.
259void IntrinsicEmitter::EmitIntrinsicBitTable(
260 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,
261 StringRef TableName, StringRef Comment,
262 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty) {
263 OS << formatv(Fmt: "// {}\n", Vals&: Comment);
264 OS << formatv(Fmt: "#ifdef {}\n", Vals&: Guard);
265 OS << formatv(Fmt: "static constexpr uint8_t {}[] = {{\n", Vals&: TableName);
266 OS << " 0\n ";
267 for (auto [I, Int] : enumerate(First: Ints)) {
268 // Add one to the index so we emit a null bit for the invalid #0 intrinsic.
269 size_t Idx = I + 1;
270 if (Idx % 8 == 0)
271 OS << ",\n 0";
272 if (GetProperty(Int))
273 OS << " | (1<<" << Idx % 8 << ')';
274 }
275 OS << "\n};\n\n";
276 OS << formatv(Fmt: "return ({}[id/8] & (1 << (id%8))) != 0;\n", Vals&: TableName);
277 OS << formatv(Fmt: "#endif // {}\n\n", Vals&: Guard);
278}
279
280void IntrinsicEmitter::EmitIntrinsicToNameTable(
281 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
282 // Built up a table of the intrinsic names.
283 constexpr StringLiteral NotIntrinsic = "not_intrinsic";
284 StringToOffsetTable Table;
285 Table.GetOrAddStringOffset(Str: NotIntrinsic);
286 for (const auto &Int : Ints)
287 Table.GetOrAddStringOffset(Str: Int.Name);
288
289 OS << R"(// Intrinsic ID to name table.
290#ifdef GET_INTRINSIC_NAME_TABLE
291// Note that entry #0 is the invalid intrinsic!
292
293)";
294
295 Table.EmitStringTableDef(OS, Name: "IntrinsicNameTable");
296
297 OS << R"(
298static constexpr unsigned IntrinsicNameOffsetTable[] = {
299)";
300
301 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: NotIntrinsic),
302 Vals: NotIntrinsic);
303 for (const auto &Int : Ints)
304 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: Int.Name), Vals: Int.Name);
305
306 OS << R"(
307}; // IntrinsicNameOffsetTable
308
309#endif
310
311)";
312}
313
314void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
315 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
316 EmitIntrinsicBitTable(
317 Ints, OS, Guard: "GET_INTRINSIC_OVERLOAD_TABLE", TableName: "OTable",
318 Comment: "Intrinsic ID to overload bitset.",
319 GetProperty: [](const CodeGenIntrinsic &Int) { return Int.isOverloaded; });
320}
321
322using TypeSigTy = SmallVector<unsigned char>;
323
324/// Computes type signature of the intrinsic \p Int.
325static TypeSigTy ComputeTypeSignature(const CodeGenIntrinsic &Int) {
326 TypeSigTy TypeSig;
327 const Record *TypeInfo = Int.TheDef->getValueAsDef(FieldName: "TypeInfo");
328 const ListInit *TypeList = TypeInfo->getValueAsListInit(FieldName: "TypeSig");
329
330 for (const auto *TypeListEntry : TypeList->getElements())
331 TypeSig.emplace_back(Args: cast<IntInit>(Val: TypeListEntry)->getValue());
332 return TypeSig;
333}
334
335// Pack the type signature into 32-bit fixed encoding word.
336static std::optional<uint32_t> encodePacked(const TypeSigTy &TypeSig) {
337 if (TypeSig.size() > 8)
338 return std::nullopt;
339
340 uint32_t Result = 0;
341 for (unsigned char C : reverse(C: TypeSig)) {
342 if (C > 15)
343 return std::nullopt;
344 Result = (Result << 4) | C;
345 }
346 return Result;
347}
348
349void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
350 raw_ostream &OS) {
351 // Note: the code below can be switched to use 32-bit fixed encoding by
352 // flipping the flag below.
353 constexpr bool Use16BitFixedEncoding = true;
354 using FixedEncodingTy =
355 std::conditional_t<Use16BitFixedEncoding, uint16_t, uint32_t>;
356 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
357 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
358 // Mask with all bits 1 except the most significant bit.
359 constexpr unsigned Mask = (1U << MSBPosition) - 1;
360 StringRef FixedEncodingTypeName =
361 Use16BitFixedEncoding ? "uint16_t" : "uint32_t";
362
363 // If we can compute a 16/32-bit fixed encoding for this intrinsic, do so and
364 // capture it in this vector, otherwise store a ~0U.
365 std::vector<FixedEncodingTy> FixedEncodings;
366 SequenceToOffsetTable<TypeSigTy> LongEncodingTable;
367
368 FixedEncodings.reserve(n: Ints.size());
369
370 // Compute the unique argument type info.
371 for (const CodeGenIntrinsic &Int : Ints) {
372 // Get the signature for the intrinsic.
373 TypeSigTy TypeSig = ComputeTypeSignature(Int);
374
375 // Check to see if we can encode it into a 16/32 bit word.
376 std::optional<uint32_t> Result = encodePacked(TypeSig);
377 if (Result && (*Result & Mask) == *Result) {
378 FixedEncodings.push_back(x: static_cast<FixedEncodingTy>(*Result));
379 continue;
380 }
381
382 LongEncodingTable.add(Seq: TypeSig);
383
384 // This is a placehold that we'll replace after the table is laid out.
385 FixedEncodings.push_back(x: static_cast<FixedEncodingTy>(~0U));
386 }
387
388 LongEncodingTable.layout();
389
390 OS << formatv(Fmt: R"(// Global intrinsic function declaration type table.
391#ifdef GET_INTRINSIC_GENERATOR_GLOBAL
392using FixedEncodingTy = {};
393static constexpr FixedEncodingTy IIT_Table[] = {{
394 )",
395 Vals&: FixedEncodingTypeName);
396
397 unsigned MaxOffset = 0;
398 for (auto [Idx, FixedEncoding, Int] : enumerate(First&: FixedEncodings, Rest: Ints)) {
399 if ((Idx & 7) == 7)
400 OS << "\n ";
401
402 // If the entry fit in the table, just emit it.
403 if ((FixedEncoding & Mask) == FixedEncoding) {
404 OS << "0x" << Twine::utohexstr(Val: FixedEncoding) << ", ";
405 continue;
406 }
407
408 TypeSigTy TypeSig = ComputeTypeSignature(Int);
409 unsigned Offset = LongEncodingTable.get(Seq: TypeSig);
410 MaxOffset = std::max(a: MaxOffset, b: Offset);
411
412 // Otherwise, emit the offset into the long encoding table. We emit it this
413 // way so that it is easier to read the offset in the .def file.
414 OS << formatv(Fmt: "(1U<<{}) | {}, ", Vals: MSBPosition, Vals&: Offset);
415 }
416
417 OS << "0\n};\n\n";
418
419 // verify that all offsets will fit in 16/32 bits.
420 if ((MaxOffset & Mask) != MaxOffset)
421 PrintFatalError(Msg: "Offset of long encoding table exceeds encoding bits");
422
423 // Emit the shared table of register lists.
424 OS << "static constexpr unsigned char IIT_LongEncodingTable[] = {\n";
425 if (!LongEncodingTable.empty())
426 LongEncodingTable.emit(
427 OS, Print: [](raw_ostream &OS, unsigned char C) { OS << (unsigned)C; });
428 OS << " 255\n};\n";
429 OS << "#endif\n\n"; // End of GET_INTRINSIC_GENERATOR_GLOBAL
430}
431
432/// Returns the effective MemoryEffects for intrinsic \p Int.
433static MemoryEffects getEffectiveME(const CodeGenIntrinsic &Int) {
434 MemoryEffects ME = Int.ME;
435 // TODO: IntrHasSideEffects should affect not only readnone intrinsics.
436 if (ME.doesNotAccessMemory() && Int.hasSideEffects)
437 ME = MemoryEffects::unknown();
438 return ME;
439}
440
441static bool compareFnAttributes(const CodeGenIntrinsic *L,
442 const CodeGenIntrinsic *R) {
443 auto TieBoolAttributes = [](const CodeGenIntrinsic *I) -> auto {
444 // Sort throwing intrinsics after non-throwing intrinsics.
445 return std::tie(args: I->canThrow, args: I->isNoDuplicate, args: I->isNoMerge, args: I->isNoReturn,
446 args: I->isNoCallback, args: I->isNoSync, args: I->isNoFree, args: I->isWillReturn,
447 args: I->isCold, args: I->isConvergent, args: I->isSpeculatable,
448 args: I->hasSideEffects, args: I->isStrictFP,
449 args: I->isNoCreateUndefOrPoison);
450 };
451
452 auto TieL = TieBoolAttributes(L);
453 auto TieR = TieBoolAttributes(R);
454
455 if (TieL != TieR)
456 return TieL < TieR;
457
458 // Try to order by readonly/readnone attribute.
459 uint32_t LME = getEffectiveME(Int: *L).toIntValue();
460 uint32_t RME = getEffectiveME(Int: *R).toIntValue();
461 if (LME != RME)
462 return LME > RME;
463
464 return false;
465}
466
467/// Returns true if \p Int has a non-empty set of function attributes. Note that
468/// NoUnwind = !canThrow, so we need to negate it's sense to test if the
469// intrinsic has NoUnwind attribute.
470static bool hasFnAttributes(const CodeGenIntrinsic &Int) {
471 return !Int.canThrow || Int.isNoReturn || Int.isNoCallback || Int.isNoSync ||
472 Int.isNoFree || Int.isWillReturn || Int.isCold || Int.isNoDuplicate ||
473 Int.isNoMerge || Int.isConvergent || Int.isSpeculatable ||
474 Int.isStrictFP || Int.isNoCreateUndefOrPoison ||
475 getEffectiveME(Int) != MemoryEffects::unknown();
476}
477
478namespace {
479struct FnAttributeComparator {
480 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
481 return compareFnAttributes(L, R);
482 }
483};
484
485struct AttributeComparator {
486 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
487 // This comparator is used to unique just the argument attributes of an
488 // intrinsic without considering any function attributes.
489 return L->ArgumentAttributes < R->ArgumentAttributes;
490 }
491};
492} // End anonymous namespace
493
494/// Returns the name of the IR enum for argument attribute kind \p Kind.
495static StringRef getArgAttrEnumName(CodeGenIntrinsic::ArgAttrKind Kind) {
496 switch (Kind) {
497 case CodeGenIntrinsic::NoCapture:
498 llvm_unreachable("Handled separately");
499 case CodeGenIntrinsic::NoAlias:
500 return "NoAlias";
501 case CodeGenIntrinsic::NoUndef:
502 return "NoUndef";
503 case CodeGenIntrinsic::NonNull:
504 return "NonNull";
505 case CodeGenIntrinsic::Returned:
506 return "Returned";
507 case CodeGenIntrinsic::ReadOnly:
508 return "ReadOnly";
509 case CodeGenIntrinsic::WriteOnly:
510 return "WriteOnly";
511 case CodeGenIntrinsic::ReadNone:
512 return "ReadNone";
513 case CodeGenIntrinsic::ImmArg:
514 return "ImmArg";
515 case CodeGenIntrinsic::Alignment:
516 return "Alignment";
517 case CodeGenIntrinsic::Dereferenceable:
518 return "Dereferenceable";
519 case CodeGenIntrinsic::Range:
520 return "Range";
521 }
522 llvm_unreachable("Unknown CodeGenIntrinsic::ArgAttrKind enum");
523}
524
525/// EmitAttributes - This emits the Intrinsic::getAttributes method.
526void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
527 raw_ostream &OS) {
528 OS << R"(// Add parameter attributes that are not common to all intrinsics.
529#ifdef GET_INTRINSIC_ATTRIBUTES
530static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID,
531 Type *ArgType) {
532 unsigned BitWidth = ArgType->getScalarSizeInBits();
533 switch (ID) {
534 default: llvm_unreachable("Invalid attribute set number");)";
535 // Compute unique argument attribute sets.
536 std::map<SmallVector<CodeGenIntrinsic::ArgAttribute, 0>, unsigned>
537 UniqArgAttributes;
538 for (const CodeGenIntrinsic &Int : Ints) {
539 for (auto &Attrs : Int.ArgumentAttributes) {
540 if (Attrs.empty())
541 continue;
542
543 unsigned ID = UniqArgAttributes.size();
544 if (!UniqArgAttributes.try_emplace(k: Attrs, args&: ID).second)
545 continue;
546
547 assert(is_sorted(Attrs) && "Argument attributes are not sorted");
548
549 OS << formatv(Fmt: R"(
550 case {}:
551 return AttributeSet::get(C, {{
552)",
553 Vals&: ID);
554 for (const CodeGenIntrinsic::ArgAttribute &Attr : Attrs) {
555 if (Attr.Kind == CodeGenIntrinsic::NoCapture) {
556 OS << " Attribute::getWithCaptureInfo(C, "
557 "CaptureInfo::none()),\n";
558 continue;
559 }
560 StringRef AttrName = getArgAttrEnumName(Kind: Attr.Kind);
561 if (Attr.Kind == CodeGenIntrinsic::Alignment ||
562 Attr.Kind == CodeGenIntrinsic::Dereferenceable)
563 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}, {}),\n",
564 Vals&: AttrName, Vals: Attr.Value);
565 else if (Attr.Kind == CodeGenIntrinsic::Range)
566 // This allows implicitTrunc because the range may only fit the
567 // type based on rules implemented in the IR verifier. E.g. the
568 // [-1, 1] range for ucmp/scmp intrinsics requires a minimum i2 type.
569 // Give the verifier a chance to diagnose this instead of asserting
570 // here.
571 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}, "
572 "ConstantRange(APInt(BitWidth, {}, /*isSigned=*/true, "
573 "/*implicitTrunc=*/true), APInt(BitWidth, {}, "
574 "/*isSigned=*/true, /*implicitTrunc=*/true))),\n",
575 Vals&: AttrName, Vals: (int64_t)Attr.Value, Vals: (int64_t)Attr.Value2);
576 else
577 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}),\n", Vals&: AttrName);
578 }
579 OS << " });";
580 }
581 }
582 OS << R"(
583 }
584} // getIntrinsicArgAttributeSet
585)";
586
587 // Compute unique function attribute sets. Note that ID 255 will be used for
588 // intrinsics with no function attributes.
589 std::map<const CodeGenIntrinsic *, unsigned, FnAttributeComparator>
590 UniqFnAttributes;
591 OS << R"(
592static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
593 switch (ID) {
594 default: llvm_unreachable("Invalid attribute set number");)";
595
596 for (const CodeGenIntrinsic &Int : Ints) {
597 if (!hasFnAttributes(Int))
598 continue;
599 unsigned ID = UniqFnAttributes.size();
600 if (!UniqFnAttributes.try_emplace(k: &Int, args&: ID).second)
601 continue;
602 OS << formatv(Fmt: R"(
603 case {}: // {}
604 return AttributeSet::get(C, {{
605)",
606 Vals&: ID, Vals: Int.Name);
607 auto addAttribute = [&OS](StringRef Attr) {
608 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}),\n", Vals&: Attr);
609 };
610 if (!Int.canThrow)
611 addAttribute("NoUnwind");
612 if (Int.isNoReturn)
613 addAttribute("NoReturn");
614 if (Int.isNoCallback)
615 addAttribute("NoCallback");
616 if (Int.isNoSync)
617 addAttribute("NoSync");
618 if (Int.isNoFree)
619 addAttribute("NoFree");
620 if (Int.isWillReturn)
621 addAttribute("WillReturn");
622 if (Int.isCold)
623 addAttribute("Cold");
624 if (Int.isNoDuplicate)
625 addAttribute("NoDuplicate");
626 if (Int.isNoMerge)
627 addAttribute("NoMerge");
628 if (Int.isConvergent)
629 addAttribute("Convergent");
630 if (Int.isSpeculatable)
631 addAttribute("Speculatable");
632 if (Int.isStrictFP)
633 addAttribute("StrictFP");
634 if (Int.isNoCreateUndefOrPoison)
635 addAttribute("NoCreateUndefOrPoison");
636
637 const MemoryEffects ME = getEffectiveME(Int);
638 if (ME != MemoryEffects::unknown()) {
639 OS << formatv(Fmt: " // {}\n", Vals: ME);
640 OS << formatv(Fmt: " Attribute::getWithMemoryEffects(C, "
641 "MemoryEffects::createFromIntValue({})),\n",
642 Vals: ME.toIntValue());
643 }
644 OS << " });";
645 }
646 OS << R"(
647 }
648} // getIntrinsicFnAttributeSet)";
649
650 // Compute unique argument attributes.
651 std::map<const CodeGenIntrinsic *, unsigned, AttributeComparator>
652 UniqAttributes;
653 for (const CodeGenIntrinsic &Int : Ints) {
654 unsigned ID = UniqAttributes.size();
655 UniqAttributes.try_emplace(k: &Int, args&: ID);
656 }
657
658 const uint8_t UniqAttributesBitSize = Log2_32_Ceil(Value: UniqAttributes.size());
659 // Note, max value is used to indicate no function attributes.
660 const uint8_t UniqFnAttributesBitSize =
661 Log2_32_Ceil(Value: UniqFnAttributes.size() + 1);
662 const uint32_t NoFunctionAttrsID =
663 maskTrailingOnes<uint32_t>(N: UniqFnAttributesBitSize);
664 uint8_t AttributesMapDataBitSize =
665 PowerOf2Ceil(A: UniqAttributesBitSize + UniqFnAttributesBitSize);
666 if (AttributesMapDataBitSize < 8)
667 AttributesMapDataBitSize = 8;
668 else if (AttributesMapDataBitSize > 64)
669 PrintFatalError(Msg: "Packed ID of IntrinsicsToAttributesMap exceeds 64b!");
670 else if (AttributesMapDataBitSize > 16)
671 PrintWarning(Msg: "Packed ID of IntrinsicsToAttributesMap exceeds 16b, "
672 "this may cause performance drop (pr106809), "
673 "please consider redesigning intrinsic sets!");
674
675 // Assign a packed ID for each intrinsic. The lower bits will be its
676 // "argument attribute ID" (index in UniqAttributes) and upper bits will be
677 // its "function attribute ID" (index in UniqFnAttributes).
678 OS << formatv(Fmt: "\nstatic constexpr uint{}_t IntrinsicsToAttributesMap[] = {{",
679 Vals&: AttributesMapDataBitSize);
680 for (const CodeGenIntrinsic &Int : Ints) {
681 uint32_t FnAttrIndex =
682 hasFnAttributes(Int) ? UniqFnAttributes[&Int] : NoFunctionAttrsID;
683 OS << formatv(Fmt: "\n {} << {} | {}, // {}", Vals&: FnAttrIndex,
684 Vals: UniqAttributesBitSize, Vals&: UniqAttributes[&Int], Vals: Int.Name);
685 }
686
687 OS << R"(
688}; // IntrinsicsToAttributesMap
689)";
690
691 // For a given intrinsic, its attributes are constructed by populating the
692 // local array `AS` below with its non-empty argument attributes followed by
693 // function attributes if any. Each argument attribute is constructed as:
694 //
695 // getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo));
696 //
697 // Create a table that records, for each argument attributes, the list of
698 // <ArgNo, ArgAttrID> pairs that are needed to construct its argument
699 // attributes. These tables for all intrinsics will be concatenated into one
700 // large table and then for each intrinsic, we remember the Staring index and
701 // number of size of its slice of entries (i.e., number of arguments with
702 // non-empty attributes), so that we can build the attribute list for an
703 // intrinsic without using a switch-case.
704
705 using ArgNoAttrIDPair = std::pair<uint16_t, uint16_t>;
706
707 // Emit the table of concatenated <ArgNo, AttrId> using SequenceToOffsetTable
708 // so that entries can be reused if possible. Individual sequences in this
709 // table do not have any terminator.
710 using ArgAttrIDSubTable = SmallVector<ArgNoAttrIDPair>;
711 SequenceToOffsetTable<ArgAttrIDSubTable> ArgAttrIdSequenceTable(std::nullopt);
712 SmallVector<ArgAttrIDSubTable> ArgAttrIdSubTables(
713 UniqAttributes.size()); // Indexed by UniqueID.
714
715 // Find the max number of attributes to create the local array.
716 unsigned MaxNumAttrs = 0;
717 for (const auto [IntPtr, UniqueID] : UniqAttributes) {
718 const CodeGenIntrinsic &Int = *IntPtr;
719 ArgAttrIDSubTable SubTable;
720
721 for (const auto &[ArgNo, Attrs] : enumerate(First: Int.ArgumentAttributes)) {
722 if (Attrs.empty())
723 continue;
724
725 uint16_t ArgAttrID = UniqArgAttributes.find(x: Attrs)->second;
726 SubTable.emplace_back(Args: (uint16_t)ArgNo, Args&: ArgAttrID);
727 }
728 ArgAttrIdSubTables[UniqueID] = SubTable;
729 if (!SubTable.empty())
730 ArgAttrIdSequenceTable.add(Seq: SubTable);
731 unsigned NumAttrs = SubTable.size() + hasFnAttributes(Int);
732 MaxNumAttrs = std::max(a: MaxNumAttrs, b: NumAttrs);
733 }
734
735 ArgAttrIdSequenceTable.layout();
736
737 if (ArgAttrIdSequenceTable.size() >= std::numeric_limits<uint16_t>::max())
738 PrintFatalError(Msg: "Size of ArgAttrIdTable exceeds supported limit");
739
740 // Emit the 2 tables (flattened ArgNo, ArgAttrID) and ArgAttributesInfoTable.
741 OS << formatv(Fmt: R"(
742namespace {{
743struct ArgNoAttrIDPair {{
744 uint16_t ArgNo, ArgAttrID;
745};
746} // namespace
747
748// Number of entries: {}
749static constexpr ArgNoAttrIDPair ArgAttrIdTable[] = {{
750)",
751 Vals: ArgAttrIdSequenceTable.size());
752
753 ArgAttrIdSequenceTable.emit(OS, Print: [](raw_ostream &OS, ArgNoAttrIDPair Elem) {
754 OS << formatv(Fmt: "{{{}, {}}", Vals&: Elem.first, Vals&: Elem.second);
755 });
756
757 OS << formatv(Fmt: R"(}; // ArgAttrIdTable
758
759namespace {{
760struct ArgAttributesInfo {{
761 uint16_t StartIndex;
762 uint16_t NumAttrs;
763};
764} // namespace
765
766// Number of entries: {}
767static constexpr ArgAttributesInfo ArgAttributesInfoTable[] = {{
768)",
769 Vals: ArgAttrIdSubTables.size());
770
771 for (const auto &SubTable : ArgAttrIdSubTables) {
772 unsigned NumAttrs = SubTable.size();
773 unsigned StartIndex = NumAttrs ? ArgAttrIdSequenceTable.get(Seq: SubTable) : 0;
774 OS << formatv(Fmt: " {{{}, {}},\n", Vals&: StartIndex, Vals&: NumAttrs);
775 }
776 OS << "}; // ArgAttributesInfoTable\n";
777
778 // Now emit the Intrinsic::getAttributes function. This will first map
779 // from intrinsic ID -> unique arg/function attr ID (using the
780 // IntrinsicsToAttributesMap) table. Then it will use the unique arg ID to
781 // construct all the argument attributes (using the ArgAttributesInfoTable and
782 // ArgAttrIdTable) and then add on the function attributes if any.
783 OS << formatv(Fmt: R"(
784
785template <typename IDTy>
786inline std::pair<uint32_t, uint32_t> unpackID(const IDTy PackedID) {{
787 constexpr uint8_t UniqAttributesBitSize = {};
788 const uint32_t FnAttrID = PackedID >> UniqAttributesBitSize;
789 const uint32_t ArgAttrID = PackedID &
790 maskTrailingOnes<uint32_t>(UniqAttributesBitSize);
791 return {{FnAttrID, ArgAttrID};
792}
793
794AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id,
795 FunctionType *FT) {{
796 if (id == 0)
797 return AttributeList();
798 auto [FnAttrID, ArgAttrID] = unpackID(IntrinsicsToAttributesMap[id - 1]);
799 using PairTy = std::pair<unsigned, AttributeSet>;
800 alignas(PairTy) char ASStorage[sizeof(PairTy) * {}];
801 PairTy *AS = reinterpret_cast<PairTy *>(ASStorage);
802
803 // Construct an ArrayRef for easier range checking.
804 ArrayRef<ArgAttributesInfo> ArgAttributesInfoTableAR(ArgAttributesInfoTable);
805 if (ArgAttrID >= ArgAttributesInfoTableAR.size())
806 llvm_unreachable("Invalid arguments attribute ID");
807
808 auto [StartIndex, NumAttrs] = ArgAttributesInfoTableAR[ArgAttrID];
809 for (unsigned Idx = 0; Idx < NumAttrs; ++Idx) {{
810 auto [ArgNo, ArgAttrID] = ArgAttrIdTable[StartIndex + Idx];
811 AS[Idx] = {{ArgNo,
812 getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo))};
813 }
814 if (FnAttrID != {}) {
815 AS[NumAttrs++] = {{AttributeList::FunctionIndex,
816 getIntrinsicFnAttributeSet(C, FnAttrID)};
817 }
818 return AttributeList::get(C, ArrayRef(AS, NumAttrs));
819}
820
821AttributeSet Intrinsic::getFnAttributes(LLVMContext &C, ID id) {{
822 if (id == 0)
823 return AttributeSet();
824 auto [FnAttrID, _] = unpackID(IntrinsicsToAttributesMap[id - 1]);
825 if (FnAttrID == {})
826 return AttributeSet();
827 return getIntrinsicFnAttributeSet(C, FnAttrID);
828}
829#endif // GET_INTRINSIC_ATTRIBUTES
830
831)",
832 Vals: UniqAttributesBitSize, Vals&: MaxNumAttrs, Vals: NoFunctionAttrsID,
833 Vals: NoFunctionAttrsID);
834}
835
836void IntrinsicEmitter::EmitIntrinsicToPrettyPrintTable(
837 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
838 EmitIntrinsicBitTable(Ints, OS, Guard: "GET_INTRINSIC_PRETTY_PRINT_TABLE", TableName: "PPTable",
839 Comment: "Intrinsic ID to pretty print bitset.",
840 GetProperty: [](const CodeGenIntrinsic &Int) {
841 return !Int.PrettyPrintFunctions.empty();
842 });
843}
844
845void IntrinsicEmitter::EmitPrettyPrintArguments(
846 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
847 OS << R"(
848#ifdef GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
849void Intrinsic::printImmArg(ID IID, unsigned ArgIdx, raw_ostream &OS, const Constant *ImmArgVal) {
850 using namespace Intrinsic;
851 switch (IID) {
852)";
853
854 for (const CodeGenIntrinsic &Int : Ints) {
855 if (Int.PrettyPrintFunctions.empty())
856 continue;
857
858 OS << " case " << Int.EnumName << ":\n";
859 OS << " switch (ArgIdx) {\n";
860 for (const auto [ArgIdx, ArgName, FuncName] : Int.PrettyPrintFunctions) {
861 OS << " case " << ArgIdx << ":\n";
862 OS << " OS << \"" << ArgName << "=\";\n";
863 if (!FuncName.empty()) {
864 OS << " ";
865 if (!Int.TargetPrefix.empty())
866 OS << Int.TargetPrefix << "::";
867 OS << FuncName << "(OS, ImmArgVal);\n";
868 }
869 OS << " return;\n";
870 }
871 OS << " }\n";
872 OS << " break;\n";
873 }
874 OS << R"( default:
875 break;
876 }
877}
878#endif // GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
879)";
880}
881
882void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
883 const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) {
884 StringRef CompilerName = IsClang ? "Clang" : "MS";
885 StringRef UpperCompilerName = IsClang ? "CLANG" : "MS";
886
887 // map<TargetPrefix, pair<map<BuiltinName, EnumName>, CommonPrefix>.
888 // Note that we iterate over both the maps in the code below and both
889 // iterations need to iterate in sorted key order. For the inner map, entries
890 // need to be emitted in the sorted order of `BuiltinName` with `CommonPrefix`
891 // rempved, because we use std::lower_bound to search these entries. For the
892 // outer map as well, entries need to be emitted in sorter order of
893 // `TargetPrefix` as we use std::lower_bound to search these entries.
894 using BIMEntryTy =
895 std::pair<std::map<StringRef, StringRef>, std::optional<StringRef>>;
896 std::map<StringRef, BIMEntryTy> BuiltinMap;
897
898 for (const CodeGenIntrinsic &Int : Ints) {
899 StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName;
900 if (BuiltinName.empty())
901 continue;
902 // Get the map for this target prefix.
903 auto &[Map, CommonPrefix] = BuiltinMap[Int.TargetPrefix];
904
905 if (!Map.try_emplace(k: BuiltinName, args: Int.EnumName).second)
906 PrintFatalError(ErrorLoc: Int.TheDef->getLoc(),
907 Msg: "Intrinsic '" + Int.TheDef->getName() + "': duplicate " +
908 CompilerName + " builtin name!");
909
910 // Update common prefix.
911 if (!CommonPrefix) {
912 // For the first builtin for this target, initialize the common prefix.
913 CommonPrefix = BuiltinName;
914 continue;
915 }
916
917 // Update the common prefix. Note that this assumes that `take_front` will
918 // never set the `Data` pointer in CommonPrefix to nullptr.
919 const char *Mismatch = mismatch(Range1&: *CommonPrefix, Range2&: BuiltinName).first;
920 *CommonPrefix = CommonPrefix->take_front(N: Mismatch - CommonPrefix->begin());
921 }
922
923 // Populate the string table with the names of all the builtins after
924 // removing this common prefix.
925 StringToOffsetTable Table;
926 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
927 auto &[Map, CommonPrefix] = Entry;
928 for (auto &[BuiltinName, EnumName] : Map) {
929 StringRef Suffix = BuiltinName.substr(Start: CommonPrefix->size());
930 Table.GetOrAddStringOffset(Str: Suffix);
931 }
932 }
933
934 OS << formatv(Fmt: R"(
935// Get the LLVM intrinsic that corresponds to a builtin. This is used by the
936// C front-end. The builtin name is passed in as BuiltinName, and a target
937// prefix (e.g. 'ppc') is passed in as TargetPrefix.
938#ifdef GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
939Intrinsic::ID
940Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
941 StringRef BuiltinName) {{
942 using namespace Intrinsic;
943)",
944 Vals&: UpperCompilerName, Vals&: CompilerName);
945
946 if (BuiltinMap.empty()) {
947 OS << formatv(Fmt: R"(
948 return not_intrinsic;
949 }
950#endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
951)",
952 Vals&: UpperCompilerName);
953 return;
954 }
955
956 if (!Table.empty()) {
957 Table.EmitStringTableDef(OS, Name: "BuiltinNames");
958
959 OS << R"(
960 struct BuiltinEntry {
961 ID IntrinsicID;
962 unsigned StrTabOffset;
963 const char *getName() const { return BuiltinNames[StrTabOffset].data(); }
964 bool operator<(StringRef RHS) const {
965 return strncmp(getName(), RHS.data(), RHS.size()) < 0;
966 }
967 };
968
969)";
970 }
971
972 // Emit a per target table of bultin names.
973 bool HasTargetIndependentBuiltins = false;
974 StringRef TargetIndepndentCommonPrefix;
975 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
976 const auto &[Map, CommonPrefix] = Entry;
977 if (!TargetPrefix.empty()) {
978 OS << formatv(Fmt: " // Builtins for {0}.\n", Vals: TargetPrefix);
979 } else {
980 OS << " // Target independent builtins.\n";
981 HasTargetIndependentBuiltins = true;
982 TargetIndepndentCommonPrefix = *CommonPrefix;
983 }
984
985 // Emit the builtin table for this target prefix.
986 OS << formatv(Fmt: " static constexpr BuiltinEntry {}Names[] = {{\n",
987 Vals: TargetPrefix);
988 for (const auto &[BuiltinName, EnumName] : Map) {
989 StringRef Suffix = BuiltinName.substr(Start: CommonPrefix->size());
990 OS << formatv(Fmt: " {{{}, {}}, // {}\n", Vals: EnumName,
991 Vals: *Table.GetStringOffset(Str: Suffix), Vals: BuiltinName);
992 }
993 OS << formatv(Fmt: " }; // {}Names\n\n", Vals: TargetPrefix);
994 }
995
996 // After emitting the builtin tables for all targets, emit a lookup table for
997 // all targets. We will use binary search, similar to the table for builtin
998 // names to lookup into this table.
999 OS << R"(
1000 struct TargetEntry {
1001 StringLiteral TargetPrefix;
1002 ArrayRef<BuiltinEntry> Names;
1003 StringLiteral CommonPrefix;
1004 bool operator<(StringRef RHS) const {
1005 return TargetPrefix < RHS;
1006 };
1007 };
1008 static constexpr TargetEntry TargetTable[] = {
1009)";
1010
1011 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
1012 const auto &[Map, CommonPrefix] = Entry;
1013 if (TargetPrefix.empty())
1014 continue;
1015 OS << formatv(Fmt: R"( {{"{0}", {0}Names, "{1}"},)", Vals: TargetPrefix,
1016 Vals: CommonPrefix)
1017 << "\n";
1018 }
1019 OS << " };\n";
1020
1021 // Now for the actual lookup, first check the target independent table if
1022 // we emitted one.
1023 if (HasTargetIndependentBuiltins) {
1024 OS << formatv(Fmt: R"(
1025 // Check if it's a target independent builtin.
1026 // Copy the builtin name so we can use it in consume_front without clobbering
1027 // if for the lookup in the target specific table.
1028 StringRef Suffix = BuiltinName;
1029 if (Suffix.consume_front("{}")) {{
1030 auto II = lower_bound(Names, Suffix);
1031 if (II != std::end(Names) && II->getName() == Suffix)
1032 return II->IntrinsicID;
1033 }
1034)",
1035 Vals&: TargetIndepndentCommonPrefix);
1036 }
1037
1038 // If a target independent builtin was not found, lookup the target specific.
1039 OS << formatv(Fmt: R"(
1040 auto TI = lower_bound(TargetTable, TargetPrefix);
1041 if (TI == std::end(TargetTable) || TI->TargetPrefix != TargetPrefix)
1042 return not_intrinsic;
1043 // This is the last use of BuiltinName, so no need to copy before using it in
1044 // consume_front.
1045 if (!BuiltinName.consume_front(TI->CommonPrefix))
1046 return not_intrinsic;
1047 auto II = lower_bound(TI->Names, BuiltinName);
1048 if (II == std::end(TI->Names) || II->getName() != BuiltinName)
1049 return not_intrinsic;
1050 return II->IntrinsicID;
1051}
1052#endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
1053
1054)",
1055 Vals&: UpperCompilerName);
1056}
1057
1058static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/true>>
1059 X("gen-intrinsic-enums", "Generate intrinsic enums");
1060
1061static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/false>>
1062 Y("gen-intrinsic-impl", "Generate intrinsic implementation code");
1063