1//===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend emits information about intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenIntrinsics.h"
14#include "SequenceToOffsetTable.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Support/CommandLine.h"
20#include "llvm/Support/ErrorHandling.h"
21#include "llvm/Support/FormatVariadic.h"
22#include "llvm/Support/ModRef.h"
23#include "llvm/Support/SourceMgr.h"
24#include "llvm/Support/raw_ostream.h"
25#include "llvm/TableGen/CodeGenHelpers.h"
26#include "llvm/TableGen/Error.h"
27#include "llvm/TableGen/Record.h"
28#include "llvm/TableGen/StringToOffsetTable.h"
29#include "llvm/TableGen/TableGenBackend.h"
30#include <algorithm>
31#include <array>
32#include <cassert>
33#include <cctype>
34#include <map>
35#include <optional>
36#include <string>
37#include <utility>
38#include <vector>
39using namespace llvm;
40
41static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
42static cl::opt<std::string>
43 IntrinsicPrefix("intrinsic-prefix",
44 cl::desc("Generate intrinsics with this target prefix"),
45 cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat));
46
47namespace {
48class IntrinsicEmitter {
49 const RecordKeeper &Records;
50
51public:
52 IntrinsicEmitter(const RecordKeeper &R) : Records(R) {}
53
54 void run(raw_ostream &OS, bool Enums);
55
56 void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
57 void EmitArgKind(raw_ostream &OS);
58 void EmitIITInfo(raw_ostream &OS);
59 void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
60 void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints,
61 raw_ostream &OS);
62 void EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable &Ints,
63 raw_ostream &OS);
64 void EmitIntrinsicToPrettyPrintTable(const CodeGenIntrinsicTable &Ints,
65 raw_ostream &OS);
66 void EmitIntrinsicBitTable(
67 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,
68 StringRef TableName, StringRef Comment,
69 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty);
70 void EmitGenerator(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
71 void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
72 void EmitPrettyPrintArguments(const CodeGenIntrinsicTable &Ints,
73 raw_ostream &OS);
74 void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints,
75 bool IsClang, raw_ostream &OS);
76};
77
78// Helper class to use with `TableGen::Emitter::OptClass`.
79template <bool Enums> class IntrinsicEmitterOpt : public IntrinsicEmitter {
80public:
81 IntrinsicEmitterOpt(const RecordKeeper &R) : IntrinsicEmitter(R) {}
82 void run(raw_ostream &OS) { IntrinsicEmitter::run(OS, Enums); }
83};
84
85} // End anonymous namespace
86
87//===----------------------------------------------------------------------===//
88// IntrinsicEmitter Implementation
89//===----------------------------------------------------------------------===//
90
91void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {
92 emitSourceFileHeader(Desc: "Intrinsic Function Source Fragment", OS);
93
94 CodeGenIntrinsicTable Ints(Records);
95
96 if (Enums) {
97 // Emit the enum information.
98 EmitEnumInfo(Ints, OS);
99
100 // Emit ArgKind for Intrinsics.h.
101 EmitArgKind(OS);
102 } else {
103 // Emit IIT_Info constants.
104 EmitIITInfo(OS);
105
106 // Emit the target metadata.
107 EmitTargetInfo(Ints, OS);
108
109 // Emit the intrinsic ID -> name table.
110 EmitIntrinsicToNameTable(Ints, OS);
111
112 // Emit the intrinsic ID -> overload table.
113 EmitIntrinsicToOverloadTable(Ints, OS);
114
115 // Emit the intrinsic declaration generator.
116 EmitGenerator(Ints, OS);
117
118 // Emit the intrinsic parameter attributes.
119 EmitAttributes(Ints, OS);
120
121 // Emit the intrinsic ID -> pretty print table.
122 EmitIntrinsicToPrettyPrintTable(Ints, OS);
123
124 // Emit Pretty Print attribute.
125 EmitPrettyPrintArguments(Ints, OS);
126
127 // Emit code to translate Clang builtins into LLVM intrinsics.
128 EmitIntrinsicToBuiltinMap(Ints, IsClang: true, OS);
129
130 // Emit code to translate MS builtins into LLVM intrinsics.
131 EmitIntrinsicToBuiltinMap(Ints, IsClang: false, OS);
132 }
133}
134
135void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
136 raw_ostream &OS) {
137 // Find the TargetSet for which to generate enums. There will be an initial
138 // set with an empty target prefix which will include target independent
139 // intrinsics like dbg.value.
140 using TargetSet = CodeGenIntrinsicTable::TargetSet;
141 const TargetSet *Set = nullptr;
142 for (const auto &Target : Ints.getTargets()) {
143 if (Target.Name == IntrinsicPrefix) {
144 Set = &Target;
145 break;
146 }
147 }
148 if (!Set) {
149 // The first entry is for target independent intrinsics, so drop it.
150 auto KnowTargets = Ints.getTargets().drop_front();
151 PrintFatalError(PrintMsg: [KnowTargets](raw_ostream &OS) {
152 OS << "tried to generate intrinsics for unknown target "
153 << IntrinsicPrefix << "\nKnown targets are: ";
154 interleaveComma(c: KnowTargets, os&: OS,
155 each_fn: [&OS](const TargetSet &Target) { OS << Target.Name; });
156 OS << '\n';
157 });
158 }
159
160 // Generate a complete header for target specific intrinsics.
161 std::optional<IfDefEmitter> IfDef;
162 std::optional<IncludeGuardEmitter> IncGuard;
163 std::optional<NamespaceEmitter> NS;
164
165 if (IntrinsicPrefix.empty()) {
166 IfDef.emplace(args&: OS, args: "GET_INTRINSIC_ENUM_VALUES");
167 } else {
168 std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
169 IncGuard.emplace(
170 args&: OS, args: formatv(Fmt: "LLVM_IR_INTRINSIC_{}_ENUMS_H", Vals&: UpperPrefix).str());
171 NS.emplace(args&: OS, args: "llvm::Intrinsic");
172 OS << formatv(Fmt: "enum {}Intrinsics : unsigned {{\n", Vals&: UpperPrefix);
173 }
174
175 OS << "// Enum values for intrinsics.\n";
176 bool First = true;
177 for (const CodeGenIntrinsic &Int : Ints[*Set]) {
178 OS << " " << Int.EnumName;
179
180 // Assign a value to the first intrinsic in this target set so that all
181 // intrinsic ids are distinct.
182 if (First) {
183 OS << " = " << Set->Offset + 1;
184 First = false;
185 }
186
187 OS << ", ";
188 if (Int.EnumName.size() < 40)
189 OS.indent(NumSpaces: 40 - Int.EnumName.size());
190 OS << formatv(
191 Fmt: " // {} ({})\n", Vals: Int.Name,
192 Vals: SrcMgr.getFormattedLocationNoOffset(Loc: Int.TheDef->getLoc().front()));
193 }
194
195 // Emit num_intrinsics into the target neutral enum.
196 if (IntrinsicPrefix.empty())
197 OS << formatv(Fmt: " num_intrinsics = {}\n", Vals: Ints.size() + 1);
198 else
199 OS << "}; // enum\n";
200}
201
202void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {
203 if (!IntrinsicPrefix.empty())
204 return;
205 IfDefEmitter IfDef(OS, "GET_INTRINSIC_ARGKIND");
206 OS << "// llvm::Intrinsic::IITDescriptor::ArgKind.\n";
207 if (const auto RecArgKind = Records.getDef(Name: "ArgKind")) {
208 for (const auto &RV : RecArgKind->getValues())
209 OS << " AK_" << RV.getName() << " = " << *RV.getValue() << ",\n";
210 } else {
211 OS << "#error \"ArgKind is not defined\"\n";
212 }
213}
214
215void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {
216 IfDefEmitter IfDef(OS, "GET_INTRINSIC_IITINFO");
217 std::array<StringRef, 256> RecsByNumber;
218 auto IIT_Base = Records.getAllDerivedDefinitionsIfDefined(ClassName: "IIT_Base");
219 for (const Record *Rec : IIT_Base) {
220 auto Number = Rec->getValueAsInt(FieldName: "Number");
221 assert(0 <= Number && Number < (int)RecsByNumber.size() &&
222 "IIT_Info.Number should be uint8_t");
223 assert(RecsByNumber[Number].empty() && "Duplicate IIT_Info.Number");
224 RecsByNumber[Number] = Rec->getName();
225 }
226 if (IIT_Base.size() > 0) {
227 for (unsigned I = 0, E = RecsByNumber.size(); I < E; ++I)
228 if (!RecsByNumber[I].empty())
229 OS << " " << RecsByNumber[I] << " = " << I << ",\n";
230 } else {
231 OS << "#error \"class IIT_Base is not defined\"\n";
232 }
233}
234
235void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
236 raw_ostream &OS) {
237 IfDefEmitter IfDef(OS, "GET_INTRINSIC_TARGET_DATA");
238 OS << R"(// Target mapping.
239struct IntrinsicTargetInfo {
240 StringLiteral Name;
241 size_t Offset;
242 size_t Count;
243};
244static constexpr IntrinsicTargetInfo TargetInfos[] = {
245)";
246 for (const auto [Name, Offset, Count] : Ints.getTargets())
247 OS << formatv(Fmt: " {{\"{}\", {}, {}},\n", Vals: Name, Vals: Offset, Vals: Count);
248 OS << "};\n";
249}
250
251/// Helper function to emit a bit table for intrinsic properties.
252/// This is used for both overload and pretty print bit tables.
253void IntrinsicEmitter::EmitIntrinsicBitTable(
254 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,
255 StringRef TableName, StringRef Comment,
256 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty) {
257 IfDefEmitter IfDef(OS, Guard);
258 OS << formatv(Fmt: "// {}\n", Vals&: Comment);
259 OS << formatv(Fmt: "static constexpr uint8_t {}[] = {{\n", Vals&: TableName);
260 OS << " 0\n ";
261 for (auto [I, Int] : enumerate(First: Ints)) {
262 // Add one to the index so we emit a null bit for the invalid #0 intrinsic.
263 size_t Idx = I + 1;
264 if (Idx % 8 == 0)
265 OS << ",\n 0";
266 if (GetProperty(Int))
267 OS << " | (1<<" << Idx % 8 << ')';
268 }
269 OS << "\n};\n\n";
270 OS << formatv(Fmt: "return ({}[id/8] & (1 << (id%8))) != 0;\n", Vals&: TableName);
271}
272
273void IntrinsicEmitter::EmitIntrinsicToNameTable(
274 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
275 // Built up a table of the intrinsic names.
276 constexpr StringLiteral NotIntrinsic = "not_intrinsic";
277 StringToOffsetTable Table;
278 Table.GetOrAddStringOffset(Str: NotIntrinsic);
279 for (const auto &Int : Ints)
280 Table.GetOrAddStringOffset(Str: Int.Name);
281
282 IfDefEmitter IfDef(OS, "GET_INTRINSIC_NAME_TABLE");
283 OS << R"(// Intrinsic ID to name table.
284// Note that entry #0 is the invalid intrinsic!
285
286)";
287
288 Table.EmitStringTableDef(OS, Name: "IntrinsicNameTable");
289
290 OS << R"(
291static constexpr unsigned IntrinsicNameOffsetTable[] = {
292)";
293
294 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: NotIntrinsic),
295 Vals: NotIntrinsic);
296 for (const auto &Int : Ints)
297 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: Int.Name), Vals: Int.Name);
298
299 OS << "\n}; // IntrinsicNameOffsetTable\n";
300}
301
302void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
303 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
304 EmitIntrinsicBitTable(
305 Ints, OS, Guard: "GET_INTRINSIC_OVERLOAD_TABLE", TableName: "OTable",
306 Comment: "Intrinsic ID to overload bitset.",
307 GetProperty: [](const CodeGenIntrinsic &Int) { return Int.isOverloaded; });
308}
309
310using TypeSigTy = SmallVector<unsigned char>;
311
312/// Computes type signature of the intrinsic \p Int.
313static TypeSigTy ComputeTypeSignature(const CodeGenIntrinsic &Int) {
314 TypeSigTy TypeSig;
315 const Record *TypeInfo = Int.TheDef->getValueAsDef(FieldName: "TypeInfo");
316 const ListInit *TypeList = TypeInfo->getValueAsListInit(FieldName: "TypeSig");
317
318 for (const auto *TypeListEntry : TypeList->getElements())
319 TypeSig.emplace_back(Args: cast<IntInit>(Val: TypeListEntry)->getValue());
320 return TypeSig;
321}
322
323// Pack the type signature into 32-bit fixed encoding word.
324static std::optional<uint32_t> encodePacked(const TypeSigTy &TypeSig) {
325 if (TypeSig.size() > 8)
326 return std::nullopt;
327
328 uint32_t Result = 0;
329 for (unsigned char C : reverse(C: TypeSig)) {
330 if (C > 15)
331 return std::nullopt;
332 Result = (Result << 4) | C;
333 }
334 return Result;
335}
336
337void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
338 raw_ostream &OS) {
339 // Note: the code below can be switched to use 32-bit fixed encoding by
340 // flipping the flag below.
341 constexpr bool Use16BitFixedEncoding = true;
342 using FixedEncodingTy =
343 std::conditional_t<Use16BitFixedEncoding, uint16_t, uint32_t>;
344 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
345 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
346 // Mask with all bits 1 except the most significant bit.
347 constexpr unsigned Mask = (1U << MSBPosition) - 1;
348 StringRef FixedEncodingTypeName =
349 Use16BitFixedEncoding ? "uint16_t" : "uint32_t";
350
351 // If we can compute a 16/32-bit fixed encoding for this intrinsic, do so and
352 // capture it in this vector, otherwise store a ~0U.
353 std::vector<FixedEncodingTy> FixedEncodings;
354 SequenceToOffsetTable<TypeSigTy> LongEncodingTable;
355
356 FixedEncodings.reserve(n: Ints.size());
357
358 // Compute the unique argument type info.
359 for (const CodeGenIntrinsic &Int : Ints) {
360 // Get the signature for the intrinsic.
361 TypeSigTy TypeSig = ComputeTypeSignature(Int);
362
363 // Check to see if we can encode it into a 16/32 bit word.
364 std::optional<uint32_t> Result = encodePacked(TypeSig);
365 if (Result && (*Result & Mask) == *Result) {
366 FixedEncodings.push_back(x: static_cast<FixedEncodingTy>(*Result));
367 continue;
368 }
369
370 LongEncodingTable.add(Seq: TypeSig);
371
372 // This is a placehold that we'll replace after the table is laid out.
373 FixedEncodings.push_back(x: static_cast<FixedEncodingTy>(~0U));
374 }
375
376 LongEncodingTable.layout();
377
378 IfDefEmitter IfDef(OS, "GET_INTRINSIC_GENERATOR_GLOBAL");
379 OS << formatv(Fmt: R"(// Global intrinsic function declaration type table.
380using FixedEncodingTy = {};
381static constexpr FixedEncodingTy IIT_Table[] = {{
382 )",
383 Vals&: FixedEncodingTypeName);
384
385 unsigned MaxOffset = 0;
386 for (auto [Idx, FixedEncoding, Int] : enumerate(First&: FixedEncodings, Rest: Ints)) {
387 if ((Idx & 7) == 7)
388 OS << "\n ";
389
390 // If the entry fit in the table, just emit it.
391 if ((FixedEncoding & Mask) == FixedEncoding) {
392 OS << "0x" << Twine::utohexstr(Val: FixedEncoding) << ", ";
393 continue;
394 }
395
396 TypeSigTy TypeSig = ComputeTypeSignature(Int);
397 unsigned Offset = LongEncodingTable.get(Seq: TypeSig);
398 MaxOffset = std::max(a: MaxOffset, b: Offset);
399
400 // Otherwise, emit the offset into the long encoding table. We emit it this
401 // way so that it is easier to read the offset in the .def file.
402 OS << formatv(Fmt: "(1U<<{}) | {}, ", Vals: MSBPosition, Vals&: Offset);
403 }
404
405 OS << "0\n};\n\n";
406
407 // verify that all offsets will fit in 16/32 bits.
408 if ((MaxOffset & Mask) != MaxOffset)
409 PrintFatalError(Msg: "Offset of long encoding table exceeds encoding bits");
410
411 // Emit the shared table of register lists.
412 OS << "static constexpr unsigned char IIT_LongEncodingTable[] = {\n";
413 if (!LongEncodingTable.empty())
414 LongEncodingTable.emit(
415 OS, Print: [](raw_ostream &OS, unsigned char C) { OS << (unsigned)C; });
416 OS << " 255\n};\n";
417}
418
419/// Returns the effective MemoryEffects for intrinsic \p Int.
420static MemoryEffects getEffectiveME(const CodeGenIntrinsic &Int) {
421 MemoryEffects ME = Int.ME;
422 // TODO: IntrHasSideEffects should affect not only readnone intrinsics.
423 if (ME.doesNotAccessMemory() && Int.hasSideEffects)
424 ME = MemoryEffects::unknown();
425 return ME;
426}
427
428static bool compareFnAttributes(const CodeGenIntrinsic *L,
429 const CodeGenIntrinsic *R) {
430 auto TieBoolAttributes = [](const CodeGenIntrinsic *I) -> auto {
431 // Sort throwing intrinsics after non-throwing intrinsics.
432 return std::tie(args: I->canThrow, args: I->isNoDuplicate, args: I->isNoMerge, args: I->isNoReturn,
433 args: I->isNoCallback, args: I->isNoSync, args: I->isNoFree, args: I->isWillReturn,
434 args: I->isCold, args: I->isConvergent, args: I->isSpeculatable,
435 args: I->hasSideEffects, args: I->isStrictFP,
436 args: I->isNoCreateUndefOrPoison);
437 };
438
439 auto TieL = TieBoolAttributes(L);
440 auto TieR = TieBoolAttributes(R);
441
442 if (TieL != TieR)
443 return TieL < TieR;
444
445 // Try to order by readonly/readnone attribute.
446 uint32_t LME = getEffectiveME(Int: *L).toIntValue();
447 uint32_t RME = getEffectiveME(Int: *R).toIntValue();
448 if (LME != RME)
449 return LME > RME;
450
451 return false;
452}
453
454/// Returns true if \p Int has a non-empty set of function attributes. Note that
455/// NoUnwind = !canThrow, so we need to negate it's sense to test if the
456// intrinsic has NoUnwind attribute.
457static bool hasFnAttributes(const CodeGenIntrinsic &Int) {
458 return !Int.canThrow || Int.isNoReturn || Int.isNoCallback || Int.isNoSync ||
459 Int.isNoFree || Int.isWillReturn || Int.isCold || Int.isNoDuplicate ||
460 Int.isNoMerge || Int.isConvergent || Int.isSpeculatable ||
461 Int.isStrictFP || Int.isNoCreateUndefOrPoison ||
462 getEffectiveME(Int) != MemoryEffects::unknown();
463}
464
465namespace {
466struct FnAttributeComparator {
467 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
468 return compareFnAttributes(L, R);
469 }
470};
471
472struct AttributeComparator {
473 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
474 // This comparator is used to unique just the argument attributes of an
475 // intrinsic without considering any function attributes.
476 return L->ArgumentAttributes < R->ArgumentAttributes;
477 }
478};
479} // End anonymous namespace
480
481/// Returns the name of the IR enum for argument attribute kind \p Kind.
482static StringRef getArgAttrEnumName(CodeGenIntrinsic::ArgAttrKind Kind) {
483 switch (Kind) {
484 case CodeGenIntrinsic::NoCapture:
485 llvm_unreachable("Handled separately");
486 case CodeGenIntrinsic::NoAlias:
487 return "NoAlias";
488 case CodeGenIntrinsic::NoUndef:
489 return "NoUndef";
490 case CodeGenIntrinsic::NonNull:
491 return "NonNull";
492 case CodeGenIntrinsic::Returned:
493 return "Returned";
494 case CodeGenIntrinsic::ReadOnly:
495 return "ReadOnly";
496 case CodeGenIntrinsic::WriteOnly:
497 return "WriteOnly";
498 case CodeGenIntrinsic::ReadNone:
499 return "ReadNone";
500 case CodeGenIntrinsic::ImmArg:
501 return "ImmArg";
502 case CodeGenIntrinsic::Alignment:
503 return "Alignment";
504 case CodeGenIntrinsic::Dereferenceable:
505 return "Dereferenceable";
506 case CodeGenIntrinsic::Range:
507 return "Range";
508 }
509 llvm_unreachable("Unknown CodeGenIntrinsic::ArgAttrKind enum");
510}
511
512/// EmitAttributes - This emits the Intrinsic::getAttributes method.
513void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
514 raw_ostream &OS) {
515 IfDefEmitter IfDef(OS, "GET_INTRINSIC_ATTRIBUTES");
516 OS << R"(// Add parameter attributes that are not common to all intrinsics.
517static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID,
518 Type *ArgType) {
519 unsigned BitWidth = ArgType->getScalarSizeInBits();
520 switch (ID) {
521 default: llvm_unreachable("Invalid attribute set number");)";
522 // Compute unique argument attribute sets.
523 std::map<SmallVector<CodeGenIntrinsic::ArgAttribute, 0>, unsigned>
524 UniqArgAttributes;
525 for (const CodeGenIntrinsic &Int : Ints) {
526 for (auto &Attrs : Int.ArgumentAttributes) {
527 if (Attrs.empty())
528 continue;
529
530 unsigned ID = UniqArgAttributes.size();
531 if (!UniqArgAttributes.try_emplace(k: Attrs, args&: ID).second)
532 continue;
533
534 assert(is_sorted(Attrs) && "Argument attributes are not sorted");
535
536 OS << formatv(Fmt: R"(
537 case {}:
538 return AttributeSet::get(C, {{
539)",
540 Vals&: ID);
541 for (const CodeGenIntrinsic::ArgAttribute &Attr : Attrs) {
542 if (Attr.Kind == CodeGenIntrinsic::NoCapture) {
543 OS << " Attribute::getWithCaptureInfo(C, "
544 "CaptureInfo::none()),\n";
545 continue;
546 }
547 StringRef AttrName = getArgAttrEnumName(Kind: Attr.Kind);
548 if (Attr.Kind == CodeGenIntrinsic::Alignment ||
549 Attr.Kind == CodeGenIntrinsic::Dereferenceable)
550 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}, {}),\n",
551 Vals&: AttrName, Vals: Attr.Value);
552 else if (Attr.Kind == CodeGenIntrinsic::Range)
553 // This allows implicitTrunc because the range may only fit the
554 // type based on rules implemented in the IR verifier. E.g. the
555 // [-1, 1] range for ucmp/scmp intrinsics requires a minimum i2 type.
556 // Give the verifier a chance to diagnose this instead of asserting
557 // here.
558 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}, "
559 "ConstantRange(APInt(BitWidth, {}, /*isSigned=*/true, "
560 "/*implicitTrunc=*/true), APInt(BitWidth, {}, "
561 "/*isSigned=*/true, /*implicitTrunc=*/true))),\n",
562 Vals&: AttrName, Vals: (int64_t)Attr.Value, Vals: (int64_t)Attr.Value2);
563 else
564 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}),\n", Vals&: AttrName);
565 }
566 OS << " });";
567 }
568 }
569 OS << R"(
570 }
571} // getIntrinsicArgAttributeSet
572)";
573
574 // Compute unique function attribute sets. Note that ID 255 will be used for
575 // intrinsics with no function attributes.
576 std::map<const CodeGenIntrinsic *, unsigned, FnAttributeComparator>
577 UniqFnAttributes;
578 OS << R"(
579static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
580 switch (ID) {
581 default: llvm_unreachable("Invalid attribute set number");)";
582
583 for (const CodeGenIntrinsic &Int : Ints) {
584 if (!hasFnAttributes(Int))
585 continue;
586 unsigned ID = UniqFnAttributes.size();
587 if (!UniqFnAttributes.try_emplace(k: &Int, args&: ID).second)
588 continue;
589 OS << formatv(Fmt: R"(
590 case {}: // {}
591 return AttributeSet::get(C, {{
592)",
593 Vals&: ID, Vals: Int.Name);
594 auto addAttribute = [&OS](StringRef Attr) {
595 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}),\n", Vals&: Attr);
596 };
597 if (!Int.canThrow)
598 addAttribute("NoUnwind");
599 if (Int.isNoReturn)
600 addAttribute("NoReturn");
601 if (Int.isNoCallback)
602 addAttribute("NoCallback");
603 if (Int.isNoSync)
604 addAttribute("NoSync");
605 if (Int.isNoFree)
606 addAttribute("NoFree");
607 if (Int.isWillReturn)
608 addAttribute("WillReturn");
609 if (Int.isCold)
610 addAttribute("Cold");
611 if (Int.isNoDuplicate)
612 addAttribute("NoDuplicate");
613 if (Int.isNoMerge)
614 addAttribute("NoMerge");
615 if (Int.isConvergent)
616 addAttribute("Convergent");
617 if (Int.isSpeculatable)
618 addAttribute("Speculatable");
619 if (Int.isStrictFP)
620 addAttribute("StrictFP");
621 if (Int.isNoCreateUndefOrPoison)
622 addAttribute("NoCreateUndefOrPoison");
623
624 const MemoryEffects ME = getEffectiveME(Int);
625 if (ME != MemoryEffects::unknown()) {
626 OS << formatv(Fmt: " // {}\n", Vals: ME);
627 OS << formatv(Fmt: " Attribute::getWithMemoryEffects(C, "
628 "MemoryEffects::createFromIntValue({})),\n",
629 Vals: ME.toIntValue());
630 }
631 OS << " });";
632 }
633 OS << R"(
634 }
635} // getIntrinsicFnAttributeSet)";
636
637 // Compute unique argument attributes.
638 std::map<const CodeGenIntrinsic *, unsigned, AttributeComparator>
639 UniqAttributes;
640 for (const CodeGenIntrinsic &Int : Ints) {
641 unsigned ID = UniqAttributes.size();
642 UniqAttributes.try_emplace(k: &Int, args&: ID);
643 }
644
645 const uint8_t UniqAttributesBitSize = Log2_32_Ceil(Value: UniqAttributes.size());
646 // Note, max value is used to indicate no function attributes.
647 const uint8_t UniqFnAttributesBitSize =
648 Log2_32_Ceil(Value: UniqFnAttributes.size() + 1);
649 const uint32_t NoFunctionAttrsID =
650 maskTrailingOnes<uint32_t>(N: UniqFnAttributesBitSize);
651 uint8_t AttributesMapDataBitSize =
652 PowerOf2Ceil(A: UniqAttributesBitSize + UniqFnAttributesBitSize);
653 if (AttributesMapDataBitSize < 8)
654 AttributesMapDataBitSize = 8;
655 else if (AttributesMapDataBitSize > 64)
656 PrintFatalError(Msg: "Packed ID of IntrinsicsToAttributesMap exceeds 64b!");
657
658 // Assign a packed ID for each intrinsic. The lower bits will be its
659 // "argument attribute ID" (index in UniqAttributes) and upper bits will be
660 // its "function attribute ID" (index in UniqFnAttributes).
661 OS << formatv(Fmt: "\nstatic constexpr uint{}_t IntrinsicsToAttributesMap[] = {{",
662 Vals&: AttributesMapDataBitSize);
663 for (const CodeGenIntrinsic &Int : Ints) {
664 uint32_t FnAttrIndex =
665 hasFnAttributes(Int) ? UniqFnAttributes[&Int] : NoFunctionAttrsID;
666 OS << formatv(Fmt: "\n {} << {} | {}, // {}", Vals&: FnAttrIndex,
667 Vals: UniqAttributesBitSize, Vals&: UniqAttributes[&Int], Vals: Int.Name);
668 }
669
670 OS << R"(
671}; // IntrinsicsToAttributesMap
672)";
673
674 // For a given intrinsic, its attributes are constructed by populating the
675 // local array `AS` below with its non-empty argument attributes followed by
676 // function attributes if any. Each argument attribute is constructed as:
677 //
678 // getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo));
679 //
680 // Create a table that records, for each argument attributes, the list of
681 // <ArgNo, ArgAttrID> pairs that are needed to construct its argument
682 // attributes. These tables for all intrinsics will be concatenated into one
683 // large table and then for each intrinsic, we remember the Staring index and
684 // number of size of its slice of entries (i.e., number of arguments with
685 // non-empty attributes), so that we can build the attribute list for an
686 // intrinsic without using a switch-case.
687
688 using ArgNoAttrIDPair = std::pair<uint16_t, uint16_t>;
689
690 // Emit the table of concatenated <ArgNo, AttrId> using SequenceToOffsetTable
691 // so that entries can be reused if possible. Individual sequences in this
692 // table do not have any terminator.
693 using ArgAttrIDSubTable = SmallVector<ArgNoAttrIDPair>;
694 SequenceToOffsetTable<ArgAttrIDSubTable> ArgAttrIdSequenceTable(std::nullopt);
695 SmallVector<ArgAttrIDSubTable> ArgAttrIdSubTables(
696 UniqAttributes.size()); // Indexed by UniqueID.
697
698 // Find the max number of attributes to create the local array.
699 unsigned MaxNumAttrs = 0;
700 for (const auto [IntPtr, UniqueID] : UniqAttributes) {
701 const CodeGenIntrinsic &Int = *IntPtr;
702 ArgAttrIDSubTable SubTable;
703
704 for (const auto &[ArgNo, Attrs] : enumerate(First: Int.ArgumentAttributes)) {
705 if (Attrs.empty())
706 continue;
707
708 uint16_t ArgAttrID = UniqArgAttributes.find(x: Attrs)->second;
709 SubTable.emplace_back(Args: (uint16_t)ArgNo, Args&: ArgAttrID);
710 }
711 ArgAttrIdSubTables[UniqueID] = SubTable;
712 if (!SubTable.empty())
713 ArgAttrIdSequenceTable.add(Seq: SubTable);
714 unsigned NumAttrs = SubTable.size() + hasFnAttributes(Int);
715 MaxNumAttrs = std::max(a: MaxNumAttrs, b: NumAttrs);
716 }
717
718 ArgAttrIdSequenceTable.layout();
719
720 if (ArgAttrIdSequenceTable.size() >= std::numeric_limits<uint16_t>::max())
721 PrintFatalError(Msg: "Size of ArgAttrIdTable exceeds supported limit");
722
723 // Emit the 2 tables (flattened ArgNo, ArgAttrID) and ArgAttributesInfoTable.
724 OS << formatv(Fmt: R"(
725namespace {{
726struct ArgNoAttrIDPair {{
727 uint16_t ArgNo, ArgAttrID;
728};
729} // namespace
730
731// Number of entries: {}
732static constexpr ArgNoAttrIDPair ArgAttrIdTable[] = {{
733)",
734 Vals: ArgAttrIdSequenceTable.size());
735
736 ArgAttrIdSequenceTable.emit(OS, Print: [](raw_ostream &OS, ArgNoAttrIDPair Elem) {
737 OS << formatv(Fmt: "{{{}, {}}", Vals&: Elem.first, Vals&: Elem.second);
738 });
739
740 OS << formatv(Fmt: R"(}; // ArgAttrIdTable
741
742namespace {{
743struct ArgAttributesInfo {{
744 uint16_t StartIndex;
745 uint16_t NumAttrs;
746};
747} // namespace
748
749// Number of entries: {}
750static constexpr ArgAttributesInfo ArgAttributesInfoTable[] = {{
751)",
752 Vals: ArgAttrIdSubTables.size());
753
754 for (const auto &SubTable : ArgAttrIdSubTables) {
755 unsigned NumAttrs = SubTable.size();
756 unsigned StartIndex = NumAttrs ? ArgAttrIdSequenceTable.get(Seq: SubTable) : 0;
757 OS << formatv(Fmt: " {{{}, {}},\n", Vals&: StartIndex, Vals&: NumAttrs);
758 }
759 OS << "}; // ArgAttributesInfoTable\n";
760
761 // Now emit the Intrinsic::getAttributes function. This will first map
762 // from intrinsic ID -> unique arg/function attr ID (using the
763 // IntrinsicsToAttributesMap) table. Then it will use the unique arg ID to
764 // construct all the argument attributes (using the ArgAttributesInfoTable and
765 // ArgAttrIdTable) and then add on the function attributes if any.
766 OS << formatv(Fmt: R"(
767
768template <typename IDTy>
769inline std::pair<uint32_t, uint32_t> unpackID(const IDTy PackedID) {{
770 constexpr uint8_t UniqAttributesBitSize = {};
771 const uint32_t FnAttrID = PackedID >> UniqAttributesBitSize;
772 const uint32_t ArgAttrID = PackedID &
773 maskTrailingOnes<uint32_t>(UniqAttributesBitSize);
774 return {{FnAttrID, ArgAttrID};
775}
776
777AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id,
778 FunctionType *FT) {{
779 if (id == 0)
780 return AttributeList();
781 auto [FnAttrID, ArgAttrID] = unpackID(IntrinsicsToAttributesMap[id - 1]);
782 using PairTy = std::pair<unsigned, AttributeSet>;
783 alignas(PairTy) char ASStorage[sizeof(PairTy) * {}];
784 PairTy *AS = reinterpret_cast<PairTy *>(ASStorage);
785
786 // Construct an ArrayRef for easier range checking.
787 ArrayRef<ArgAttributesInfo> ArgAttributesInfoTableAR(ArgAttributesInfoTable);
788 if (ArgAttrID >= ArgAttributesInfoTableAR.size())
789 llvm_unreachable("Invalid arguments attribute ID");
790
791 auto [StartIndex, NumAttrs] = ArgAttributesInfoTableAR[ArgAttrID];
792 for (unsigned Idx = 0; Idx < NumAttrs; ++Idx) {{
793 auto [ArgNo, ArgAttrID] = ArgAttrIdTable[StartIndex + Idx];
794 AS[Idx] = {{ArgNo,
795 getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo))};
796 }
797 if (FnAttrID != {}) {
798 AS[NumAttrs++] = {{AttributeList::FunctionIndex,
799 getIntrinsicFnAttributeSet(C, FnAttrID)};
800 }
801 return AttributeList::get(C, ArrayRef(AS, NumAttrs));
802}
803
804AttributeSet Intrinsic::getFnAttributes(LLVMContext &C, ID id) {{
805 if (id == 0)
806 return AttributeSet();
807 auto [FnAttrID, _] = unpackID(IntrinsicsToAttributesMap[id - 1]);
808 if (FnAttrID == {})
809 return AttributeSet();
810 return getIntrinsicFnAttributeSet(C, FnAttrID);
811}
812)",
813 Vals: UniqAttributesBitSize, Vals&: MaxNumAttrs, Vals: NoFunctionAttrsID,
814 Vals: NoFunctionAttrsID);
815}
816
817void IntrinsicEmitter::EmitIntrinsicToPrettyPrintTable(
818 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
819 EmitIntrinsicBitTable(Ints, OS, Guard: "GET_INTRINSIC_PRETTY_PRINT_TABLE", TableName: "PPTable",
820 Comment: "Intrinsic ID to pretty print bitset.",
821 GetProperty: [](const CodeGenIntrinsic &Int) {
822 return !Int.PrettyPrintFunctions.empty();
823 });
824}
825
826void IntrinsicEmitter::EmitPrettyPrintArguments(
827 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
828 IfDefEmitter IfDef(OS, "GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS");
829 OS << R"(
830void Intrinsic::printImmArg(ID IID, unsigned ArgIdx, raw_ostream &OS, const Constant *ImmArgVal) {
831 using namespace Intrinsic;
832 switch (IID) {
833)";
834
835 for (const CodeGenIntrinsic &Int : Ints) {
836 if (Int.PrettyPrintFunctions.empty())
837 continue;
838
839 OS << " case " << Int.EnumName << ":\n";
840 OS << " switch (ArgIdx) {\n";
841 for (const auto [ArgIdx, ArgName, FuncName] : Int.PrettyPrintFunctions) {
842 OS << " case " << ArgIdx << ":\n";
843 OS << " OS << \"" << ArgName << "=\";\n";
844 if (!FuncName.empty()) {
845 OS << " ";
846 if (!Int.TargetPrefix.empty())
847 OS << Int.TargetPrefix << "::";
848 OS << FuncName << "(OS, ImmArgVal);\n";
849 }
850 OS << " return;\n";
851 }
852 OS << " }\n";
853 OS << " break;\n";
854 }
855 OS << R"( default:
856 break;
857 }
858})";
859}
860
861void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
862 const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) {
863 StringRef CompilerName = IsClang ? "Clang" : "MS";
864 StringRef UpperCompilerName = IsClang ? "CLANG" : "MS";
865
866 // map<TargetPrefix, pair<map<BuiltinName, EnumName>, CommonPrefix>.
867 // Note that we iterate over both the maps in the code below and both
868 // iterations need to iterate in sorted key order. For the inner map, entries
869 // need to be emitted in the sorted order of `BuiltinName` with `CommonPrefix`
870 // rempved, because we use std::lower_bound to search these entries. For the
871 // outer map as well, entries need to be emitted in sorter order of
872 // `TargetPrefix` as we use std::lower_bound to search these entries.
873 using BIMEntryTy =
874 std::pair<std::map<StringRef, StringRef>, std::optional<StringRef>>;
875 std::map<StringRef, BIMEntryTy> BuiltinMap;
876
877 for (const CodeGenIntrinsic &Int : Ints) {
878 StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName;
879 if (BuiltinName.empty())
880 continue;
881 // Get the map for this target prefix.
882 auto &[Map, CommonPrefix] = BuiltinMap[Int.TargetPrefix];
883
884 if (!Map.try_emplace(k: BuiltinName, args: Int.EnumName).second)
885 PrintFatalError(ErrorLoc: Int.TheDef->getLoc(),
886 Msg: "Intrinsic '" + Int.TheDef->getName() + "': duplicate " +
887 CompilerName + " builtin name!");
888
889 // Update common prefix.
890 if (!CommonPrefix) {
891 // For the first builtin for this target, initialize the common prefix.
892 CommonPrefix = BuiltinName;
893 continue;
894 }
895
896 // Update the common prefix. Note that this assumes that `take_front` will
897 // never set the `Data` pointer in CommonPrefix to nullptr.
898 const char *Mismatch = mismatch(Range1&: *CommonPrefix, Range2&: BuiltinName).first;
899 *CommonPrefix = CommonPrefix->take_front(N: Mismatch - CommonPrefix->begin());
900 }
901
902 // Populate the string table with the names of all the builtins after
903 // removing this common prefix.
904 StringToOffsetTable Table;
905 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
906 auto &[Map, CommonPrefix] = Entry;
907 for (auto &[BuiltinName, EnumName] : Map) {
908 StringRef Suffix = BuiltinName.substr(Start: CommonPrefix->size());
909 Table.GetOrAddStringOffset(Str: Suffix);
910 }
911 }
912
913 IfDefEmitter IfDef(
914 OS,
915 formatv(Fmt: "GET_LLVM_INTRINSIC_FOR_{}_BUILTIN", Vals&: UpperCompilerName).str());
916 OS << formatv(Fmt: R"(
917// Get the LLVM intrinsic that corresponds to a builtin. This is used by the
918// C front-end. The builtin name is passed in as BuiltinName, and a target
919// prefix (e.g. 'ppc') is passed in as TargetPrefix.
920Intrinsic::ID
921Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
922 StringRef BuiltinName) {{
923 using namespace Intrinsic;
924)",
925 Vals&: CompilerName);
926
927 if (BuiltinMap.empty()) {
928 OS << "return not_intrinsic;\n";
929 return;
930 }
931
932 if (!Table.empty()) {
933 Table.EmitStringTableDef(OS, Name: "BuiltinNames");
934
935 OS << R"(
936 struct BuiltinEntry {
937 ID IntrinsicID;
938 unsigned StrTabOffset;
939 const char *getName() const { return BuiltinNames[StrTabOffset].data(); }
940 bool operator<(StringRef RHS) const {
941 return strncmp(getName(), RHS.data(), RHS.size()) < 0;
942 }
943 };
944
945)";
946 }
947
948 // Emit a per target table of bultin names.
949 bool HasTargetIndependentBuiltins = false;
950 StringRef TargetIndepndentCommonPrefix;
951 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
952 const auto &[Map, CommonPrefix] = Entry;
953 if (!TargetPrefix.empty()) {
954 OS << formatv(Fmt: " // Builtins for {0}.\n", Vals: TargetPrefix);
955 } else {
956 OS << " // Target independent builtins.\n";
957 HasTargetIndependentBuiltins = true;
958 TargetIndepndentCommonPrefix = *CommonPrefix;
959 }
960
961 // Emit the builtin table for this target prefix.
962 OS << formatv(Fmt: " static constexpr BuiltinEntry {}Names[] = {{\n",
963 Vals: TargetPrefix);
964 for (const auto &[BuiltinName, EnumName] : Map) {
965 StringRef Suffix = BuiltinName.substr(Start: CommonPrefix->size());
966 OS << formatv(Fmt: " {{{}, {}}, // {}\n", Vals: EnumName,
967 Vals: *Table.GetStringOffset(Str: Suffix), Vals: BuiltinName);
968 }
969 OS << formatv(Fmt: " }; // {}Names\n\n", Vals: TargetPrefix);
970 }
971
972 // After emitting the builtin tables for all targets, emit a lookup table for
973 // all targets. We will use binary search, similar to the table for builtin
974 // names to lookup into this table.
975 OS << R"(
976 struct TargetEntry {
977 StringLiteral TargetPrefix;
978 ArrayRef<BuiltinEntry> Names;
979 StringLiteral CommonPrefix;
980 bool operator<(StringRef RHS) const {
981 return TargetPrefix < RHS;
982 };
983 };
984 static constexpr TargetEntry TargetTable[] = {
985)";
986
987 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
988 const auto &[Map, CommonPrefix] = Entry;
989 if (TargetPrefix.empty())
990 continue;
991 OS << formatv(Fmt: R"( {{"{0}", {0}Names, "{1}"},)", Vals: TargetPrefix,
992 Vals: CommonPrefix)
993 << "\n";
994 }
995 OS << " };\n";
996
997 // Now for the actual lookup, first check the target independent table if
998 // we emitted one.
999 if (HasTargetIndependentBuiltins) {
1000 OS << formatv(Fmt: R"(
1001 // Check if it's a target independent builtin.
1002 // Copy the builtin name so we can use it in consume_front without clobbering
1003 // if for the lookup in the target specific table.
1004 StringRef Suffix = BuiltinName;
1005 if (Suffix.consume_front("{}")) {{
1006 auto II = lower_bound(Names, Suffix);
1007 if (II != std::end(Names) && II->getName() == Suffix)
1008 return II->IntrinsicID;
1009 }
1010)",
1011 Vals&: TargetIndepndentCommonPrefix);
1012 }
1013
1014 // If a target independent builtin was not found, lookup the target specific.
1015 OS << R"(
1016 auto TI = lower_bound(TargetTable, TargetPrefix);
1017 if (TI == std::end(TargetTable) || TI->TargetPrefix != TargetPrefix)
1018 return not_intrinsic;
1019 // This is the last use of BuiltinName, so no need to copy before using it in
1020 // consume_front.
1021 if (!BuiltinName.consume_front(TI->CommonPrefix))
1022 return not_intrinsic;
1023 auto II = lower_bound(TI->Names, BuiltinName);
1024 if (II == std::end(TI->Names) || II->getName() != BuiltinName)
1025 return not_intrinsic;
1026 return II->IntrinsicID;
1027}
1028)";
1029}
1030
1031static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/true>>
1032 X("gen-intrinsic-enums", "Generate intrinsic enums");
1033
1034static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/false>>
1035 Y("gen-intrinsic-impl", "Generate intrinsic implementation code");
1036