1//===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend emits information about intrinsic functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenIntrinsics.h"
14#include "SequenceToOffsetTable.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Support/CommandLine.h"
20#include "llvm/Support/ErrorHandling.h"
21#include "llvm/Support/FormatVariadic.h"
22#include "llvm/Support/ModRef.h"
23#include "llvm/Support/SourceMgr.h"
24#include "llvm/Support/raw_ostream.h"
25#include "llvm/TableGen/CodeGenHelpers.h"
26#include "llvm/TableGen/Error.h"
27#include "llvm/TableGen/Record.h"
28#include "llvm/TableGen/StringToOffsetTable.h"
29#include "llvm/TableGen/TableGenBackend.h"
30#include <algorithm>
31#include <array>
32#include <cassert>
33#include <cctype>
34#include <map>
35#include <optional>
36#include <string>
37#include <utility>
38#include <vector>
39using namespace llvm;
40
41static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
42static cl::opt<std::string>
43 IntrinsicPrefix("intrinsic-prefix",
44 cl::desc("Generate intrinsics with this target prefix"),
45 cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat));
46
47namespace {
48class IntrinsicEmitter {
49 const RecordKeeper &Records;
50
51public:
52 IntrinsicEmitter(const RecordKeeper &R) : Records(R) {}
53
54 void run(raw_ostream &OS, bool Enums);
55
56 void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
57 void EmitAnyKindEnums(raw_ostream &OS);
58 void EmitIITInfo(raw_ostream &OS);
59 void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
60 void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints,
61 raw_ostream &OS);
62 void EmitIntrinsicToTargetFeaturesTable(const CodeGenIntrinsicTable &Ints,
63 raw_ostream &OS);
64 void EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable &Ints,
65 raw_ostream &OS);
66 void EmitIntrinsicToScalarizableTable(const CodeGenIntrinsicTable &Ints,
67 raw_ostream &OS);
68 void EmitIntrinsicToPrettyPrintTable(const CodeGenIntrinsicTable &Ints,
69 raw_ostream &OS);
70 void EmitIntrinsicBitTable(
71 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,
72 StringRef TableName, StringRef Comment,
73 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty);
74 void EmitGenerator(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
75 void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
76 void EmitImmArgRangeSetChecks(const CodeGenIntrinsicTable &Ints,
77 raw_ostream &OS);
78 void EmitPrettyPrintArguments(const CodeGenIntrinsicTable &Ints,
79 raw_ostream &OS);
80 void EmitDefaultArgValuesTable(const CodeGenIntrinsicTable &Ints,
81 raw_ostream &OS);
82 void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints,
83 bool IsClang, raw_ostream &OS);
84};
85
86// Helper class to use with `TableGen::Emitter::OptClass`.
87template <bool Enums> class IntrinsicEmitterOpt : public IntrinsicEmitter {
88public:
89 IntrinsicEmitterOpt(const RecordKeeper &R) : IntrinsicEmitter(R) {}
90 void run(raw_ostream &OS) { IntrinsicEmitter::run(OS, Enums); }
91};
92
93} // End anonymous namespace
94
95//===----------------------------------------------------------------------===//
96// IntrinsicEmitter Implementation
97//===----------------------------------------------------------------------===//
98
99void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {
100 emitSourceFileHeader(Desc: "Intrinsic Function Source Fragment", OS);
101
102 CodeGenIntrinsicTable Ints(Records);
103
104 if (Enums) {
105 // Emit the enum information.
106 EmitEnumInfo(Ints, OS);
107
108 // Emit AnyKind enums for Intrinsics.h.
109 EmitAnyKindEnums(OS);
110 } else {
111 // Emit IIT_Info constants.
112 EmitIITInfo(OS);
113
114 // Emit the target metadata.
115 EmitTargetInfo(Ints, OS);
116
117 // Emit the intrinsic ID -> name table.
118 EmitIntrinsicToNameTable(Ints, OS);
119
120 // Emit the intrinsic ID -> required target features table.
121 EmitIntrinsicToTargetFeaturesTable(Ints, OS);
122
123 // Emit the intrinsic ID -> overload table.
124 EmitIntrinsicToOverloadTable(Ints, OS);
125
126 // Emit the intrinsic ID -> trivially scalarizable table.
127 EmitIntrinsicToScalarizableTable(Ints, OS);
128
129 // Emit the intrinsic declaration generator.
130 EmitGenerator(Ints, OS);
131
132 // Emit the intrinsic parameter attributes.
133 EmitAttributes(Ints, OS);
134
135 // Emit immediate argument range-set checks.
136 EmitImmArgRangeSetChecks(Ints, OS);
137
138 // Emit the intrinsic ID -> pretty print table.
139 EmitIntrinsicToPrettyPrintTable(Ints, OS);
140
141 // Emit Pretty Print attribute.
142 EmitPrettyPrintArguments(Ints, OS);
143
144 // Emit the default-argument values table and lookup function.
145 EmitDefaultArgValuesTable(Ints, OS);
146
147 // Emit code to translate Clang builtins into LLVM intrinsics.
148 EmitIntrinsicToBuiltinMap(Ints, IsClang: true, OS);
149
150 // Emit code to translate MS builtins into LLVM intrinsics.
151 EmitIntrinsicToBuiltinMap(Ints, IsClang: false, OS);
152 }
153}
154
155void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
156 raw_ostream &OS) {
157 // Find the TargetSet for which to generate enums. There will be an initial
158 // set with an empty target prefix which will include target independent
159 // intrinsics like dbg.value.
160 using TargetSet = CodeGenIntrinsicTable::TargetSet;
161 const TargetSet *Set = nullptr;
162 for (const auto &Target : Ints.getTargets()) {
163 if (Target.Name == IntrinsicPrefix) {
164 Set = &Target;
165 break;
166 }
167 }
168 if (!Set) {
169 // The first entry is for target independent intrinsics, so drop it.
170 auto KnowTargets = Ints.getTargets().drop_front();
171 PrintFatalError(PrintMsg: [KnowTargets](raw_ostream &OS) {
172 OS << "tried to generate intrinsics for unknown target "
173 << IntrinsicPrefix << "\nKnown targets are: ";
174 interleaveComma(c: KnowTargets, os&: OS,
175 each_fn: [&OS](const TargetSet &Target) { OS << Target.Name; });
176 OS << '\n';
177 });
178 }
179
180 // Generate a complete header for target specific intrinsics.
181 std::optional<IfDefEmitter> IfDef;
182 std::optional<IncludeGuardEmitter> IncGuard;
183 std::optional<NamespaceEmitter> NS;
184
185 if (IntrinsicPrefix.empty()) {
186 IfDef.emplace(args&: OS, args: "GET_INTRINSIC_ENUM_VALUES");
187 } else {
188 std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
189 IncGuard.emplace(
190 args&: OS, args: formatv(Fmt: "LLVM_IR_INTRINSIC_{}_ENUMS_H", Vals&: UpperPrefix).str());
191 NS.emplace(args&: OS, args: "llvm::Intrinsic");
192 OS << formatv(Fmt: "enum {}Intrinsics : unsigned {{\n", Vals&: UpperPrefix);
193 }
194
195 OS << "// Enum values for intrinsics.\n";
196 bool First = true;
197 for (const CodeGenIntrinsic &Int : Ints[*Set]) {
198 OS << " " << Int.EnumName;
199
200 // Assign a value to the first intrinsic in this target set so that all
201 // intrinsic ids are distinct.
202 if (First) {
203 OS << " = " << Set->Offset + 1;
204 First = false;
205 }
206
207 OS << ", ";
208 if (Int.EnumName.size() < 40)
209 OS.indent(NumSpaces: 40 - Int.EnumName.size());
210 OS << formatv(
211 Fmt: " // {} ({})\n", Vals: Int.Name,
212 Vals: SrcMgr.getFormattedLocationNoOffset(Loc: Int.TheDef->getLoc().front()));
213 }
214
215 // Emit num_intrinsics into the target neutral enum.
216 if (IntrinsicPrefix.empty())
217 OS << formatv(Fmt: " num_intrinsics = {}\n", Vals: Ints.size() + 1);
218 else
219 OS << "}; // enum\n";
220}
221
222void IntrinsicEmitter::EmitAnyKindEnums(raw_ostream &OS) {
223 if (!IntrinsicPrefix.empty())
224 return;
225 IfDefEmitter IfDef(OS, "GET_INTRINSIC_ANYKIND_ENUMS");
226
227 auto GenerateAnyKindEnums = [&OS, this](StringRef EnumName,
228 StringRef Prefix) {
229 OS << "// llvm::Intrinsic::IITDescriptor::" << EnumName << "\n";
230 if (const Record *EnumDef = Records.getDef(Name: EnumName)) {
231 OS << "enum " << EnumName << " {\n";
232 for (const auto &RV : EnumDef->getValues())
233 OS << " " << Prefix << RV.getName() << " = " << *RV.getValue()
234 << ",\n";
235 OS << "}; // " << EnumName << "\n\n";
236 } else {
237 OS << "#error \"" << EnumName << " is not defined\"\n";
238 }
239 };
240 GenerateAnyKindEnums("AnyKindVectorConstraint", "VC_");
241 GenerateAnyKindEnums("AnyKindElementConstraint", "EC_");
242}
243
244void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {
245 IfDefEmitter IfDef(OS, "GET_INTRINSIC_IITINFO");
246 std::array<StringRef, 256> RecsByNumber;
247 auto IIT_Base = Records.getAllDerivedDefinitionsIfDefined(ClassName: "IIT_Base");
248 for (const Record *Rec : IIT_Base) {
249 auto Number = Rec->getValueAsInt(FieldName: "Number");
250 assert(0 <= Number && Number < (int)RecsByNumber.size() &&
251 "IIT_Info.Number should be uint8_t");
252 assert(RecsByNumber[Number].empty() && "Duplicate IIT_Info.Number");
253 RecsByNumber[Number] = Rec->getName();
254 }
255 if (IIT_Base.size() > 0) {
256 if (RecsByNumber[0] != "IIT_Done")
257 PrintFatalError(Msg: "IIT_Done expected to have value 0");
258 for (unsigned I = 0, E = RecsByNumber.size(); I < E; ++I)
259 if (!RecsByNumber[I].empty())
260 OS << " " << RecsByNumber[I] << " = " << I << ",\n";
261 } else {
262 OS << "#error \"class IIT_Base is not defined\"\n";
263 }
264}
265
266void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
267 raw_ostream &OS) {
268 IfDefEmitter IfDef(OS, "GET_INTRINSIC_TARGET_DATA");
269 OS << R"(// Target mapping.
270struct IntrinsicTargetInfo {
271 StringLiteral Name;
272 size_t Offset;
273 size_t Count;
274};
275static constexpr IntrinsicTargetInfo TargetInfos[] = {
276)";
277 for (const auto [Name, Offset, Count] : Ints.getTargets())
278 OS << formatv(Fmt: " {{\"{}\", {}, {}},\n", Vals: Name, Vals: Offset, Vals: Count);
279 OS << "};\n";
280}
281
282/// Helper function to emit a bit table for intrinsic properties.
283/// This is used for both overload and pretty print bit tables.
284void IntrinsicEmitter::EmitIntrinsicBitTable(
285 const CodeGenIntrinsicTable &Ints, raw_ostream &OS, StringRef Guard,
286 StringRef TableName, StringRef Comment,
287 function_ref<bool(const CodeGenIntrinsic &Int)> GetProperty) {
288 IfDefEmitter IfDef(OS, Guard);
289 OS << formatv(Fmt: "// {}\n", Vals&: Comment);
290 OS << formatv(Fmt: "static constexpr uint8_t {}[] = {{\n", Vals&: TableName);
291 OS << " 0\n ";
292 for (auto [I, Int] : enumerate(First: Ints)) {
293 // Add one to the index so we emit a null bit for the invalid #0 intrinsic.
294 size_t Idx = I + 1;
295 if (Idx % 8 == 0)
296 OS << ",\n 0";
297 if (GetProperty(Int))
298 OS << " | (1<<" << Idx % 8 << ')';
299 }
300 OS << "\n};\n\n";
301 OS << formatv(Fmt: "return ({}[id/8] & (1 << (id%8))) != 0;\n", Vals&: TableName);
302}
303
304void IntrinsicEmitter::EmitIntrinsicToNameTable(
305 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
306 // Built up a table of the intrinsic names.
307 constexpr StringLiteral NotIntrinsic = "not_intrinsic";
308 StringToOffsetTable Table;
309 Table.GetOrAddStringOffset(Str: NotIntrinsic);
310 for (const auto &Int : Ints)
311 Table.GetOrAddStringOffset(Str: Int.Name);
312
313 IfDefEmitter IfDef(OS, "GET_INTRINSIC_NAME_TABLE");
314 OS << R"(// Intrinsic ID to name table.
315// Note that entry #0 is the invalid intrinsic!
316
317)";
318
319 Table.EmitStringTableDef(OS, Name: "IntrinsicNameTable");
320
321 OS << R"(
322static constexpr unsigned IntrinsicNameOffsetTable[] = {
323)";
324
325 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: NotIntrinsic),
326 Vals: NotIntrinsic);
327 for (const auto &Int : Ints)
328 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: Int.Name), Vals: Int.Name);
329
330 OS << "\n}; // IntrinsicNameOffsetTable\n";
331}
332
333void IntrinsicEmitter::EmitIntrinsicToTargetFeaturesTable(
334 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
335 StringToOffsetTable Table;
336 for (const CodeGenIntrinsic &Int : Ints)
337 Table.GetOrAddStringOffset(Str: Int.TargetFeatures);
338
339 IfDefEmitter IfDef(OS, "GET_INTRINSIC_TARGET_FEATURES_TABLE");
340 OS << R"(// Intrinsic ID to required target features table.
341// Note that entry #0 is the invalid intrinsic!
342
343)";
344
345 Table.EmitStringTableDef(OS, Name: "IntrinsicTargetFeaturesTable");
346
347 OS << R"(
348static constexpr unsigned IntrinsicTargetFeaturesOffsetTable[] = {
349)";
350
351 OS << " 0, // not_intrinsic\n";
352 for (const CodeGenIntrinsic &Int : Ints) {
353 OS << formatv(Fmt: " {}, // {}\n", Vals: *Table.GetStringOffset(Str: Int.TargetFeatures),
354 Vals: Int.Name);
355 }
356
357 OS << "\n}; // IntrinsicTargetFeaturesOffsetTable\n";
358}
359
360void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
361 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
362 EmitIntrinsicBitTable(
363 Ints, OS, Guard: "GET_INTRINSIC_OVERLOAD_TABLE", TableName: "OTable",
364 Comment: "Intrinsic ID to overload bitset.",
365 GetProperty: [](const CodeGenIntrinsic &Int) { return Int.isOverloaded; });
366}
367
368void IntrinsicEmitter::EmitIntrinsicToScalarizableTable(
369 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
370 EmitIntrinsicBitTable(
371 Ints, OS, Guard: "GET_INTRINSIC_SCALARIZABLE_TABLE", TableName: "STable",
372 Comment: "Intrinsic ID to trivially scalarizable bitset.",
373 GetProperty: [](const CodeGenIntrinsic &Int) { return Int.isTriviallyScalarizable; });
374}
375
376using TypeSigTy = SmallVector<unsigned char>;
377
378/// Computes type signature of the intrinsic \p Int.
379static TypeSigTy ComputeTypeSignature(const CodeGenIntrinsic &Int) {
380 TypeSigTy TypeSig;
381 const Record *TypeInfo = Int.TheDef->getValueAsDef(FieldName: "TypeInfo");
382 const ListInit *TypeList = TypeInfo->getValueAsListInit(FieldName: "TypeSig");
383
384 for (const auto *TypeListEntry : TypeList->getElements()) {
385 int64_t Value = cast<IntInit>(Val: TypeListEntry)->getValue();
386 if (Value < 0 || Value > 255)
387 PrintFatalError(Rec: Int.TheDef, Msg: "Unresolved type signature");
388 TypeSig.emplace_back(Args&: Value);
389 }
390 return TypeSig;
391}
392
393// Note: the code below can be switched to use 32-bit fixed encoding by
394// flipping the flag below.
395constexpr bool Use16BitFixedEncoding = true;
396using FixedEncodingTy =
397 std::conditional_t<Use16BitFixedEncoding, uint16_t, uint32_t>;
398
399// Pack the type signature into 16/32-bit fixed encoding word, where each byte
400// in the type signature is packed into a nibble (4 bits) if possible.
401static std::optional<FixedEncodingTy> encodePacked(const TypeSigTy &TypeSig) {
402 constexpr size_t NUM_NIBBLES = sizeof(FixedEncodingTy) * 2;
403 if (TypeSig.size() > NUM_NIBBLES)
404 return std::nullopt;
405
406 FixedEncodingTy Result = 0;
407 for (unsigned char C : reverse(C: TypeSig)) {
408 if (C > 15)
409 return std::nullopt;
410 Result = (Result << 4) | C;
411 }
412 return Result;
413}
414
415void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
416 raw_ostream &OS) {
417 constexpr unsigned FixedEncodingBits = sizeof(FixedEncodingTy) * CHAR_BIT;
418 constexpr unsigned MSBPosition = FixedEncodingBits - 1;
419 // Mask with all bits 1 except the most significant bit.
420 constexpr FixedEncodingTy Mask = (1U << MSBPosition) - 1;
421 StringRef FixedEncodingTypeName =
422 Use16BitFixedEncoding ? "uint16_t" : "uint32_t";
423
424 // If we can compute a 16/32-bit fixed encoding for this intrinsic, do so and
425 // capture it in this vector, otherwise store a ~0U.
426 std::vector<FixedEncodingTy> FixedEncodings;
427
428 // Each IIT encoding sequence in the long encoding table is terminated by
429 // IIT_Done(=0) token.
430 constexpr unsigned char IIT_Done = 0;
431 SequenceToOffsetTable<TypeSigTy> LongEncodingTable(IIT_Done);
432
433 FixedEncodings.reserve(n: Ints.size());
434
435 // Compute the unique argument type info.
436 for (const CodeGenIntrinsic &Int : Ints) {
437 // Get the signature for the intrinsic.
438 TypeSigTy TypeSig = ComputeTypeSignature(Int);
439
440 // Check to see if we can encode it into a 16/32 bit word.
441 std::optional<FixedEncodingTy> Result = encodePacked(TypeSig);
442 if (Result && (*Result & Mask) == *Result) {
443 FixedEncodings.push_back(x: *Result);
444 continue;
445 }
446
447 LongEncodingTable.add(Seq: TypeSig);
448
449 // This is a placehold that we'll replace after the table is laid out.
450 FixedEncodings.push_back(x: static_cast<FixedEncodingTy>(~0U));
451 }
452
453 LongEncodingTable.layout();
454
455 IfDefEmitter IfDef(OS, "GET_INTRINSIC_GENERATOR_GLOBAL");
456 OS << formatv(Fmt: R"(// Global intrinsic function declaration type table.
457using FixedEncodingTy = {};
458static constexpr FixedEncodingTy IIT_Table[] = {{
459 )",
460 Vals&: FixedEncodingTypeName);
461
462 unsigned MaxOffset = 0;
463 for (auto [Idx, FixedEncoding, Int] : enumerate(First&: FixedEncodings, Rest: Ints)) {
464 if ((Idx & 7) == 7)
465 OS << "\n ";
466
467 // If the entry fit in the table, just emit it.
468 if ((FixedEncoding & Mask) == FixedEncoding) {
469 OS << "0x" << Twine::utohexstr(Val: FixedEncoding) << ", ";
470 continue;
471 }
472
473 TypeSigTy TypeSig = ComputeTypeSignature(Int);
474 unsigned Offset = LongEncodingTable.get(Seq: TypeSig);
475 MaxOffset = std::max(a: MaxOffset, b: Offset);
476
477 // Otherwise, emit the offset into the long encoding table. We emit it this
478 // way so that it is easier to read the offset in the .def file.
479 OS << formatv(Fmt: "(1U<<{}) | {}, ", Vals: MSBPosition, Vals&: Offset);
480 }
481
482 OS << "0\n};\n\n";
483
484 // verify that all offsets will fit in 16/32 bits.
485 if ((MaxOffset & Mask) != MaxOffset)
486 PrintFatalError(Msg: "Offset of long encoding table exceeds encoding bits");
487
488 // Emit the shared table of register lists.
489 OS << "static constexpr unsigned char IIT_LongEncodingTable[] = {\n";
490 if (!LongEncodingTable.empty())
491 LongEncodingTable.emit(
492 OS, Print: [](raw_ostream &OS, unsigned char C) { OS << (unsigned)C; });
493 OS << " 255\n};\n";
494}
495
496/// Returns the effective MemoryEffects for intrinsic \p Int.
497static MemoryEffects getEffectiveME(const CodeGenIntrinsic &Int) {
498 MemoryEffects ME = Int.ME;
499 // TODO: IntrHasSideEffects should affect not only readnone intrinsics.
500 if (ME.doesNotAccessMemory() && Int.hasSideEffects)
501 ME = MemoryEffects::unknown();
502 return ME;
503}
504
505static bool compareFnAttributes(const CodeGenIntrinsic *L,
506 const CodeGenIntrinsic *R) {
507 auto TieBoolAttributes = [](const CodeGenIntrinsic *I) -> auto {
508 // Sort throwing intrinsics after non-throwing intrinsics.
509 return std::tie(args: I->canThrow, args: I->isNoDuplicate, args: I->isNoMerge, args: I->isNoReturn,
510 args: I->isNoCallback, args: I->isNoSync, args: I->isNoFree, args: I->isWillReturn,
511 args: I->isCold, args: I->isConvergent, args: I->isSpeculatable,
512 args: I->hasSideEffects, args: I->isStrictFP,
513 args: I->isNoCreateUndefOrPoison);
514 };
515
516 auto TieL = TieBoolAttributes(L);
517 auto TieR = TieBoolAttributes(R);
518
519 if (TieL != TieR)
520 return TieL < TieR;
521
522 // Try to order by readonly/readnone attribute.
523 uint32_t LME = getEffectiveME(Int: *L).toIntValue();
524 uint32_t RME = getEffectiveME(Int: *R).toIntValue();
525 if (LME != RME)
526 return LME > RME;
527
528 return false;
529}
530
531/// Returns true if \p Int has a non-empty set of function attributes. Note that
532/// NoUnwind = !canThrow, so we need to negate it's sense to test if the
533// intrinsic has NoUnwind attribute.
534static bool hasFnAttributes(const CodeGenIntrinsic &Int) {
535 return !Int.canThrow || Int.isNoReturn || Int.isNoCallback || Int.isNoSync ||
536 Int.isNoFree || Int.isWillReturn || Int.isCold || Int.isNoDuplicate ||
537 Int.isNoMerge || Int.isConvergent || Int.isSpeculatable ||
538 Int.isStrictFP || Int.isNoCreateUndefOrPoison ||
539 getEffectiveME(Int) != MemoryEffects::unknown();
540}
541
542namespace {
543struct FnAttributeComparator {
544 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
545 return compareFnAttributes(L, R);
546 }
547};
548
549struct AttributeComparator {
550 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
551 // This comparator is used to unique just the argument attributes of an
552 // intrinsic without considering any function attributes.
553 return L->ArgumentAttributes < R->ArgumentAttributes;
554 }
555};
556} // End anonymous namespace
557
558/// Returns the name of the IR enum for argument attribute kind \p Kind.
559static StringRef getArgAttrEnumName(CodeGenIntrinsic::ArgAttrKind Kind) {
560 switch (Kind) {
561 case CodeGenIntrinsic::NoCapture:
562 llvm_unreachable("Handled separately");
563 case CodeGenIntrinsic::NoAlias:
564 return "NoAlias";
565 case CodeGenIntrinsic::NoUndef:
566 return "NoUndef";
567 case CodeGenIntrinsic::NonNull:
568 return "NonNull";
569 case CodeGenIntrinsic::Returned:
570 return "Returned";
571 case CodeGenIntrinsic::ReadOnly:
572 return "ReadOnly";
573 case CodeGenIntrinsic::WriteOnly:
574 return "WriteOnly";
575 case CodeGenIntrinsic::ReadNone:
576 return "ReadNone";
577 case CodeGenIntrinsic::ImmArg:
578 return "ImmArg";
579 case CodeGenIntrinsic::Alignment:
580 return "Alignment";
581 case CodeGenIntrinsic::Dereferenceable:
582 return "Dereferenceable";
583 case CodeGenIntrinsic::Range:
584 return "Range";
585 case CodeGenIntrinsic::NoFreeObj:
586 return "NoFreeObj";
587 }
588 llvm_unreachable("Unknown CodeGenIntrinsic::ArgAttrKind enum");
589}
590
591/// EmitAttributes - This emits the Intrinsic::getAttributes method.
592void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
593 raw_ostream &OS) {
594 IfDefEmitter IfDef(OS, "GET_INTRINSIC_ATTRIBUTES");
595 OS << R"(// Add parameter attributes that are not common to all intrinsics.
596static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID,
597 Type *ArgType) {
598 unsigned BitWidth = ArgType->getScalarSizeInBits();
599 switch (ID) {
600 default: llvm_unreachable("Invalid attribute set number");)";
601 // Compute unique argument attribute sets.
602 std::map<SmallVector<CodeGenIntrinsic::ArgAttribute, 0>, unsigned>
603 UniqArgAttributes;
604 for (const CodeGenIntrinsic &Int : Ints) {
605 for (auto &Attrs : Int.ArgumentAttributes) {
606 if (Attrs.empty())
607 continue;
608
609 unsigned ID = UniqArgAttributes.size();
610 if (!UniqArgAttributes.try_emplace(k: Attrs, args&: ID).second)
611 continue;
612
613 assert(is_sorted(Attrs) && "Argument attributes are not sorted");
614
615 OS << formatv(Fmt: R"(
616 case {}:
617 return AttributeSet::get(C, {{
618)",
619 Vals&: ID);
620 for (const CodeGenIntrinsic::ArgAttribute &Attr : Attrs) {
621 if (Attr.Kind == CodeGenIntrinsic::NoCapture) {
622 OS << " Attribute::getWithCaptureInfo(C, "
623 "CaptureInfo::none()),\n";
624 continue;
625 }
626 StringRef AttrName = getArgAttrEnumName(Kind: Attr.Kind);
627 if (Attr.Kind == CodeGenIntrinsic::Alignment ||
628 Attr.Kind == CodeGenIntrinsic::Dereferenceable)
629 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}, {}),\n",
630 Vals&: AttrName, Vals: Attr.Value);
631 else if (Attr.Kind == CodeGenIntrinsic::Range)
632 // This allows implicitTrunc because the range may only fit the
633 // type based on rules implemented in the IR verifier. E.g. the
634 // [-1, 1] range for ucmp/scmp intrinsics requires a minimum i2 type.
635 // Give the verifier a chance to diagnose this instead of asserting
636 // here.
637 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}, "
638 "ConstantRange(APInt(BitWidth, {}, /*isSigned=*/true, "
639 "/*implicitTrunc=*/true), APInt(BitWidth, {}, "
640 "/*isSigned=*/true, /*implicitTrunc=*/true))),\n",
641 Vals&: AttrName, Vals: (int64_t)Attr.Value, Vals: (int64_t)Attr.Value2);
642 else
643 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}),\n", Vals&: AttrName);
644 }
645 OS << " });";
646 }
647 }
648 OS << R"(
649 }
650} // getIntrinsicArgAttributeSet
651)";
652
653 // Compute unique function attribute sets. Note that ID 255 will be used for
654 // intrinsics with no function attributes.
655 std::map<const CodeGenIntrinsic *, unsigned, FnAttributeComparator>
656 UniqFnAttributes;
657 OS << R"(
658static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
659 switch (ID) {
660 default: llvm_unreachable("Invalid attribute set number");)";
661
662 for (const CodeGenIntrinsic &Int : Ints) {
663 if (!hasFnAttributes(Int))
664 continue;
665 unsigned ID = UniqFnAttributes.size();
666 if (!UniqFnAttributes.try_emplace(k: &Int, args&: ID).second)
667 continue;
668 OS << formatv(Fmt: R"(
669 case {}: // {}
670 return AttributeSet::get(C, {{
671)",
672 Vals&: ID, Vals: Int.Name);
673 auto addAttribute = [&OS](StringRef Attr) {
674 OS << formatv(Fmt: " Attribute::get(C, Attribute::{}),\n", Vals&: Attr);
675 };
676 if (!Int.canThrow)
677 addAttribute("NoUnwind");
678 if (Int.isNoReturn)
679 addAttribute("NoReturn");
680 if (Int.isNoCallback)
681 addAttribute("NoCallback");
682 if (Int.isNoSync)
683 addAttribute("NoSync");
684 if (Int.isNoFree)
685 addAttribute("NoFree");
686 if (Int.isWillReturn)
687 addAttribute("WillReturn");
688 if (Int.isCold)
689 addAttribute("Cold");
690 if (Int.isNoDuplicate)
691 addAttribute("NoDuplicate");
692 if (Int.isNoMerge)
693 addAttribute("NoMerge");
694 if (Int.isConvergent)
695 addAttribute("Convergent");
696 if (Int.isSpeculatable)
697 addAttribute("Speculatable");
698 if (Int.isStrictFP)
699 addAttribute("StrictFP");
700 if (Int.isNoCreateUndefOrPoison)
701 addAttribute("NoCreateUndefOrPoison");
702
703 const MemoryEffects ME = getEffectiveME(Int);
704 if (ME != MemoryEffects::unknown()) {
705 OS << formatv(Fmt: " // {}\n", Vals: ME);
706 OS << formatv(Fmt: " Attribute::getWithMemoryEffects(C, "
707 "MemoryEffects::createFromIntValue({})),\n",
708 Vals: ME.toIntValue());
709 }
710 OS << " });";
711 }
712 OS << R"(
713 }
714} // getIntrinsicFnAttributeSet)";
715
716 // Compute unique argument attributes.
717 std::map<const CodeGenIntrinsic *, unsigned, AttributeComparator>
718 UniqAttributes;
719 for (const CodeGenIntrinsic &Int : Ints) {
720 unsigned ID = UniqAttributes.size();
721 UniqAttributes.try_emplace(k: &Int, args&: ID);
722 }
723
724 const uint8_t UniqAttributesBitSize = Log2_32_Ceil(Value: UniqAttributes.size());
725 // Note, max value is used to indicate no function attributes.
726 const uint8_t UniqFnAttributesBitSize =
727 Log2_32_Ceil(Value: UniqFnAttributes.size() + 1);
728 const uint32_t NoFunctionAttrsID =
729 maskTrailingOnes<uint32_t>(N: UniqFnAttributesBitSize);
730 uint8_t AttributesMapDataBitSize =
731 PowerOf2Ceil(A: UniqAttributesBitSize + UniqFnAttributesBitSize);
732 if (AttributesMapDataBitSize < 8)
733 AttributesMapDataBitSize = 8;
734 else if (AttributesMapDataBitSize > 64)
735 PrintFatalError(Msg: "Packed ID of IntrinsicsToAttributesMap exceeds 64b!");
736
737 // Assign a packed ID for each intrinsic. The lower bits will be its
738 // "argument attribute ID" (index in UniqAttributes) and upper bits will be
739 // its "function attribute ID" (index in UniqFnAttributes).
740 OS << formatv(Fmt: "\nstatic constexpr uint{}_t IntrinsicsToAttributesMap[] = {{",
741 Vals&: AttributesMapDataBitSize);
742 for (const CodeGenIntrinsic &Int : Ints) {
743 uint32_t FnAttrIndex =
744 hasFnAttributes(Int) ? UniqFnAttributes[&Int] : NoFunctionAttrsID;
745 OS << formatv(Fmt: "\n {} << {} | {}, // {}", Vals&: FnAttrIndex,
746 Vals: UniqAttributesBitSize, Vals&: UniqAttributes[&Int], Vals: Int.Name);
747 }
748
749 OS << R"(
750}; // IntrinsicsToAttributesMap
751)";
752
753 // For a given intrinsic, its attributes are constructed by populating the
754 // local array `AS` below with its non-empty argument attributes followed by
755 // function attributes if any. Each argument attribute is constructed as:
756 //
757 // getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo));
758 //
759 // Create a table that records, for each argument attributes, the list of
760 // <ArgNo, ArgAttrID> pairs that are needed to construct its argument
761 // attributes. These tables for all intrinsics will be concatenated into one
762 // large table and then for each intrinsic, we remember the Staring index and
763 // number of size of its slice of entries (i.e., number of arguments with
764 // non-empty attributes), so that we can build the attribute list for an
765 // intrinsic without using a switch-case.
766
767 using ArgNoAttrIDPair = std::pair<uint16_t, uint16_t>;
768
769 // Emit the table of concatenated <ArgNo, AttrId> using SequenceToOffsetTable
770 // so that entries can be reused if possible. Individual sequences in this
771 // table do not have any terminator.
772 using ArgAttrIDSubTable = SmallVector<ArgNoAttrIDPair>;
773 SequenceToOffsetTable<ArgAttrIDSubTable> ArgAttrIdSequenceTable(std::nullopt);
774 SmallVector<ArgAttrIDSubTable> ArgAttrIdSubTables(
775 UniqAttributes.size()); // Indexed by UniqueID.
776
777 // Find the max number of attributes to create the local array.
778 unsigned MaxNumAttrs = 0;
779 for (const auto [IntPtr, UniqueID] : UniqAttributes) {
780 const CodeGenIntrinsic &Int = *IntPtr;
781 ArgAttrIDSubTable SubTable;
782
783 for (const auto &[ArgNo, Attrs] : enumerate(First: Int.ArgumentAttributes)) {
784 if (Attrs.empty())
785 continue;
786
787 uint16_t ArgAttrID = UniqArgAttributes.find(x: Attrs)->second;
788 SubTable.emplace_back(Args: (uint16_t)ArgNo, Args&: ArgAttrID);
789 }
790 ArgAttrIdSubTables[UniqueID] = SubTable;
791 if (!SubTable.empty())
792 ArgAttrIdSequenceTable.add(Seq: SubTable);
793 unsigned NumAttrs = SubTable.size() + hasFnAttributes(Int);
794 MaxNumAttrs = std::max(a: MaxNumAttrs, b: NumAttrs);
795 }
796
797 ArgAttrIdSequenceTable.layout();
798
799 if (ArgAttrIdSequenceTable.size() >= std::numeric_limits<uint16_t>::max())
800 PrintFatalError(Msg: "Size of ArgAttrIdTable exceeds supported limit");
801
802 // Emit the 2 tables (flattened ArgNo, ArgAttrID) and ArgAttributesInfoTable.
803 OS << formatv(Fmt: R"(
804namespace {{
805struct ArgNoAttrIDPair {{
806 uint16_t ArgNo, ArgAttrID;
807};
808} // namespace
809
810// Number of entries: {}
811static constexpr ArgNoAttrIDPair ArgAttrIdTable[] = {{
812)",
813 Vals: ArgAttrIdSequenceTable.size());
814
815 ArgAttrIdSequenceTable.emit(OS, Print: [](raw_ostream &OS, ArgNoAttrIDPair Elem) {
816 OS << formatv(Fmt: "{{{}, {}}", Vals&: Elem.first, Vals&: Elem.second);
817 });
818
819 OS << formatv(Fmt: R"(}; // ArgAttrIdTable
820
821namespace {{
822struct ArgAttributesInfo {{
823 uint16_t StartIndex;
824 uint16_t NumAttrs;
825};
826} // namespace
827
828// Number of entries: {}
829static constexpr ArgAttributesInfo ArgAttributesInfoTable[] = {{
830)",
831 Vals: ArgAttrIdSubTables.size());
832
833 for (const auto &SubTable : ArgAttrIdSubTables) {
834 unsigned NumAttrs = SubTable.size();
835 unsigned StartIndex = NumAttrs ? ArgAttrIdSequenceTable.get(Seq: SubTable) : 0;
836 OS << formatv(Fmt: " {{{}, {}},\n", Vals&: StartIndex, Vals&: NumAttrs);
837 }
838 OS << "}; // ArgAttributesInfoTable\n";
839
840 // Now emit the Intrinsic::getAttributes function. This will first map
841 // from intrinsic ID -> unique arg/function attr ID (using the
842 // IntrinsicsToAttributesMap) table. Then it will use the unique arg ID to
843 // construct all the argument attributes (using the ArgAttributesInfoTable and
844 // ArgAttrIdTable) and then add on the function attributes if any.
845 OS << formatv(Fmt: R"(
846
847template <typename IDTy>
848inline std::pair<uint32_t, uint32_t> unpackID(const IDTy PackedID) {{
849 constexpr uint8_t UniqAttributesBitSize = {};
850 const uint32_t FnAttrID = PackedID >> UniqAttributesBitSize;
851 const uint32_t ArgAttrID = PackedID &
852 maskTrailingOnes<uint32_t>(UniqAttributesBitSize);
853 return {{FnAttrID, ArgAttrID};
854}
855
856AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id,
857 FunctionType *FT) {{
858 if (id == 0)
859 return AttributeList();
860 auto [FnAttrID, ArgAttrID] = unpackID(IntrinsicsToAttributesMap[id - 1]);
861 using PairTy = std::pair<unsigned, AttributeSet>;
862 alignas(PairTy) char ASStorage[sizeof(PairTy) * {}];
863 PairTy *AS = reinterpret_cast<PairTy *>(ASStorage);
864
865 // Construct an ArrayRef for easier range checking.
866 ArrayRef<ArgAttributesInfo> ArgAttributesInfoTableAR(ArgAttributesInfoTable);
867 if (ArgAttrID >= ArgAttributesInfoTableAR.size())
868 llvm_unreachable("Invalid arguments attribute ID");
869
870 auto [StartIndex, NumAttrs] = ArgAttributesInfoTableAR[ArgAttrID];
871 for (unsigned Idx = 0; Idx < NumAttrs; ++Idx) {{
872 auto [ArgNo, ArgAttrID] = ArgAttrIdTable[StartIndex + Idx];
873 AS[Idx] = {{ArgNo,
874 getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo))};
875 }
876 if (FnAttrID != {}) {
877 AS[NumAttrs++] = {{AttributeList::FunctionIndex,
878 getIntrinsicFnAttributeSet(C, FnAttrID)};
879 }
880 return AttributeList::get(C, ArrayRef(AS, NumAttrs));
881}
882
883AttributeSet Intrinsic::getFnAttributes(LLVMContext &C, ID id) {{
884 if (id == 0)
885 return AttributeSet();
886 auto [FnAttrID, _] = unpackID(IntrinsicsToAttributesMap[id - 1]);
887 if (FnAttrID == {})
888 return AttributeSet();
889 return getIntrinsicFnAttributeSet(C, FnAttrID);
890}
891)",
892 Vals: UniqAttributesBitSize, Vals&: MaxNumAttrs, Vals: NoFunctionAttrsID,
893 Vals: NoFunctionAttrsID);
894}
895
896void IntrinsicEmitter::EmitImmArgRangeSetChecks(
897 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
898 using ImmArgRangeSet = CodeGenIntrinsic::ImmArgRangeSet;
899 struct RangeSetInfo {
900 uint32_t ArgNo;
901 uint32_t RangesStart;
902 uint32_t NumRanges;
903 };
904 struct RangeSetListInfo {
905 uint32_t RangeSetsStart;
906 uint32_t NumRangeSets;
907 };
908
909 SequenceToOffsetTable<ImmArgRangeSet::RangeList> RangeTable(
910 /*Terminator=*/std::nullopt);
911
912 for (const CodeGenIntrinsic &Int : Ints) {
913 for (const ImmArgRangeSet &RangeSet : Int.ImmArgRangeSets)
914 RangeTable.add(Seq: RangeSet.Ranges);
915 }
916
917 IfDefEmitter IfDef(OS, "GET_INTRINSIC_IMMARG_RANGE_SET_CHECKS");
918 if (RangeTable.empty()) {
919 OS << R"(
920bool Intrinsic::isImmArgValueInRangeSet(ID IID, unsigned ArgIdx,
921 const APInt &Value) {
922 return true;
923}
924)";
925 return;
926 }
927
928 RangeTable.layout();
929
930 std::vector<RangeSetInfo> RangeSetInfos;
931 std::vector<RangeSetListInfo> PerIntrinsicRangeSetInfos;
932 PerIntrinsicRangeSetInfos.reserve(n: Ints.size() + 1);
933 PerIntrinsicRangeSetInfos.push_back(x: {.RangeSetsStart: 0, .NumRangeSets: 0}); // not_intrinsic
934
935 for (const CodeGenIntrinsic &Int : Ints) {
936 uint32_t Start = static_cast<uint32_t>(RangeSetInfos.size());
937 for (const ImmArgRangeSet &RangeSet : Int.ImmArgRangeSets)
938 RangeSetInfos.push_back(x: {.ArgNo: RangeSet.ArgNo, .RangesStart: RangeTable.get(Seq: RangeSet.Ranges),
939 .NumRanges: static_cast<uint32_t>(RangeSet.Ranges.size())});
940 PerIntrinsicRangeSetInfos.push_back(
941 x: {.RangeSetsStart: Start, .NumRangeSets: static_cast<uint32_t>(Int.ImmArgRangeSets.size())});
942 }
943
944 OS << R"(
945namespace {
946struct ImmArgRange {
947 int64_t Lower, Upper;
948};
949
950struct ImmArgRangeSet {
951 uint32_t ArgIdx;
952 uint32_t RangesStart;
953 uint32_t NumRanges;
954};
955
956struct ImmArgRangeSetList {
957 uint32_t RangeSetsStart;
958 uint32_t NumRangeSets;
959};
960} // namespace
961
962static constexpr ImmArgRange ImmArgRangeTable[] = {
963)";
964 RangeTable.emit(OS, Print: [](raw_ostream &OS, ImmArgRangeSet::Range Elem) {
965 OS << "{" << Elem.first << ", " << Elem.second << "}";
966 });
967 OS << R"(}; // ImmArgRangeTable
968
969static constexpr ImmArgRangeSet ImmArgRangeSetTable[] = {
970)";
971 for (const RangeSetInfo &Info : RangeSetInfos)
972 OS << " {" << Info.ArgNo << ", " << Info.RangesStart << ", "
973 << Info.NumRanges << "},\n";
974 OS << R"(}; // ImmArgRangeSetTable
975
976static constexpr ImmArgRangeSetList ImmArgRangeSetListTable[] = {
977)";
978 for (const auto &[Idx, Infos] : enumerate(First&: PerIntrinsicRangeSetInfos)) {
979 OS << " {" << Infos.RangeSetsStart << ", " << Infos.NumRangeSets << "},";
980 if (Idx == 0)
981 OS << " // not_intrinsic\n";
982 else
983 OS << " // " << Ints[Idx - 1].Name << "\n";
984 }
985 OS << R"(}; // ImmArgRangeSetListTable
986
987static bool isValueInImmArgRange(const ImmArgRange &Range,
988 const APInt &Value) {
989 unsigned BitWidth = Value.getBitWidth();
990 return ConstantRange(
991 APInt(BitWidth, Range.Lower, /*isSigned=*/true,
992 /*implicitTrunc=*/true),
993 APInt(BitWidth, Range.Upper, /*isSigned=*/true,
994 /*implicitTrunc=*/true))
995 .contains(Value);
996}
997
998bool Intrinsic::isImmArgValueInRangeSet(ID IID, unsigned ArgIdx,
999 const APInt &Value) {
1000 if (IID >= Intrinsic::num_intrinsics)
1001 return true;
1002
1003 const ImmArgRangeSetList &List = ImmArgRangeSetListTable[IID];
1004 ArrayRef<ImmArgRangeSet> RangeSets(
1005 ImmArgRangeSetTable + List.RangeSetsStart, List.NumRangeSets);
1006 for (const ImmArgRangeSet &RangeSet : RangeSets) {
1007 if (RangeSet.ArgIdx != ArgIdx)
1008 continue;
1009
1010 ArrayRef<ImmArgRange> Ranges(ImmArgRangeTable + RangeSet.RangesStart,
1011 RangeSet.NumRanges);
1012 for (const ImmArgRange &Range : Ranges) {
1013 if (isValueInImmArgRange(Range, Value))
1014 return true;
1015 }
1016 return false;
1017 }
1018 return true;
1019}
1020)";
1021}
1022
1023void IntrinsicEmitter::EmitIntrinsicToPrettyPrintTable(
1024 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
1025 EmitIntrinsicBitTable(Ints, OS, Guard: "GET_INTRINSIC_PRETTY_PRINT_TABLE", TableName: "PPTable",
1026 Comment: "Intrinsic ID to pretty print bitset.",
1027 GetProperty: [](const CodeGenIntrinsic &Int) {
1028 return !Int.PrettyPrintFunctions.empty();
1029 });
1030}
1031
1032void IntrinsicEmitter::EmitPrettyPrintArguments(
1033 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
1034 IfDefEmitter IfDef(OS, "GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS");
1035 OS << R"(
1036void Intrinsic::printImmArg(ID IID, unsigned ArgIdx, raw_ostream &OS, const Constant *ImmArgVal) {
1037 using namespace Intrinsic;
1038 switch (IID) {
1039)";
1040
1041 for (const CodeGenIntrinsic &Int : Ints) {
1042 if (Int.PrettyPrintFunctions.empty())
1043 continue;
1044
1045 OS << " case " << Int.EnumName << ":\n";
1046 OS << " switch (ArgIdx) {\n";
1047 for (const auto [ArgIdx, ArgName, FuncName] : Int.PrettyPrintFunctions) {
1048 OS << " case " << ArgIdx << ":\n";
1049 if (!ArgName.empty())
1050 OS << " OS << \"" << ArgName << "=\";\n";
1051 if (!FuncName.empty()) {
1052 OS << " ";
1053 if (!Int.TargetPrefix.empty())
1054 OS << Int.TargetPrefix << "::";
1055 OS << FuncName << "(OS, ImmArgVal);\n";
1056 }
1057 OS << " return;\n";
1058 }
1059 OS << " }\n";
1060 OS << " break;\n";
1061 }
1062 OS << R"( default:
1063 break;
1064 }
1065})";
1066}
1067
1068void IntrinsicEmitter::EmitDefaultArgValuesTable(
1069 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
1070 // Build the per-intrinsic default-value sequences:
1071 // [Header = (NumDefaults << 32) | FirstDefault, val0, val1, ...]
1072 // Each value is the (non-negative) default for one parameter, stored as a
1073 // uint64_t bit pattern.
1074 //
1075 // Offset 0 of the values table is reserved as the "no defaults" sentinel
1076 // (a single 0 word, decoding to NumDefaults = 0). Intrinsics without
1077 // defaults point to offset 0; real sequences are emitted after it.
1078 // SequenceToOffsetTable deduplicates the real sequences.
1079
1080 using Sequence = SmallVector<uint64_t, 8>;
1081
1082 SequenceToOffsetTable<Sequence> Table;
1083 // An empty Sequence means "no defaults" (maps to the reserved offset 0);
1084 // otherwise it holds the intrinsic's value sequence.
1085 SmallVector<Sequence> PerIntrinsic;
1086 PerIntrinsic.reserve(N: Ints.size());
1087
1088 for (const CodeGenIntrinsic &Int : Ints) {
1089 if (Int.ParamDefaultValues.empty()) {
1090 PerIntrinsic.push_back(Elt: {});
1091 continue;
1092 }
1093
1094 // Find the first parameter with a default.
1095 unsigned FirstDefault = 0;
1096 for (size_t j = 0U, N = Int.ParamDefaultValues.size(); j < N; ++j) {
1097 if (Int.ParamDefaultValues[j].has_value()) {
1098 FirstDefault = j;
1099 break;
1100 }
1101 }
1102 unsigned NumDefaults =
1103 static_cast<unsigned>(Int.ParamDefaultValues.size()) - FirstDefault;
1104
1105 Sequence Seq;
1106 Seq.push_back(Elt: (static_cast<uint64_t>(NumDefaults) << 32) | FirstDefault);
1107 for (size_t j = FirstDefault, N = Int.ParamDefaultValues.size(); j < N;
1108 ++j) {
1109 assert(Int.ParamDefaultValues[j].has_value() &&
1110 "Default block must be contiguous");
1111 Seq.push_back(Elt: *Int.ParamDefaultValues[j]);
1112 }
1113 Table.add(Seq);
1114 PerIntrinsic.push_back(Elt: std::move(Seq));
1115 }
1116
1117 Table.layout();
1118
1119 IfDefEmitter IfDef(OS, "GET_INTRINSIC_DEFAULT_ARG_VALUES");
1120
1121 // Emit the flat values table. Offset 0 is the reserved "no defaults"
1122 // sentinel; the deduplicated real sequences follow it.
1123 OS << "static constexpr uint64_t DefaultArgValuesTable[] = {\n";
1124 OS << " 0, // offset 0: sentinel for intrinsics without defaults\n";
1125 Table.emit(OS, Print: [](raw_ostream &OS, uint64_t Val) { OS << " " << Val; });
1126 OS << "};\n\n";
1127
1128 // Emit the per-intrinsic offset table. Entry #0 is for the invalid
1129 // Intrinsic::not_intrinsic (IID 0); it and every intrinsic without defaults
1130 // point to the reserved sentinel at offset 0. Real sequences are shifted by
1131 // +1 to skip past the sentinel slot.
1132 OS << "static constexpr uint32_t DefaultArgValuesTableOffset[] = {\n";
1133 OS << " 0, // not_intrinsic\n";
1134 for (const Sequence &Seq : PerIntrinsic) {
1135 if (!Seq.empty())
1136 OS << " " << (Table.get(Seq) + 1) << ",\n";
1137 else
1138 OS << " 0,\n";
1139 }
1140 OS << "};\n\n";
1141
1142 // Emit the lookup function body.
1143 OS << R"(
1144std::pair<unsigned, ArrayRef<uint64_t>>
1145Intrinsic::getAllDefaultArgValues(ID IID) {
1146 uint32_t Offset = DefaultArgValuesTableOffset[IID];
1147 uint64_t Header = DefaultArgValuesTable[Offset];
1148 uint32_t FirstDefault = Header & 0xFFFFFFFFu;
1149 uint32_t NumDefaults = (Header >> 32) & 0xFFFFFFFFu;
1150 if (NumDefaults == 0)
1151 return {0, {}};
1152 return {FirstDefault,
1153 ArrayRef(&DefaultArgValuesTable[Offset + 1], NumDefaults)};
1154}
1155)";
1156}
1157
1158void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
1159 const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) {
1160 StringRef CompilerName = IsClang ? "Clang" : "MS";
1161 StringRef UpperCompilerName = IsClang ? "CLANG" : "MS";
1162
1163 // map<TargetPrefix, pair<map<BuiltinName, EnumName>, CommonPrefix>.
1164 // Note that we iterate over both the maps in the code below and both
1165 // iterations need to iterate in sorted key order. For the inner map, entries
1166 // need to be emitted in the sorted order of `BuiltinName` with `CommonPrefix`
1167 // rempved, because we use std::lower_bound to search these entries. For the
1168 // outer map as well, entries need to be emitted in sorter order of
1169 // `TargetPrefix` as we use std::lower_bound to search these entries.
1170 using BIMEntryTy =
1171 std::pair<std::map<StringRef, StringRef>, std::optional<StringRef>>;
1172 std::map<StringRef, BIMEntryTy> BuiltinMap;
1173
1174 for (const CodeGenIntrinsic &Int : Ints) {
1175 StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName;
1176 if (BuiltinName.empty())
1177 continue;
1178 // Get the map for this target prefix.
1179 auto &[Map, CommonPrefix] = BuiltinMap[Int.TargetPrefix];
1180
1181 if (!Map.try_emplace(k: BuiltinName, args: Int.EnumName).second)
1182 PrintFatalError(ErrorLoc: Int.TheDef->getLoc(),
1183 Msg: "Intrinsic '" + Int.TheDef->getName() + "': duplicate " +
1184 CompilerName + " builtin name!");
1185
1186 // Update common prefix.
1187 if (!CommonPrefix) {
1188 // For the first builtin for this target, initialize the common prefix.
1189 CommonPrefix = BuiltinName;
1190 continue;
1191 }
1192
1193 // Update the common prefix. Note that this assumes that `take_front` will
1194 // never set the `Data` pointer in CommonPrefix to nullptr.
1195 const char *Mismatch = mismatch(Range1&: *CommonPrefix, Range2&: BuiltinName).first;
1196 *CommonPrefix = CommonPrefix->take_front(N: Mismatch - CommonPrefix->begin());
1197 }
1198
1199 // Populate the string table with the names of all the builtins after
1200 // removing this common prefix.
1201 StringToOffsetTable Table;
1202 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
1203 auto &[Map, CommonPrefix] = Entry;
1204 for (auto &[BuiltinName, EnumName] : Map) {
1205 StringRef Suffix = BuiltinName.substr(Start: CommonPrefix->size());
1206 Table.GetOrAddStringOffset(Str: Suffix);
1207 }
1208 }
1209
1210 IfDefEmitter IfDef(
1211 OS,
1212 formatv(Fmt: "GET_LLVM_INTRINSIC_FOR_{}_BUILTIN", Vals&: UpperCompilerName).str());
1213 OS << formatv(Fmt: R"(
1214// Get the LLVM intrinsic that corresponds to a builtin. This is used by the
1215// C front-end. The builtin name is passed in as BuiltinName, and a target
1216// prefix (e.g. 'ppc') is passed in as TargetPrefix.
1217Intrinsic::ID
1218Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
1219 StringRef BuiltinName) {{
1220 using namespace Intrinsic;
1221)",
1222 Vals&: CompilerName);
1223
1224 if (BuiltinMap.empty()) {
1225 OS << "return not_intrinsic;\n";
1226 return;
1227 }
1228
1229 if (!Table.empty()) {
1230 Table.EmitStringTableDef(OS, Name: "BuiltinNames");
1231
1232 OS << R"(
1233 struct BuiltinEntry {
1234 ID IntrinsicID;
1235 unsigned StrTabOffset;
1236 const char *getName() const { return BuiltinNames[StrTabOffset].data(); }
1237 bool operator<(StringRef RHS) const {
1238 return strncmp(getName(), RHS.data(), RHS.size()) < 0;
1239 }
1240 };
1241
1242)";
1243 }
1244
1245 // Emit a per target table of bultin names.
1246 bool HasTargetIndependentBuiltins = false;
1247 StringRef TargetIndepndentCommonPrefix;
1248 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
1249 const auto &[Map, CommonPrefix] = Entry;
1250 if (!TargetPrefix.empty()) {
1251 OS << formatv(Fmt: " // Builtins for {0}.\n", Vals: TargetPrefix);
1252 } else {
1253 OS << " // Target independent builtins.\n";
1254 HasTargetIndependentBuiltins = true;
1255 TargetIndepndentCommonPrefix = *CommonPrefix;
1256 }
1257
1258 // Emit the builtin table for this target prefix.
1259 OS << formatv(Fmt: " static constexpr BuiltinEntry {}Names[] = {{\n",
1260 Vals: TargetPrefix);
1261 for (const auto &[BuiltinName, EnumName] : Map) {
1262 StringRef Suffix = BuiltinName.substr(Start: CommonPrefix->size());
1263 OS << formatv(Fmt: " {{{}, {}}, // {}\n", Vals: EnumName,
1264 Vals: *Table.GetStringOffset(Str: Suffix), Vals: BuiltinName);
1265 }
1266 OS << formatv(Fmt: " }; // {}Names\n\n", Vals: TargetPrefix);
1267 }
1268
1269 // After emitting the builtin tables for all targets, emit a lookup table for
1270 // all targets. We will use binary search, similar to the table for builtin
1271 // names to lookup into this table.
1272 OS << R"(
1273 struct TargetEntry {
1274 StringLiteral TargetPrefix;
1275 ArrayRef<BuiltinEntry> Names;
1276 StringLiteral CommonPrefix;
1277 bool operator<(StringRef RHS) const {
1278 return TargetPrefix < RHS;
1279 };
1280 };
1281 static constexpr TargetEntry TargetTable[] = {
1282)";
1283
1284 for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
1285 const auto &[Map, CommonPrefix] = Entry;
1286 if (TargetPrefix.empty())
1287 continue;
1288 OS << formatv(Fmt: R"( {{"{0}", {0}Names, "{1}"},)", Vals: TargetPrefix,
1289 Vals: CommonPrefix)
1290 << "\n";
1291 }
1292 OS << " };\n";
1293
1294 // Now for the actual lookup, first check the target independent table if
1295 // we emitted one.
1296 if (HasTargetIndependentBuiltins) {
1297 OS << formatv(Fmt: R"(
1298 // Check if it's a target independent builtin.
1299 // Copy the builtin name so we can use it in consume_front without clobbering
1300 // if for the lookup in the target specific table.
1301 StringRef Suffix = BuiltinName;
1302 if (Suffix.consume_front("{}")) {{
1303 auto II = lower_bound(Names, Suffix);
1304 if (II != std::end(Names) && II->getName() == Suffix)
1305 return II->IntrinsicID;
1306 }
1307)",
1308 Vals&: TargetIndepndentCommonPrefix);
1309 }
1310
1311 // If a target independent builtin was not found, lookup the target specific.
1312 OS << R"(
1313 auto TI = lower_bound(TargetTable, TargetPrefix);
1314 if (TI == std::end(TargetTable) || TI->TargetPrefix != TargetPrefix)
1315 return not_intrinsic;
1316 // This is the last use of BuiltinName, so no need to copy before using it in
1317 // consume_front.
1318 if (!BuiltinName.consume_front(TI->CommonPrefix))
1319 return not_intrinsic;
1320 auto II = lower_bound(TI->Names, BuiltinName);
1321 if (II == std::end(TI->Names) || II->getName() != BuiltinName)
1322 return not_intrinsic;
1323 return II->IntrinsicID;
1324}
1325)";
1326}
1327
1328static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/true>>
1329 X("gen-intrinsic-enums", "Generate intrinsic enums");
1330
1331static TableGen::Emitter::OptClass<IntrinsicEmitterOpt</*Enums=*/false>>
1332 Y("gen-intrinsic-impl", "Generate intrinsic implementation code");
1333