1//===- RuntimeLibcallEmitter.cpp - Properties from RuntimeLibcalls.td -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#define DEBUG_TYPE "runtime-libcall-emitter"
10
11#include "RuntimeLibcalls.h"
12
13#include "SequenceToOffsetTable.h"
14#include "llvm/ADT/DenseSet.h"
15#include "llvm/ADT/MapVector.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/Format.h"
20#include "llvm/Support/FormatVariadic.h"
21#include "llvm/Support/raw_ostream.h"
22#include "llvm/Support/xxhash.h"
23#include "llvm/TableGen/CodeGenHelpers.h"
24#include "llvm/TableGen/Error.h"
25#include "llvm/TableGen/Record.h"
26#include "llvm/TableGen/SetTheory.h"
27#include "llvm/TableGen/StringToOffsetTable.h"
28#include "llvm/TableGen/TableGenBackend.h"
29
30using namespace llvm;
31
32namespace {
33// Pair of a RuntimeLibcallAvailability and LibcallCallingConv to use as a map
34// key.
35struct PredicateWithCC {
36 const Record *Availability = nullptr;
37 const Record *CallingConv = nullptr;
38
39 PredicateWithCC() = default;
40 PredicateWithCC(std::pair<const Record *, const Record *> P)
41 : Availability(P.first), CallingConv(P.second) {}
42
43 PredicateWithCC(const Record *P, const Record *C)
44 : Availability(P), CallingConv(C) {}
45};
46
47inline bool operator==(PredicateWithCC LHS, PredicateWithCC RHS) {
48 return LHS.Availability == RHS.Availability &&
49 LHS.CallingConv == RHS.CallingConv;
50}
51} // namespace
52
53namespace {
54/// A floating-point libcall family parsed from a RuntimeLibcallFamily record.
55struct FPLibcallFamily {
56 StringRef Base;
57 std::vector<StringRef> Intrinsics;
58 std::vector<StringRef> VectorSuffixes;
59
60 explicit FPLibcallFamily(const Record *R)
61 : Base(R->getValueAsString(FieldName: "LibcallBase")),
62 Intrinsics(R->getValueAsListOfStrings(FieldName: "Intrinsics")),
63 VectorSuffixes(R->getValueAsListOfStrings(FieldName: "VectorSuffixes")) {}
64};
65} // namespace
66
67namespace llvm {
68template <> struct DenseMapInfo<PredicateWithCC, void> {
69 static unsigned getHashValue(const PredicateWithCC Val) {
70 auto Pair = std::make_pair(x: Val.Availability, y: Val.CallingConv);
71 return DenseMapInfo<
72 std::pair<const Record *, const Record *>>::getHashValue(PairVal: Pair);
73 }
74
75 static bool isEqual(PredicateWithCC LHS, PredicateWithCC RHS) {
76 return LHS == RHS;
77 }
78};
79
80class RuntimeLibcallEmitter {
81private:
82 const RecordKeeper &Records;
83 RuntimeLibcalls Libcalls;
84
85 void emitGetRuntimeLibcallEnum(raw_ostream &OS) const;
86
87 void emitNameMatchHashTable(raw_ostream &OS,
88 StringToOffsetTable &OffsetTable) const;
89
90 void emitGetInitRuntimeLibcallNames(raw_ostream &OS) const;
91 void emitGetInitRuntimeLibcallSignatures(raw_ostream &OS) const;
92
93 // Emit the sorted per-predicate `setAvailable` tables/loops. The
94 // always-available bucket emits at \p BaseIndent; each predicated bucket is
95 // wrapped in `if (pred)`. All calls are emitted in member context (no
96 // receiver prefix).
97 void
98 emitPredicateGroups(raw_ostream &OS, const Record *R,
99 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
100 SetVector<PredicateWithCC> &PredicateSorter,
101 unsigned BaseIndent) const;
102
103 // A LibraryRef opt-out: the impls a consumer drops from a shared library,
104 // plus the consumer's triple predicate.
105 struct LibraryExclusion {
106 const Record *TriplePred;
107 std::vector<const RuntimeLibcallImpl *> Impls;
108 };
109
110 // A single LibcallLibrary variant, expanded into its per-predicate impl
111 // groups. Unconditional impls are tracked separately for cross-variant
112 // deduplication. A variant is Deferred when it re-adds an impl its own
113 // consumer excludes; deferred variants are emitted after the LibraryRef
114 // exclusions so the re-add wins over the opt-out.
115 struct ExpandedLibrary {
116 const Record *Lib;
117 DenseMap<PredicateWithCC, LibcallsWithCC> Pred2Funcs;
118 SetVector<PredicateWithCC> PredicateSorter;
119 SetVector<const RuntimeLibcallImpl *> Unconditional;
120 bool Deferred = false;
121 };
122
123 // Emit one variant's guarded `setAvailable` block into the enclosing
124 // `setAvailableLibFuncs_<name>` function.
125 void emitLibraryVariant(raw_ostream &OS, ExpandedLibrary &EL) const;
126
127 // Emit a `setAvailableLibFuncs_<name>` member function for all LibcallLibrary
128 // defs sharing \p Name, each gated by its own availability predicate. \p
129 // Exclusions are emitted as guarded setUnavailable calls at the end.
130 void emitLibraryFunction(raw_ostream &OS, StringRef Name,
131 ArrayRef<const Record *> Libs,
132 ArrayRef<LibraryExclusion> Exclusions) const;
133
134 // Group all LibcallLibrary defs by their shared LibraryName, preserving
135 // definition order. Both the member-declaration fragment and the definitions
136 // iterate this to stay in lockstep.
137 MapVector<StringRef, std::vector<const Record *>>
138 collectLibrariesByName() const;
139
140 void emitRuntimeLibcallsInfoMemberDecls(raw_ostream &OS) const;
141
142 void emitSystemRuntimeLibrarySetCalls(raw_ostream &OS) const;
143
144 DenseSet<StringRef> collectLibcallNames() const;
145
146 void checkFPLibcallFamilies(ArrayRef<FPLibcallFamily> Families,
147 const DenseSet<StringRef> &LibcallNames) const;
148
149 void emitFPLibcallSelectorDecls(raw_ostream &OS,
150 ArrayRef<FPLibcallFamily> Families) const;
151
152 void emitFPLibcallSelectors(raw_ostream &OS,
153 ArrayRef<FPLibcallFamily> Families,
154 const DenseSet<StringRef> &LibcallNames) const;
155
156 void
157 emitGetLibcallForIntrinsic(raw_ostream &OS,
158 ArrayRef<FPLibcallFamily> Families,
159 const DenseSet<StringRef> &LibcallNames) const;
160
161public:
162 RuntimeLibcallEmitter(const RecordKeeper &R) : Records(R), Libcalls(R) {}
163
164 void run(raw_ostream &OS);
165};
166
167} // End anonymous namespace.
168
169void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
170 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_ENUM");
171
172 OS << "namespace llvm {\n"
173 "namespace RTLIB {\n"
174 "enum Libcall : unsigned short {\n";
175
176 for (const RuntimeLibcall &LibCall : Libcalls.getRuntimeLibcallDefList()) {
177 StringRef Name = LibCall.getName();
178 OS << " " << Name << " = " << LibCall.getEnumVal() << ",\n";
179 }
180
181 OS << " UNKNOWN_LIBCALL = " << Libcalls.getRuntimeLibcallDefList().size()
182 << "\n};\n\n"
183 "enum LibcallImpl : unsigned short {\n"
184 " Unsupported = 0,\n";
185
186 for (const RuntimeLibcallImpl &LibCall :
187 Libcalls.getRuntimeLibcallImplDefList()) {
188 OS << " impl_" << LibCall.getName() << " = " << LibCall.getEnumVal()
189 << ", // " << LibCall.getLibcallFuncName() << '\n';
190 }
191
192 OS << "};\n"
193 << "constexpr size_t NumLibcallImpls = "
194 << Libcalls.getRuntimeLibcallImplDefList().size() + 1
195 << ";\n\n"
196 "enum FuncArgTypeID : char {\n"
197 " NoFuncArgType = 0,\n";
198
199 for (const auto *R : Libcalls.getFuncArgTypeList()) {
200 if (R->getName() == "NoneType")
201 continue;
202 OS << " " << R->getName() << ",\n";
203 }
204
205 OS << "};\n"
206 "} // End namespace RTLIB\n"
207 "} // End namespace llvm\n";
208}
209
210// StringMap uses xxh3_64bits, truncated to uint32_t.
211static uint64_t hash(StringRef Str) {
212 return static_cast<uint32_t>(xxh3_64bits(data: Str));
213}
214
215static void emitHashFunction(raw_ostream &OS) {
216 OS << "static inline uint64_t hash(StringRef Str) {\n"
217 " return static_cast<uint32_t>(xxh3_64bits(Str));\n"
218 "}\n\n";
219}
220
221/// Return the table size, maximum number of collisions for the set of hashes
222static std::pair<int, int>
223computePerfectHashParameters(ArrayRef<uint64_t> Hashes) {
224 // Chosen based on experimentation with llvm/benchmarks/RuntimeLibcalls.cpp
225 const int SizeOverhead = 4;
226
227 // Index derived from hash -> number of collisions.
228 DenseMap<uint64_t, int> Table;
229
230 unsigned NumHashes = Hashes.size();
231
232 for (int MaxCollisions = 1;; ++MaxCollisions) {
233 for (unsigned N = NextPowerOf2(A: NumHashes - 1); N < SizeOverhead * NumHashes;
234 N <<= 1) {
235 Table.clear();
236
237 bool NeedResize = false;
238 for (uint64_t H : Hashes) {
239 uint64_t Idx = H % static_cast<uint64_t>(N);
240 if (++Table[Idx] > MaxCollisions) {
241 // Need to resize the final table if we increased the collision count.
242 NeedResize = true;
243 break;
244 }
245 }
246
247 if (!NeedResize)
248 return {N, MaxCollisions};
249 }
250 }
251}
252
253static std::vector<unsigned>
254constructPerfectHashTable(ArrayRef<RuntimeLibcallImpl> Keywords,
255 ArrayRef<uint64_t> Hashes,
256 ArrayRef<unsigned> TableValues, int Size,
257 int Collisions, StringToOffsetTable &OffsetTable) {
258 std::vector<unsigned> Lookup(Size * Collisions);
259
260 for (auto [HashValue, TableValue] : zip(t&: Hashes, u&: TableValues)) {
261 uint64_t Idx = (HashValue % static_cast<uint64_t>(Size)) *
262 static_cast<uint64_t>(Collisions);
263
264 bool Found = false;
265 for (int J = 0; J < Collisions; ++J) {
266 unsigned &Entry = Lookup[Idx + J];
267 if (Entry == 0) {
268 Entry = TableValue;
269 Found = true;
270 break;
271 }
272 }
273
274 if (!Found)
275 reportFatalInternalError(reason: "failure to hash");
276 }
277
278 return Lookup;
279}
280
281/// Generate hash table based lookup by name.
282void RuntimeLibcallEmitter::emitNameMatchHashTable(
283 raw_ostream &OS, StringToOffsetTable &OffsetTable) const {
284 ArrayRef<RuntimeLibcallImpl> RuntimeLibcallImplDefList =
285 Libcalls.getRuntimeLibcallImplDefList();
286 std::vector<uint64_t> Hashes(RuntimeLibcallImplDefList.size());
287 std::vector<unsigned> TableValues(RuntimeLibcallImplDefList.size());
288 DenseSet<StringRef> SeenFuncNames;
289
290 size_t MaxFuncNameSize = 0;
291 size_t Index = 0;
292
293 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
294 StringRef ImplName = LibCallImpl.getLibcallFuncName();
295 if (SeenFuncNames.insert(V: ImplName).second) {
296 MaxFuncNameSize = std::max(a: MaxFuncNameSize, b: ImplName.size());
297 TableValues[Index] = LibCallImpl.getEnumVal();
298 Hashes[Index++] = hash(Str: ImplName);
299 }
300 }
301
302 // Trim excess elements from non-unique entries.
303 Hashes.resize(new_size: SeenFuncNames.size());
304 TableValues.resize(new_size: SeenFuncNames.size());
305
306 LLVM_DEBUG({
307 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
308 StringRef ImplName = LibCallImpl.getLibcallFuncName();
309 if (ImplName.size() == MaxFuncNameSize) {
310 dbgs() << "Maximum runtime libcall name size: " << ImplName << '('
311 << MaxFuncNameSize << ")\n";
312 }
313 }
314 });
315
316 // Early exiting on the symbol name provides a significant speedup in the miss
317 // case on the set of symbols in a clang binary. Emit this as an inlinable
318 // precondition in the header.
319 //
320 // The empty check is also used to get sensible behavior on anonymous
321 // functions.
322 //
323 // TODO: It may make more sense to split the search by string size more. There
324 // are a few outliers, most call names are small.
325 {
326 IfDefEmitter IfDef(OS, "GET_LOOKUP_LIBCALL_IMPL_NAME_BODY");
327
328 OS << " size_t Size = Name.size();\n"
329 " if (Size == 0 || Size > "
330 << MaxFuncNameSize
331 << ")\n"
332 " return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);\n"
333 " return lookupLibcallImplNameImpl(Name);\n";
334 }
335
336 auto [Size, Collisions] = computePerfectHashParameters(Hashes);
337 std::vector<unsigned> Lookup =
338 constructPerfectHashTable(Keywords: RuntimeLibcallImplDefList, Hashes, TableValues,
339 Size, Collisions, OffsetTable);
340
341 LLVM_DEBUG(dbgs() << "Runtime libcall perfect hashing parameters: Size = "
342 << Size << ", maximum collisions = " << Collisions << '\n');
343
344 IfDefEmitter IfDef(OS, "DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME");
345 emitHashFunction(OS);
346
347 OS << "iota_range<RTLIB::LibcallImpl> RTLIB::RuntimeLibcallsInfo::"
348 "lookupLibcallImplNameImpl(StringRef Name) {\n";
349
350 // Emit RTLIB::LibcallImpl values
351 OS << " static constexpr uint16_t HashTableNameToEnum[" << Lookup.size()
352 << "] = {\n";
353
354 for (unsigned TableVal : Lookup)
355 OS << " " << TableVal << ",\n";
356
357 OS << " };\n\n";
358
359 OS << " unsigned Idx = (hash(Name) % " << Size << ") * " << Collisions
360 << ";\n\n"
361 " for (int I = 0; I != "
362 << Collisions << R"(; ++I) {
363 const uint16_t Entry = HashTableNameToEnum[Idx + I];
364 const uint16_t StrOffset = RuntimeLibcallNameOffsetTable[Entry];
365 const uint8_t StrSize = RuntimeLibcallNameSizeTable[Entry];
366 StringRef Str(
367 &RTLIB::RuntimeLibcallsInfo::RuntimeLibcallImplNameTableStorage[StrOffset],
368 StrSize);
369 if (Str == Name)
370 return libcallImplNameHit(Entry, StrOffset);
371 }
372
373 return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);
374}
375)";
376}
377
378void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
379 raw_ostream &OS) const {
380 // Emit the implementation names
381 StringToOffsetTable Table(/*AppendZero=*/true,
382 "RTLIB::RuntimeLibcallsInfo::");
383
384 {
385 IfDefEmitter IfDef(OS, "GET_INIT_RUNTIME_LIBCALL_NAMES");
386
387 for (const RuntimeLibcallImpl &LibCallImpl :
388 Libcalls.getRuntimeLibcallImplDefList())
389 Table.GetOrAddStringOffset(Str: LibCallImpl.getLibcallFuncName());
390
391 Table.EmitStringTableDef(OS, Name: "RuntimeLibcallImplNameTable");
392 OS << R"(
393const uint16_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameOffsetTable[] = {
394)";
395
396 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ""),
397 Vals: ""); // Unsupported entry
398 for (const RuntimeLibcallImpl &LibCallImpl :
399 Libcalls.getRuntimeLibcallImplDefList()) {
400 StringRef ImplName = LibCallImpl.getLibcallFuncName();
401 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ImplName), Vals&: ImplName);
402 }
403 OS << "};\n";
404
405 OS << R"(
406const uint8_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameSizeTable[] = {
407)";
408
409 OS << " 0,\n";
410 for (const RuntimeLibcallImpl &LibCallImpl :
411 Libcalls.getRuntimeLibcallImplDefList())
412 OS << " " << LibCallImpl.getLibcallFuncName().size() << ",\n";
413 OS << "};\n\n";
414
415 // Emit the reverse mapping from implementation libraries to RTLIB::Libcall
416 OS << "const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::"
417 "ImplToLibcall[RTLIB::NumLibcallImpls] = {\n"
418 " RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported\n";
419
420 for (const RuntimeLibcallImpl &LibCallImpl :
421 Libcalls.getRuntimeLibcallImplDefList()) {
422 const RuntimeLibcall *Provides = LibCallImpl.getProvides();
423 OS << " ";
424 Provides->emitEnumEntry(OS);
425 OS << ", // ";
426 LibCallImpl.emitEnumEntry(OS);
427 OS << '\n';
428 }
429
430 OS << "};\n\n";
431 }
432
433 emitNameMatchHashTable(OS, OffsetTable&: Table);
434}
435
436using Signature = std::vector<StringRef>;
437
438static Signature getSignature(const Record *R) {
439 const auto *Tys = R->getValueAsListInit(FieldName: "ArgumentTypes");
440 Signature Sig;
441 Sig.reserve(n: Tys->size() + 1);
442 const Record *RetType = R->getValueAsOptionalDef(FieldName: "ReturnType");
443 if (RetType && RetType->getName() != "NoneType")
444 Sig.push_back(x: RetType->getName());
445 for (unsigned I = 0, E = Tys->size(); I < E; ++I)
446 Sig.push_back(x: Tys->getElementAsRecord(Idx: I)->getName());
447 return Sig;
448}
449
450void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallSignatures(
451 raw_ostream &OS) const {
452 SequenceToOffsetTable<Signature> SignatureTable("NoFuncArgType");
453
454 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList())
455 SignatureTable.add(Seq: getSignature(R: LC.getDef()));
456 SignatureTable.layout();
457
458 IfDefEmitter IfDef(OS, "GET_INIT_RUNTIME_LIBCALL_SIGNATURES");
459
460 OS << R"(
461const FuncArgTypeID RTLIB::RuntimeLibcallsInfo::SignatureTable[] = {
462)";
463 SignatureTable.emit(OS, Print: [](raw_ostream &OS, StringRef E) { OS << E; });
464 OS << "};\n";
465
466 OS << R"(
467const uint16_t RTLIB::RuntimeLibcallsInfo::SignatureOffset[] = {
468)";
469 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList()) {
470 const Record *LibcallDef = LC.getDef();
471 OS << formatv(Fmt: " {}, // {}\n", Vals: SignatureTable.get(Seq: getSignature(R: LibcallDef)),
472 Vals: LibcallDef->getName());
473 }
474 OS << "};\n";
475}
476
477void RuntimeLibcallEmitter::emitPredicateGroups(
478 raw_ostream &OS, const Record *R,
479 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
480 SetVector<PredicateWithCC> &PredicateSorter, unsigned BaseIndent) const {
481 SmallVector<PredicateWithCC, 0> SortedPredicates =
482 PredicateSorter.takeVector();
483
484 llvm::sort(C&: SortedPredicates, Comp: [](PredicateWithCC A, PredicateWithCC B) {
485 StringRef AName = A.Availability ? A.Availability->getName() : "";
486 StringRef BName = B.Availability ? B.Availability->getName() : "";
487 if (AName != BName)
488 return AName < BName;
489 // Break name ties on the calling convention for a deterministic order.
490 StringRef ACC = A.CallingConv ? A.CallingConv->getName() : "";
491 StringRef BCC = B.CallingConv ? B.CallingConv->getName() : "";
492 return ACC < BCC;
493 });
494
495 for (PredicateWithCC Entry : SortedPredicates) {
496 AvailabilityPredicate SubsetPredicate(Entry.Availability);
497 unsigned IndentDepth = BaseIndent;
498
499 auto It = Pred2Funcs.find(Val: Entry);
500 if (It == Pred2Funcs.end())
501 continue;
502
503 // Shared-core deduplication can empty a bucket.
504 if (It->second.LibcallImpls.empty())
505 continue;
506
507 if (!SubsetPredicate.isAlwaysAvailable()) {
508 IndentDepth = BaseIndent + 2;
509
510 OS << indent(IndentDepth);
511 SubsetPredicate.emitIf(OS);
512 }
513
514 LibcallsWithCC &FuncsWithCC = It->second;
515
516 std::vector<const RuntimeLibcallImpl *> &Funcs = FuncsWithCC.LibcallImpls;
517
518 // Records which impls are available, not which is selected, so a libcall
519 // may have more than one. Order is irrelevant (each entry is a setAvailable
520 // call); sort by the provided libcall, breaking ties on the impl enum for a
521 // deterministic total order.
522 llvm::sort(C&: Funcs, Comp: [](const RuntimeLibcallImpl *A,
523 const RuntimeLibcallImpl *B) {
524 return std::make_pair(x: A->getProvides()->getEnumVal(), y: A->getEnumVal()) <
525 std::make_pair(x: B->getProvides()->getEnumVal(), y: B->getEnumVal());
526 });
527
528 OS << indent(IndentDepth + 2)
529 << "static const RTLIB::LibcallImpl LibraryCalls";
530 SubsetPredicate.emitTableVariableNameSuffix(OS);
531 if (FuncsWithCC.CallingConv)
532 OS << '_' << FuncsWithCC.CallingConv->getName();
533
534 OS << "[] = {\n";
535 for (const RuntimeLibcallImpl *LibCallImpl : Funcs) {
536 OS << indent(IndentDepth + 6);
537 LibCallImpl->emitEnumEntry(OS);
538 OS << ", // " << LibCallImpl->getLibcallFuncName() << '\n';
539 }
540
541 OS << indent(IndentDepth + 2) << "};\n\n"
542 << indent(IndentDepth + 2)
543 << "for (const RTLIB::LibcallImpl Impl : LibraryCalls";
544 SubsetPredicate.emitTableVariableNameSuffix(OS);
545 if (FuncsWithCC.CallingConv)
546 OS << '_' << FuncsWithCC.CallingConv->getName();
547
548 OS << ") {\n" << indent(IndentDepth + 4) << "setAvailable(Impl);\n";
549
550 if (FuncsWithCC.CallingConv) {
551 StringRef CCEnum =
552 FuncsWithCC.CallingConv->getValueAsString(FieldName: "CallingConv");
553 OS << indent(IndentDepth + 4) << "setLibcallImplCallingConv(Impl, "
554 << CCEnum << ");\n";
555 }
556
557 OS << indent(IndentDepth + 2) << "}\n";
558 OS << '\n';
559
560 if (!SubsetPredicate.isAlwaysAvailable()) {
561 OS << indent(IndentDepth);
562 SubsetPredicate.emitEndIf(OS);
563 OS << '\n';
564 }
565 }
566}
567
568// Emit the linker name \p Name as a C++ identifier suffix, replacing characters
569// invalid in an identifier (e.g. the '-' in "compiler-rt") with '_'.
570static void emitLibFuncSuffix(raw_ostream &OS, StringRef Name) {
571 for (char C : Name)
572 OS << (isAlnum(C) || C == '_' ? C : '_');
573}
574
575void RuntimeLibcallEmitter::emitLibraryVariant(raw_ostream &OS,
576 ExpandedLibrary &EL) const {
577 AvailabilityPredicate LibPred(EL.Lib->getValueAsDef(FieldName: "Pred"));
578
579 if (!LibPred.isAlwaysAvailable()) {
580 OS << indent(2);
581 LibPred.emitIf(OS);
582 } else {
583 // Own block scope so per-variant `LibraryCalls` tables do not collide.
584 OS << indent(2) << "{\n";
585 }
586
587 emitPredicateGroups(OS, R: EL.Lib, Pred2Funcs&: EL.Pred2Funcs, PredicateSorter&: EL.PredicateSorter,
588 /*BaseIndent=*/2);
589
590 if (!LibPred.isAlwaysAvailable()) {
591 OS << indent(2);
592 LibPred.emitEndIf(OS);
593 } else {
594 OS << indent(2) << "}\n";
595 }
596}
597
598void RuntimeLibcallEmitter::emitLibraryFunction(
599 raw_ostream &OS, StringRef Name, ArrayRef<const Record *> Libs,
600 ArrayRef<LibraryExclusion> Exclusions) const {
601 OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibFuncs_";
602 emitLibFuncSuffix(OS, Name);
603 OS << "(const llvm::Triple &TT, "
604 "ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, "
605 "StringRef ABIName, "
606 "LongDoubleFormat LongDoubleFormat, CallingConv::ID DefaultCC) {\n";
607
608 SmallVector<ExpandedLibrary, 2> Expanded;
609 for (const Record *Lib : Libs) {
610 ExpandedLibrary EL;
611 EL.Lib = Lib;
612
613 // Expand this library's members with a library-local Func2Preds.
614 SetTheory Sets;
615 DenseMap<const RuntimeLibcallImpl *,
616 std::pair<std::vector<const Record *>, const Record *>>
617 Func2Preds;
618 Sets.addExpander(ClassName: "LibcallImpls", std::make_unique<LibcallPredicateExpander>(
619 args: Libcalls, args&: Func2Preds));
620
621 SetTheory::RecSet Elements;
622 Sets.evaluate(Expr: Lib->getValueInit(FieldName: "Impls"), Elts&: Elements, Loc: Lib->getLoc());
623
624 EL.PredicateSorter.insert(
625 X: PredicateWithCC()); // No predicate or CC override first.
626
627 for (const Record *Elt : Elements) {
628 const RuntimeLibcallImpl *LibCallImpl =
629 Libcalls.getRuntimeLibcallImpl(Def: Elt);
630 if (!LibCallImpl) {
631 PrintError(Rec: Lib, Msg: "entry for LibcallLibrary is not a RuntimeLibcallImpl");
632 PrintNote(NoteLoc: Elt->getLoc(), Msg: "invalid entry `" + Elt->getName() + "`");
633 continue;
634 }
635
636 auto It = Func2Preds.find(Val: LibCallImpl);
637 if (It == Func2Preds.end()) {
638 EL.Pred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: LibCallImpl);
639 EL.Unconditional.insert(X: LibCallImpl);
640 continue;
641 }
642
643 for (const Record *Pred : It->second.first) {
644 const Record *CC = It->second.second;
645 PredicateWithCC Key(Pred, CC);
646 auto &Entry = EL.Pred2Funcs[Key];
647 Entry.LibcallImpls.push_back(x: LibCallImpl);
648 Entry.CallingConv = CC;
649 EL.PredicateSorter.insert(X: Key);
650 }
651 }
652
653 Expanded.push_back(Elt: std::move(EL));
654 }
655
656 // Impls unconditional in every variant are emitted once and stripped from
657 // each variant, so the shared core is not repeated.
658 SetVector<const RuntimeLibcallImpl *> SharedCore;
659 if (Expanded.size() > 1) {
660 for (const RuntimeLibcallImpl *Impl : Expanded.front().Unconditional) {
661 if (all_of(Range: drop_begin(RangeOrContainer&: Expanded), P: [&](const ExpandedLibrary &EL) {
662 return EL.Unconditional.contains(key: Impl);
663 }))
664 SharedCore.insert(X: Impl);
665 }
666 }
667
668 if (!SharedCore.empty()) {
669 // Emit the shared core once, then strip it from every variant.
670 DenseMap<PredicateWithCC, LibcallsWithCC> CorePred2Funcs;
671 SetVector<PredicateWithCC> CoreSorter;
672 CoreSorter.insert(X: PredicateWithCC());
673 for (const RuntimeLibcallImpl *Impl : SharedCore)
674 CorePred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: Impl);
675 emitPredicateGroups(OS, R: Libs.front(), Pred2Funcs&: CorePred2Funcs, PredicateSorter&: CoreSorter,
676 /*BaseIndent=*/0);
677
678 for (ExpandedLibrary &EL : Expanded) {
679 auto &Funcs = EL.Pred2Funcs[PredicateWithCC()].LibcallImpls;
680 llvm::erase_if(C&: Funcs, P: [&](const RuntimeLibcallImpl *Impl) {
681 return SharedCore.contains(key: Impl);
682 });
683 }
684 }
685
686 // Mark a variant deferred when it re-adds an impl its own consumer excludes
687 // (same triple, via LibraryRef). Such a variant must be emitted after the
688 // exclusion so the re-add wins while the exclusion still suppresses every
689 // other variant's contribution.
690 for (ExpandedLibrary &EL : Expanded) {
691 const Record *ELPred = EL.Lib->getValueAsDef(FieldName: "Pred");
692 SetVector<const RuntimeLibcallImpl *> Impls;
693 for (const auto &[Key, Funcs] : EL.Pred2Funcs)
694 Impls.insert(Start: Funcs.LibcallImpls.begin(), End: Funcs.LibcallImpls.end());
695
696 for (const LibraryExclusion &Excl : Exclusions) {
697 if (Excl.TriplePred != ELPred)
698 continue;
699 if (any_of(Range: Excl.Impls, P: [&](const RuntimeLibcallImpl *Impl) {
700 return Impls.contains(key: Impl);
701 })) {
702 EL.Deferred = true;
703 break;
704 }
705 }
706 }
707
708 // Emit each non-deferred variant under its own Pred.
709 for (ExpandedLibrary &EL : Expanded) {
710 if (!EL.Deferred)
711 emitLibraryVariant(OS, EL);
712 }
713
714 // Emit each consumer's LibraryRef opt-outs.
715 for (const LibraryExclusion &Excl : Exclusions) {
716 OS << '\n' << indent(2);
717 AvailabilityPredicate ExcludePred(Excl.TriplePred);
718 ExcludePred.emitIf(OS);
719 for (const RuntimeLibcallImpl *Impl : Excl.Impls) {
720 OS << indent(4) << "setUnavailable(";
721 Impl->emitEnumEntry(OS);
722 OS << "); // " << Impl->getLibcallFuncName() << '\n';
723 }
724
725 OS << indent(2);
726 ExcludePred.emitEndIf(OS);
727 }
728
729 // Deferred variants: emitted after exclusions so a target's own re-adds
730 // override its own LibraryRef opt-outs (the exclusion still applied above
731 // to every other variant's contributions).
732 for (ExpandedLibrary &EL : Expanded) {
733 if (EL.Deferred)
734 emitLibraryVariant(OS, EL);
735 }
736
737 OS << "}\n\n";
738}
739
740// The setAvailableLibFuncs_ suffix (and merge key) for a library: its shared
741// LibraryName normally, or its own def name if isolated (so it does not merge).
742static StringRef libFuncKey(const Record *Lib) {
743 return Lib->getValueAsBit(FieldName: "Isolated") ? Lib->getName()
744 : Lib->getValueAsString(FieldName: "LibraryName");
745}
746
747MapVector<StringRef, std::vector<const Record *>>
748RuntimeLibcallEmitter::collectLibrariesByName() const {
749 MapVector<StringRef, std::vector<const Record *>> LibsByName;
750 for (const Record *Lib : Records.getAllDerivedDefinitions(ClassName: "LibcallLibrary"))
751 LibsByName[libFuncKey(Lib)].push_back(x: Lib);
752 return LibsByName;
753}
754
755void RuntimeLibcallEmitter::emitRuntimeLibcallsInfoMemberDecls(
756 raw_ostream &OS) const {
757 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALLS_INFO_MEMBER_DECLS");
758 for (const auto &[Name, Libs] : collectLibrariesByName()) {
759 OS << "void setAvailableLibFuncs_";
760 emitLibFuncSuffix(OS, Name);
761 OS << "(const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
762 "FloatABI::ABIType FloatABI, StringRef ABIName, "
763 "LongDoubleFormat LongDoubleFormat, CallingConv::ID DefaultCC);\n";
764 }
765}
766
767void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
768 raw_ostream &OS) const {
769 ArrayRef<const Record *> AllLibs =
770 Records.getAllDerivedDefinitions(ClassName: "SystemRuntimeLibrary");
771
772 // Collect each shared library's LibraryRef opt-outs, keyed by library name,
773 // so its library function can emit them.
774 MapVector<StringRef, std::vector<LibraryExclusion>> ExclusionsByLibName;
775 for (const Record *R : AllLibs) {
776 const DagInit *MemberDag =
777 R->getValueAsDef(FieldName: "MemberList")->getValueAsDag(FieldName: "MemberList");
778 for (const Init *Arg : MemberDag->getArgs()) {
779 const auto *DI = dyn_cast<DefInit>(Val: Arg);
780 if (!DI || !DI->getDef()->isSubClassOf(Name: "LibraryRef"))
781 continue;
782 const Record *Def = DI->getDef();
783 LibraryExclusion Excl{.TriplePred: R->getValueAsDef(FieldName: "TriplePred"), .Impls: {}};
784 for (const Record *ExcludeRec : Def->getValueAsListOfDefs(FieldName: "Exclude")) {
785 if (const RuntimeLibcallImpl *Impl =
786 Libcalls.getRuntimeLibcallImpl(Def: ExcludeRec))
787 Excl.Impls.push_back(x: Impl);
788 }
789
790 if (!Excl.Impls.empty()) {
791 StringRef LibName =
792 Def->getValueAsDef(FieldName: "Library")->getValueAsString(FieldName: "LibraryName");
793 ExclusionsByLibName[LibName].push_back(x: std::move(Excl));
794 }
795 }
796 }
797
798 for (const auto &[Name, Libs] : collectLibrariesByName())
799 emitLibraryFunction(OS, Name, Libs, Exclusions: ExclusionsByLibName.lookup(Key: Name));
800
801 OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets("
802 "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
803 "FloatABI::ABIType FloatABI, "
804 "StringRef ABIName, LongDoubleFormat LongDoubleFormat) {\n";
805
806 for (const Record *R : AllLibs) {
807 OS << '\n';
808
809 AvailabilityPredicate TopLevelPredicate(R->getValueAsDef(FieldName: "TriplePred"));
810
811 OS << indent(2);
812 TopLevelPredicate.emitIf(OS);
813
814 // The default is a property of the target, and is passed to the libraries
815 // as an argument.
816 StringRef DefaultCCArg = "CallingConv::C";
817 if (const Record *DefaultCCClass =
818 R->getValueAsDef(FieldName: "DefaultLibcallCallingConv")) {
819 StringRef DefaultCC =
820 DefaultCCClass->getValueAsString(FieldName: "CallingConv").trim();
821
822 if (!DefaultCC.empty()) {
823 OS << " const CallingConv::ID DefaultCC = " << DefaultCC << ";\n"
824 << " for (CallingConv::ID &Entry : LibcallImplCallingConvs) {\n"
825 " Entry = DefaultCC;\n"
826 " }\n\n";
827 DefaultCCArg = "DefaultCC";
828 }
829 }
830
831 // Dispatch each library to setAvailableLibFuncs_<FuncSuffix> under an
832 // isLibraryAvailable(Name) guard. LibraryRef exclusions are applied inside
833 // the library function. A library referenced more than once is dispatched
834 // once.
835 const DagInit *MemberDag =
836 R->getValueAsDef(FieldName: "MemberList")->getValueAsDag(FieldName: "MemberList");
837 SetVector<std::pair<StringRef, StringRef>> DispatchLibs;
838 for (const Init *Arg : MemberDag->getArgs()) {
839 const auto *DI = dyn_cast<DefInit>(Val: Arg);
840 const Record *Lib = DI ? DI->getDef() : nullptr;
841 if (Lib && Lib->isSubClassOf(Name: "LibraryRef"))
842 Lib = Lib->getValueAsDef(FieldName: "Library");
843 if (!Lib || !Lib->isSubClassOf(Name: "LibcallLibrary")) {
844 PrintError(Rec: R, Msg: "SystemRuntimeLibrary member `" + Arg->getAsString() +
845 "` is not a LibcallLibrary or LibraryRef");
846 continue;
847 }
848 DispatchLibs.insert(
849 X: {Lib->getValueAsString(FieldName: "LibraryName"), libFuncKey(Lib)});
850 }
851
852 for (auto [Name, FuncSuffix] : DispatchLibs) {
853 OS << indent(4) << "if (isLibraryAvailable(\"" << Name << "\"))\n"
854 << indent(6) << "setAvailableLibFuncs_";
855 emitLibFuncSuffix(OS, Name: FuncSuffix);
856 OS << "(TT, ExceptionModel, FloatABI, ABIName, LongDoubleFormat, "
857 << DefaultCCArg << ");\n";
858 }
859 if (!DispatchLibs.empty())
860 OS << '\n';
861
862 OS << indent(4) << "return;\n" << indent(2);
863 TopLevelPredicate.emitEndIf(OS);
864 }
865
866 // FIXME: This should be a fatal error. A few contexts are improperly relying
867 // on RuntimeLibcalls constructed with fully unknown triples.
868 OS << " LLVM_DEBUG(dbgs() << \"no system runtime library applied to target "
869 "\\'\" << TT.str() << \"\\'\\n\");\n"
870 "}\n\n";
871}
872
873// Scalar FP type suffixes in the argument order of RTLIB::getFPLibCall, paired
874// with the llvm::Type predicate used by the IR-level mapping.
875static constexpr std::pair<StringRef, StringRef> ScalarFPSuffixes[] = {
876 {"F32", "isFloatTy()"}, {"F64", "isDoubleTy()"},
877 {"F80", "isX86_FP80Ty()"}, {"F128", "isFP128Ty()"},
878 {"PPCF128", "isPPC_FP128Ty()"},
879};
880
881static std::vector<FPLibcallFamily>
882collectFPLibcallFamilies(const RecordKeeper &Records) {
883 std::vector<FPLibcallFamily> Families;
884 for (const Record *R :
885 Records.getAllDerivedDefinitions(ClassName: "RuntimeLibcallFamily"))
886 Families.emplace_back(args&: R);
887 llvm::sort(C&: Families, Comp: [](const FPLibcallFamily &A, const FPLibcallFamily &B) {
888 return A.Base < B.Base;
889 });
890 return Families;
891}
892
893DenseSet<StringRef> RuntimeLibcallEmitter::collectLibcallNames() const {
894 DenseSet<StringRef> LibcallNames;
895 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList())
896 LibcallNames.insert(V: LC.getName());
897 return LibcallNames;
898}
899
900void RuntimeLibcallEmitter::checkFPLibcallFamilies(
901 ArrayRef<FPLibcallFamily> Families,
902 const DenseSet<StringRef> &LibcallNames) const {
903 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
904 for (const FPLibcallFamily &Family : Families)
905 for (StringRef Intrinsic : Family.Intrinsics)
906 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
907 llvm::sort(C&: IntrinsicToBase);
908
909 for (size_t I = 1, E = IntrinsicToBase.size(); I < E; ++I) {
910 if (IntrinsicToBase[I].first == IntrinsicToBase[I - 1].first)
911 PrintFatalError(
912 Msg: "intrinsic '" + IntrinsicToBase[I].first +
913 "' is mapped by multiple RuntimeLibcallFamily records ('" +
914 IntrinsicToBase[I - 1].second + "' and '" +
915 IntrinsicToBase[I].second + "')");
916 }
917
918 for (const FPLibcallFamily &Family : Families) {
919 bool AnyScalarLibcall = any_of(
920 Range: ScalarFPSuffixes, P: [&](const std::pair<StringRef, StringRef> &Entry) {
921 return LibcallNames.contains(V: (Family.Base + "_" + Entry.first).str());
922 });
923 if (!AnyScalarLibcall)
924 PrintFatalError(Msg: "no runtime libcall found for base name '" + Family.Base +
925 "'");
926 }
927}
928
929/// Generate the declarations for the RTLIB::get<base>(EVT) selectors.
930void RuntimeLibcallEmitter::emitFPLibcallSelectorDecls(
931 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families) const {
932 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTOR_DECLS");
933 for (const FPLibcallFamily &Family : Families)
934 OS << "LLVM_ABI Libcall get" << Family.Base << "(EVT VT);\n";
935}
936
937/// Generate the backend RTLIB::get<base>(EVT) selectors from the floating-point
938/// libcall families.
939void RuntimeLibcallEmitter::emitFPLibcallSelectors(
940 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
941 const DenseSet<StringRef> &LibcallNames) const {
942 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTORS");
943
944 // Only emit a libcall enumerator if it actually exists in the declared set.
945 auto scalarEnum = [&](StringRef Base, StringRef Suffix) -> std::string {
946 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
947 return ("RTLIB::" + Base + "_" + Suffix).str();
948 return "RTLIB::UNKNOWN_LIBCALL";
949 };
950
951 for (const FPLibcallFamily &Family : Families) {
952 StringRef Base = Family.Base;
953 OS << "RTLIB::Libcall llvm::RTLIB::get" << Base << "(EVT VT) {\n";
954
955 if (!Family.VectorSuffixes.empty()) {
956 OS << " if (VT.isVector()) {\n"
957 " if (!VT.isSimple())\n"
958 " return RTLIB::UNKNOWN_LIBCALL;\n"
959 " switch (VT.getSimpleVT().SimpleTy) {\n";
960 for (StringRef Suffix : Family.VectorSuffixes) {
961 OS << " case MVT::" << Suffix.lower()
962 << ":\n return RTLIB::" << Base << "_" << Suffix << ";\n";
963 }
964 OS << " default:\n"
965 " return RTLIB::UNKNOWN_LIBCALL;\n"
966 " }\n"
967 " }\n";
968 }
969
970 OS << " return getFPLibCall(VT";
971 for (auto [Suffix, Pred] : ScalarFPSuffixes)
972 OS << ", " << scalarEnum(Base, Suffix);
973 OS << ");\n}\n\n";
974 }
975}
976
977/// Emit the mapping from floating-point math intrinsics to the runtime libcall
978/// they may lower to, keyed by intrinsic ID and floating-point type. This is
979/// the IR-level counterpart to the backend's RTLIB::getXXX(EVT) selectors.
980void RuntimeLibcallEmitter::emitGetLibcallForIntrinsic(
981 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
982 const DenseSet<StringRef> &LibcallNames) const {
983 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_INTRINSIC_TO_LIBCALL");
984
985 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
986 for (const FPLibcallFamily &Family : Families)
987 for (StringRef Intrinsic : Family.Intrinsics)
988 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
989 llvm::sort(C&: IntrinsicToBase);
990
991 MapVector<StringRef, SmallVector<StringRef, 2>> BaseToIntrinsics;
992 for (auto [Intrinsic, Base] : IntrinsicToBase)
993 BaseToIntrinsics[Base].push_back(Elt: Intrinsic);
994
995 OS << "RTLIB::Libcall "
996 "llvm::RTLIB::RuntimeLibcallsInfo::getLibcallForIntrinsic("
997 "Intrinsic::ID ID, FunctionType *FTy) {\n"
998 " Type *Ty = FTy->getReturnType();\n"
999 " if (!Ty->isFloatingPointTy()) {\n"
1000 " for (Type *ParamTy : FTy->params()) {\n"
1001 " if (ParamTy->isFloatingPointTy()) {\n"
1002 " Ty = ParamTy;\n"
1003 " break;\n"
1004 " }\n"
1005 " }\n"
1006 " }\n"
1007 " if (!Ty->isFloatingPointTy())\n"
1008 " return RTLIB::UNKNOWN_LIBCALL;\n"
1009 " switch (ID) {\n";
1010
1011 for (const auto &[Base, Intrinsics] : BaseToIntrinsics) {
1012 SmallVector<std::pair<StringRef, StringRef>, 5> Arms;
1013 for (auto [Suffix, Pred] : ScalarFPSuffixes)
1014 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
1015 Arms.emplace_back(Args&: Suffix, Args&: Pred);
1016
1017 for (StringRef Intrinsic : Intrinsics)
1018 OS << " case Intrinsic::" << Intrinsic << ":\n";
1019 for (auto [Suffix, Pred] : Arms)
1020 OS << " if (Ty->" << Pred << ")\n return RTLIB::" << Base << "_"
1021 << Suffix << ";\n";
1022 OS << " return RTLIB::UNKNOWN_LIBCALL;\n";
1023 }
1024
1025 OS << " default:\n"
1026 " return RTLIB::UNKNOWN_LIBCALL;\n"
1027 " }\n"
1028 "}\n";
1029}
1030
1031void RuntimeLibcallEmitter::run(raw_ostream &OS) {
1032 emitSourceFileHeader(Desc: "Runtime LibCalls Source Fragment", OS, Record: Records);
1033 emitGetRuntimeLibcallEnum(OS);
1034
1035 emitGetInitRuntimeLibcallNames(OS);
1036 emitGetInitRuntimeLibcallSignatures(OS);
1037
1038 emitRuntimeLibcallsInfoMemberDecls(OS);
1039
1040 {
1041 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALLS_INFO");
1042 emitSystemRuntimeLibrarySetCalls(OS);
1043 }
1044
1045 std::vector<FPLibcallFamily> FPFamilies = collectFPLibcallFamilies(Records);
1046 DenseSet<StringRef> LibcallNames = collectLibcallNames();
1047 checkFPLibcallFamilies(Families: FPFamilies, LibcallNames);
1048 emitFPLibcallSelectorDecls(OS, Families: FPFamilies);
1049 emitFPLibcallSelectors(OS, Families: FPFamilies, LibcallNames);
1050 emitGetLibcallForIntrinsic(OS, Families: FPFamilies, LibcallNames);
1051}
1052
1053static TableGen::Emitter::OptClass<RuntimeLibcallEmitter>
1054 X("gen-runtime-libcalls", "Generate RuntimeLibcalls");
1055