1//===- RuntimeLibcallEmitter.cpp - Properties from RuntimeLibcalls.td -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#define DEBUG_TYPE "runtime-libcall-emitter"
10
11#include "RuntimeLibcalls.h"
12
13#include "llvm/ADT/DenseSet.h"
14#include "llvm/ADT/MapVector.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/Support/Debug.h"
18#include "llvm/Support/Format.h"
19#include "llvm/Support/FormatVariadic.h"
20#include "llvm/Support/raw_ostream.h"
21#include "llvm/Support/xxhash.h"
22#include "llvm/TableGen/CodeGenHelpers.h"
23#include "llvm/TableGen/Error.h"
24#include "llvm/TableGen/Record.h"
25#include "llvm/TableGen/SetTheory.h"
26#include "llvm/TableGen/StringToOffsetTable.h"
27#include "llvm/TableGen/TableGenBackend.h"
28
29using namespace llvm;
30
31namespace {
32// Pair of a RuntimeLibcallAvailability and LibcallCallingConv to use as a map
33// key.
34struct PredicateWithCC {
35 const Record *Availability = nullptr;
36 const Record *CallingConv = nullptr;
37
38 PredicateWithCC() = default;
39 PredicateWithCC(std::pair<const Record *, const Record *> P)
40 : Availability(P.first), CallingConv(P.second) {}
41
42 PredicateWithCC(const Record *P, const Record *C)
43 : Availability(P), CallingConv(C) {}
44};
45
46inline bool operator==(PredicateWithCC LHS, PredicateWithCC RHS) {
47 return LHS.Availability == RHS.Availability &&
48 LHS.CallingConv == RHS.CallingConv;
49}
50} // namespace
51
52namespace {
53/// A floating-point libcall family parsed from a RuntimeLibcallFamily record.
54struct FPLibcallFamily {
55 StringRef Base;
56 std::vector<StringRef> Intrinsics;
57 std::vector<StringRef> VectorSuffixes;
58
59 explicit FPLibcallFamily(const Record *R)
60 : Base(R->getValueAsString(FieldName: "LibcallBase")),
61 Intrinsics(R->getValueAsListOfStrings(FieldName: "Intrinsics")),
62 VectorSuffixes(R->getValueAsListOfStrings(FieldName: "VectorSuffixes")) {}
63};
64} // namespace
65
66namespace llvm {
67template <> struct DenseMapInfo<PredicateWithCC, void> {
68 static unsigned getHashValue(const PredicateWithCC Val) {
69 auto Pair = std::make_pair(x: Val.Availability, y: Val.CallingConv);
70 return DenseMapInfo<
71 std::pair<const Record *, const Record *>>::getHashValue(PairVal: Pair);
72 }
73
74 static bool isEqual(PredicateWithCC LHS, PredicateWithCC RHS) {
75 return LHS == RHS;
76 }
77};
78
79class RuntimeLibcallEmitter {
80private:
81 const RecordKeeper &Records;
82 RuntimeLibcalls Libcalls;
83
84 void emitGetRuntimeLibcallEnum(raw_ostream &OS) const;
85
86 void emitNameMatchHashTable(raw_ostream &OS,
87 StringToOffsetTable &OffsetTable) const;
88
89 void emitGetInitRuntimeLibcallNames(raw_ostream &OS) const;
90
91 // Emit the sorted per-predicate `setAvailable` tables/loops. The
92 // always-available bucket emits at \p BaseIndent; each predicated bucket is
93 // wrapped in `if (pred)`. \p Receiver prefixes the calls ("Info." for the
94 // standalone library functions, empty otherwise).
95 void
96 emitPredicateGroups(raw_ostream &OS, const Record *R,
97 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
98 SetVector<PredicateWithCC> &PredicateSorter,
99 unsigned BaseIndent, StringRef Receiver) const;
100
101 // Emit a file-local `setAvailableLibFuncs_<name>` for all LibcallLibrary defs
102 // sharing \p Name, each gated by its own availability predicate.
103 void emitLibraryFunction(raw_ostream &OS, StringRef Name,
104 ArrayRef<const Record *> Libs) const;
105
106 void emitSystemRuntimeLibrarySetCalls(raw_ostream &OS) const;
107
108 DenseSet<StringRef> collectLibcallNames() const;
109
110 void checkFPLibcallFamilies(ArrayRef<FPLibcallFamily> Families,
111 const DenseSet<StringRef> &LibcallNames) const;
112
113 void emitFPLibcallSelectorDecls(raw_ostream &OS,
114 ArrayRef<FPLibcallFamily> Families) const;
115
116 void emitFPLibcallSelectors(raw_ostream &OS,
117 ArrayRef<FPLibcallFamily> Families,
118 const DenseSet<StringRef> &LibcallNames) const;
119
120 void
121 emitGetLibcallForIntrinsic(raw_ostream &OS,
122 ArrayRef<FPLibcallFamily> Families,
123 const DenseSet<StringRef> &LibcallNames) const;
124
125public:
126 RuntimeLibcallEmitter(const RecordKeeper &R) : Records(R), Libcalls(R) {}
127
128 void run(raw_ostream &OS);
129};
130
131} // End anonymous namespace.
132
133void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
134 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_ENUM");
135
136 OS << "namespace llvm {\n"
137 "namespace RTLIB {\n"
138 "enum Libcall : unsigned short {\n";
139
140 for (const RuntimeLibcall &LibCall : Libcalls.getRuntimeLibcallDefList()) {
141 StringRef Name = LibCall.getName();
142 OS << " " << Name << " = " << LibCall.getEnumVal() << ",\n";
143 }
144
145 OS << " UNKNOWN_LIBCALL = " << Libcalls.getRuntimeLibcallDefList().size()
146 << "\n};\n\n"
147 "enum LibcallImpl : unsigned short {\n"
148 " Unsupported = 0,\n";
149
150 for (const RuntimeLibcallImpl &LibCall :
151 Libcalls.getRuntimeLibcallImplDefList()) {
152 OS << " impl_" << LibCall.getName() << " = " << LibCall.getEnumVal()
153 << ", // " << LibCall.getLibcallFuncName() << '\n';
154 }
155
156 OS << "};\n"
157 << "constexpr size_t NumLibcallImpls = "
158 << Libcalls.getRuntimeLibcallImplDefList().size() + 1
159 << ";\n"
160 "} // End namespace RTLIB\n"
161 "} // End namespace llvm\n";
162}
163
164// StringMap uses xxh3_64bits, truncated to uint32_t.
165static uint64_t hash(StringRef Str) {
166 return static_cast<uint32_t>(xxh3_64bits(data: Str));
167}
168
169static void emitHashFunction(raw_ostream &OS) {
170 OS << "static inline uint64_t hash(StringRef Str) {\n"
171 " return static_cast<uint32_t>(xxh3_64bits(Str));\n"
172 "}\n\n";
173}
174
175/// Return the table size, maximum number of collisions for the set of hashes
176static std::pair<int, int>
177computePerfectHashParameters(ArrayRef<uint64_t> Hashes) {
178 // Chosen based on experimentation with llvm/benchmarks/RuntimeLibcalls.cpp
179 const int SizeOverhead = 4;
180
181 // Index derived from hash -> number of collisions.
182 DenseMap<uint64_t, int> Table;
183
184 unsigned NumHashes = Hashes.size();
185
186 for (int MaxCollisions = 1;; ++MaxCollisions) {
187 for (unsigned N = NextPowerOf2(A: NumHashes - 1); N < SizeOverhead * NumHashes;
188 N <<= 1) {
189 Table.clear();
190
191 bool NeedResize = false;
192 for (uint64_t H : Hashes) {
193 uint64_t Idx = H % static_cast<uint64_t>(N);
194 if (++Table[Idx] > MaxCollisions) {
195 // Need to resize the final table if we increased the collision count.
196 NeedResize = true;
197 break;
198 }
199 }
200
201 if (!NeedResize)
202 return {N, MaxCollisions};
203 }
204 }
205}
206
207static std::vector<unsigned>
208constructPerfectHashTable(ArrayRef<RuntimeLibcallImpl> Keywords,
209 ArrayRef<uint64_t> Hashes,
210 ArrayRef<unsigned> TableValues, int Size,
211 int Collisions, StringToOffsetTable &OffsetTable) {
212 std::vector<unsigned> Lookup(Size * Collisions);
213
214 for (auto [HashValue, TableValue] : zip(t&: Hashes, u&: TableValues)) {
215 uint64_t Idx = (HashValue % static_cast<uint64_t>(Size)) *
216 static_cast<uint64_t>(Collisions);
217
218 bool Found = false;
219 for (int J = 0; J < Collisions; ++J) {
220 unsigned &Entry = Lookup[Idx + J];
221 if (Entry == 0) {
222 Entry = TableValue;
223 Found = true;
224 break;
225 }
226 }
227
228 if (!Found)
229 reportFatalInternalError(reason: "failure to hash");
230 }
231
232 return Lookup;
233}
234
235/// Generate hash table based lookup by name.
236void RuntimeLibcallEmitter::emitNameMatchHashTable(
237 raw_ostream &OS, StringToOffsetTable &OffsetTable) const {
238 ArrayRef<RuntimeLibcallImpl> RuntimeLibcallImplDefList =
239 Libcalls.getRuntimeLibcallImplDefList();
240 std::vector<uint64_t> Hashes(RuntimeLibcallImplDefList.size());
241 std::vector<unsigned> TableValues(RuntimeLibcallImplDefList.size());
242 DenseSet<StringRef> SeenFuncNames;
243
244 size_t MaxFuncNameSize = 0;
245 size_t Index = 0;
246
247 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
248 StringRef ImplName = LibCallImpl.getLibcallFuncName();
249 if (SeenFuncNames.insert(V: ImplName).second) {
250 MaxFuncNameSize = std::max(a: MaxFuncNameSize, b: ImplName.size());
251 TableValues[Index] = LibCallImpl.getEnumVal();
252 Hashes[Index++] = hash(Str: ImplName);
253 }
254 }
255
256 // Trim excess elements from non-unique entries.
257 Hashes.resize(new_size: SeenFuncNames.size());
258 TableValues.resize(new_size: SeenFuncNames.size());
259
260 LLVM_DEBUG({
261 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
262 StringRef ImplName = LibCallImpl.getLibcallFuncName();
263 if (ImplName.size() == MaxFuncNameSize) {
264 dbgs() << "Maximum runtime libcall name size: " << ImplName << '('
265 << MaxFuncNameSize << ")\n";
266 }
267 }
268 });
269
270 // Early exiting on the symbol name provides a significant speedup in the miss
271 // case on the set of symbols in a clang binary. Emit this as an inlinable
272 // precondition in the header.
273 //
274 // The empty check is also used to get sensible behavior on anonymous
275 // functions.
276 //
277 // TODO: It may make more sense to split the search by string size more. There
278 // are a few outliers, most call names are small.
279 {
280 IfDefEmitter IfDef(OS, "GET_LOOKUP_LIBCALL_IMPL_NAME_BODY");
281
282 OS << " size_t Size = Name.size();\n"
283 " if (Size == 0 || Size > "
284 << MaxFuncNameSize
285 << ")\n"
286 " return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);\n"
287 " return lookupLibcallImplNameImpl(Name);\n";
288 }
289
290 auto [Size, Collisions] = computePerfectHashParameters(Hashes);
291 std::vector<unsigned> Lookup =
292 constructPerfectHashTable(Keywords: RuntimeLibcallImplDefList, Hashes, TableValues,
293 Size, Collisions, OffsetTable);
294
295 LLVM_DEBUG(dbgs() << "Runtime libcall perfect hashing parameters: Size = "
296 << Size << ", maximum collisions = " << Collisions << '\n');
297
298 IfDefEmitter IfDef(OS, "DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME");
299 emitHashFunction(OS);
300
301 OS << "iota_range<RTLIB::LibcallImpl> RTLIB::RuntimeLibcallsInfo::"
302 "lookupLibcallImplNameImpl(StringRef Name) {\n";
303
304 // Emit RTLIB::LibcallImpl values
305 OS << " static constexpr uint16_t HashTableNameToEnum[" << Lookup.size()
306 << "] = {\n";
307
308 for (unsigned TableVal : Lookup)
309 OS << " " << TableVal << ",\n";
310
311 OS << " };\n\n";
312
313 OS << " unsigned Idx = (hash(Name) % " << Size << ") * " << Collisions
314 << ";\n\n"
315 " for (int I = 0; I != "
316 << Collisions << R"(; ++I) {
317 const uint16_t Entry = HashTableNameToEnum[Idx + I];
318 const uint16_t StrOffset = RuntimeLibcallNameOffsetTable[Entry];
319 const uint8_t StrSize = RuntimeLibcallNameSizeTable[Entry];
320 StringRef Str(
321 &RTLIB::RuntimeLibcallsInfo::RuntimeLibcallImplNameTableStorage[StrOffset],
322 StrSize);
323 if (Str == Name)
324 return libcallImplNameHit(Entry, StrOffset);
325 }
326
327 return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);
328}
329)";
330}
331
332void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
333 raw_ostream &OS) const {
334 // Emit the implementation names
335 StringToOffsetTable Table(/*AppendZero=*/true,
336 "RTLIB::RuntimeLibcallsInfo::");
337
338 {
339 IfDefEmitter IfDef(OS, "GET_INIT_RUNTIME_LIBCALL_NAMES");
340
341 for (const RuntimeLibcallImpl &LibCallImpl :
342 Libcalls.getRuntimeLibcallImplDefList())
343 Table.GetOrAddStringOffset(Str: LibCallImpl.getLibcallFuncName());
344
345 Table.EmitStringTableDef(OS, Name: "RuntimeLibcallImplNameTable");
346 OS << R"(
347const uint16_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameOffsetTable[] = {
348)";
349
350 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ""),
351 Vals: ""); // Unsupported entry
352 for (const RuntimeLibcallImpl &LibCallImpl :
353 Libcalls.getRuntimeLibcallImplDefList()) {
354 StringRef ImplName = LibCallImpl.getLibcallFuncName();
355 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ImplName), Vals&: ImplName);
356 }
357 OS << "};\n";
358
359 OS << R"(
360const uint8_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameSizeTable[] = {
361)";
362
363 OS << " 0,\n";
364 for (const RuntimeLibcallImpl &LibCallImpl :
365 Libcalls.getRuntimeLibcallImplDefList())
366 OS << " " << LibCallImpl.getLibcallFuncName().size() << ",\n";
367 OS << "};\n\n";
368
369 // Emit the reverse mapping from implementation libraries to RTLIB::Libcall
370 OS << "const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::"
371 "ImplToLibcall[RTLIB::NumLibcallImpls] = {\n"
372 " RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported\n";
373
374 for (const RuntimeLibcallImpl &LibCallImpl :
375 Libcalls.getRuntimeLibcallImplDefList()) {
376 const RuntimeLibcall *Provides = LibCallImpl.getProvides();
377 OS << " ";
378 Provides->emitEnumEntry(OS);
379 OS << ", // ";
380 LibCallImpl.emitEnumEntry(OS);
381 OS << '\n';
382 }
383
384 OS << "};\n\n";
385 }
386
387 emitNameMatchHashTable(OS, OffsetTable&: Table);
388}
389
390void RuntimeLibcallEmitter::emitPredicateGroups(
391 raw_ostream &OS, const Record *R,
392 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
393 SetVector<PredicateWithCC> &PredicateSorter, unsigned BaseIndent,
394 StringRef Receiver) const {
395 SmallVector<PredicateWithCC, 0> SortedPredicates =
396 PredicateSorter.takeVector();
397
398 llvm::sort(C&: SortedPredicates, Comp: [](PredicateWithCC A, PredicateWithCC B) {
399 StringRef AName = A.Availability ? A.Availability->getName() : "";
400 StringRef BName = B.Availability ? B.Availability->getName() : "";
401 if (AName != BName)
402 return AName < BName;
403 // Break name ties on the calling convention for a deterministic order.
404 StringRef ACC = A.CallingConv ? A.CallingConv->getName() : "";
405 StringRef BCC = B.CallingConv ? B.CallingConv->getName() : "";
406 return ACC < BCC;
407 });
408
409 for (PredicateWithCC Entry : SortedPredicates) {
410 AvailabilityPredicate SubsetPredicate(Entry.Availability);
411 unsigned IndentDepth = BaseIndent;
412
413 auto It = Pred2Funcs.find(Val: Entry);
414 if (It == Pred2Funcs.end())
415 continue;
416
417 // Shared-core deduplication can empty a bucket.
418 if (It->second.LibcallImpls.empty())
419 continue;
420
421 if (!SubsetPredicate.isAlwaysAvailable()) {
422 IndentDepth = BaseIndent + 2;
423
424 OS << indent(IndentDepth);
425 SubsetPredicate.emitIf(OS);
426 }
427
428 LibcallsWithCC &FuncsWithCC = It->second;
429
430 std::vector<const RuntimeLibcallImpl *> &Funcs = FuncsWithCC.LibcallImpls;
431
432 // Records which impls are available, not which is selected, so a libcall
433 // may have more than one. Order is irrelevant (each entry is a setAvailable
434 // call); sort by the provided libcall, breaking ties on the impl enum for a
435 // deterministic total order.
436 llvm::sort(C&: Funcs, Comp: [](const RuntimeLibcallImpl *A,
437 const RuntimeLibcallImpl *B) {
438 return std::make_pair(x: A->getProvides()->getEnumVal(), y: A->getEnumVal()) <
439 std::make_pair(x: B->getProvides()->getEnumVal(), y: B->getEnumVal());
440 });
441
442 OS << indent(IndentDepth + 2)
443 << "static const RTLIB::LibcallImpl LibraryCalls";
444 SubsetPredicate.emitTableVariableNameSuffix(OS);
445 if (FuncsWithCC.CallingConv)
446 OS << '_' << FuncsWithCC.CallingConv->getName();
447
448 OS << "[] = {\n";
449 for (const RuntimeLibcallImpl *LibCallImpl : Funcs) {
450 OS << indent(IndentDepth + 6);
451 LibCallImpl->emitEnumEntry(OS);
452 OS << ", // " << LibCallImpl->getLibcallFuncName() << '\n';
453 }
454
455 OS << indent(IndentDepth + 2) << "};\n\n"
456 << indent(IndentDepth + 2)
457 << "for (const RTLIB::LibcallImpl Impl : LibraryCalls";
458 SubsetPredicate.emitTableVariableNameSuffix(OS);
459 if (FuncsWithCC.CallingConv)
460 OS << '_' << FuncsWithCC.CallingConv->getName();
461
462 OS << ") {\n"
463 << indent(IndentDepth + 4) << Receiver << "setAvailable(Impl);\n";
464
465 if (FuncsWithCC.CallingConv) {
466 StringRef CCEnum =
467 FuncsWithCC.CallingConv->getValueAsString(FieldName: "CallingConv");
468 OS << indent(IndentDepth + 4) << Receiver
469 << "setLibcallImplCallingConv(Impl, " << CCEnum << ");\n";
470 }
471
472 OS << indent(IndentDepth + 2) << "}\n";
473 OS << '\n';
474
475 if (!SubsetPredicate.isAlwaysAvailable()) {
476 OS << indent(IndentDepth);
477 SubsetPredicate.emitEndIf(OS);
478 OS << '\n';
479 }
480 }
481}
482
483// Emit the linker name \p Name as a C++ identifier suffix, replacing characters
484// invalid in an identifier (e.g. the '-' in "compiler-rt") with '_'.
485static void emitLibFuncSuffix(raw_ostream &OS, StringRef Name) {
486 for (char C : Name)
487 OS << (isAlnum(C) || C == '_' ? C : '_');
488}
489
490void RuntimeLibcallEmitter::emitLibraryFunction(
491 raw_ostream &OS, StringRef Name, ArrayRef<const Record *> Libs) const {
492 // File-local; referenced only from the driver in the same fragment.
493 OS << "static void setAvailableLibFuncs_";
494 emitLibFuncSuffix(OS, Name);
495 OS << "(llvm::RTLIB::RuntimeLibcallsInfo &Info, const llvm::Triple &TT, "
496 "ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, "
497 "EABI EABIVersion, StringRef ABIName, "
498 "LongDoubleFormat LongDoubleFormat) {\n";
499
500 // Per-variant expansion. Unconditional impls are tracked separately for
501 // cross-variant deduplication.
502 struct ExpandedLibrary {
503 const Record *Lib;
504 DenseMap<PredicateWithCC, LibcallsWithCC> Pred2Funcs;
505 SetVector<PredicateWithCC> PredicateSorter;
506 SetVector<const RuntimeLibcallImpl *> Unconditional;
507 };
508
509 SmallVector<ExpandedLibrary, 2> Expanded;
510 for (const Record *Lib : Libs) {
511 ExpandedLibrary EL;
512 EL.Lib = Lib;
513
514 // Expand this library's members with a library-local Func2Preds.
515 SetTheory Sets;
516 DenseMap<const RuntimeLibcallImpl *,
517 std::pair<std::vector<const Record *>, const Record *>>
518 Func2Preds;
519 Sets.addExpander(ClassName: "LibcallImpls", std::make_unique<LibcallPredicateExpander>(
520 args: Libcalls, args&: Func2Preds));
521
522 SetTheory::RecSet Elements;
523 Sets.evaluate(Expr: Lib->getValueInit(FieldName: "Impls"), Elts&: Elements, Loc: Lib->getLoc());
524
525 EL.PredicateSorter.insert(
526 X: PredicateWithCC()); // No predicate or CC override first.
527
528 for (const Record *Elt : Elements) {
529 const RuntimeLibcallImpl *LibCallImpl =
530 Libcalls.getRuntimeLibcallImpl(Def: Elt);
531 if (!LibCallImpl) {
532 PrintError(Rec: Lib, Msg: "entry for LibcallLibrary is not a RuntimeLibcallImpl");
533 PrintNote(NoteLoc: Elt->getLoc(), Msg: "invalid entry `" + Elt->getName() + "`");
534 continue;
535 }
536
537 auto It = Func2Preds.find(Val: LibCallImpl);
538 if (It == Func2Preds.end()) {
539 EL.Pred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: LibCallImpl);
540 EL.Unconditional.insert(X: LibCallImpl);
541 continue;
542 }
543
544 for (const Record *Pred : It->second.first) {
545 const Record *CC = It->second.second;
546 PredicateWithCC Key(Pred, CC);
547 auto &Entry = EL.Pred2Funcs[Key];
548 Entry.LibcallImpls.push_back(x: LibCallImpl);
549 Entry.CallingConv = CC;
550 EL.PredicateSorter.insert(X: Key);
551 }
552 }
553
554 Expanded.push_back(Elt: std::move(EL));
555 }
556
557 // Impls unconditional in every variant are emitted once and stripped from
558 // each variant, so the shared core is not repeated.
559 SetVector<const RuntimeLibcallImpl *> SharedCore;
560 if (Expanded.size() > 1) {
561 for (const RuntimeLibcallImpl *Impl : Expanded.front().Unconditional) {
562 if (all_of(Range: drop_begin(RangeOrContainer&: Expanded), P: [&](const ExpandedLibrary &EL) {
563 return EL.Unconditional.contains(key: Impl);
564 }))
565 SharedCore.insert(X: Impl);
566 }
567 }
568
569 if (!SharedCore.empty()) {
570 // Emit the shared core once, then strip it from every variant.
571 DenseMap<PredicateWithCC, LibcallsWithCC> CorePred2Funcs;
572 SetVector<PredicateWithCC> CoreSorter;
573 CoreSorter.insert(X: PredicateWithCC());
574 for (const RuntimeLibcallImpl *Impl : SharedCore)
575 CorePred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: Impl);
576 emitPredicateGroups(OS, R: Libs.front(), Pred2Funcs&: CorePred2Funcs, PredicateSorter&: CoreSorter,
577 /*BaseIndent=*/0, /*Receiver=*/"Info.");
578
579 for (ExpandedLibrary &EL : Expanded) {
580 auto &Funcs = EL.Pred2Funcs[PredicateWithCC()].LibcallImpls;
581 llvm::erase_if(C&: Funcs, P: [&](const RuntimeLibcallImpl *Impl) {
582 return SharedCore.contains(key: Impl);
583 });
584 }
585 }
586
587 // Emit each variant under its own Pred.
588 for (ExpandedLibrary &EL : Expanded) {
589 AvailabilityPredicate LibPred(EL.Lib->getValueAsDef(FieldName: "Pred"));
590
591 if (!LibPred.isAlwaysAvailable()) {
592 OS << indent(2);
593 LibPred.emitIf(OS);
594 } else {
595 // Own block scope so per-variant `LibraryCalls` tables do not collide.
596 OS << indent(2) << "{\n";
597 }
598
599 emitPredicateGroups(OS, R: EL.Lib, Pred2Funcs&: EL.Pred2Funcs, PredicateSorter&: EL.PredicateSorter,
600 /*BaseIndent=*/2, /*Receiver=*/"Info.");
601
602 if (!LibPred.isAlwaysAvailable()) {
603 OS << indent(2);
604 LibPred.emitEndIf(OS);
605 } else {
606 OS << indent(2) << "}\n";
607 }
608 }
609
610 OS << "}\n\n";
611}
612
613void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
614 raw_ostream &OS) const {
615 // Emit one function per distinct library name; same-named defs merge.
616 //
617 // TODO: SystemRuntimeLibrary does not yet dispatch to these
618 MapVector<StringRef, std::vector<const Record *>> LibsByName;
619 for (const Record *Lib : Records.getAllDerivedDefinitions(ClassName: "LibcallLibrary"))
620 LibsByName[Lib->getValueAsString(FieldName: "LibraryName")].push_back(x: Lib);
621
622 for (const auto &[Name, Libs] : LibsByName)
623 emitLibraryFunction(OS, Name, Libs);
624
625 ArrayRef<const Record *> AllLibs =
626 Records.getAllDerivedDefinitions(ClassName: "SystemRuntimeLibrary");
627
628 OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets("
629 "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
630 "FloatABI::ABIType FloatABI, EABI EABIVersion, "
631 "StringRef ABIName, LongDoubleFormat LongDoubleFormat) {\n";
632
633 for (const Record *R : AllLibs) {
634 OS << '\n';
635
636 AvailabilityPredicate TopLevelPredicate(R->getValueAsDef(FieldName: "TriplePred"));
637
638 OS << indent(2);
639 TopLevelPredicate.emitIf(OS);
640
641 if (const Record *DefaultCCClass =
642 R->getValueAsDef(FieldName: "DefaultLibcallCallingConv")) {
643 StringRef DefaultCC =
644 DefaultCCClass->getValueAsString(FieldName: "CallingConv").trim();
645
646 if (!DefaultCC.empty()) {
647 OS << " const CallingConv::ID DefaultCC = " << DefaultCC << ";\n"
648 << " for (CallingConv::ID &Entry : LibcallImplCallingConvs) {\n"
649 " Entry = DefaultCC;\n"
650 " }\n\n";
651 }
652 }
653
654 SetTheory Sets;
655
656 DenseMap<const RuntimeLibcallImpl *,
657 std::pair<std::vector<const Record *>, const Record *>>
658 Func2Preds;
659 Sets.addExpander(ClassName: "LibcallImpls", std::make_unique<LibcallPredicateExpander>(
660 args: Libcalls, args&: Func2Preds));
661
662 const SetTheory::RecVec *Elements =
663 Sets.expand(Set: R->getValueAsDef(FieldName: "MemberList"));
664
665 // Sort to get deterministic output
666 SetVector<PredicateWithCC> PredicateSorter;
667 PredicateSorter.insert(
668 X: PredicateWithCC()); // No predicate or CC override first.
669
670 constexpr unsigned BitsPerStorageElt = 64;
671 DenseMap<PredicateWithCC, LibcallsWithCC> Pred2Funcs;
672
673 SmallVector<uint64_t, 32> BitsetValues(divideCeil(
674 Numerator: Libcalls.getRuntimeLibcallImplDefList().size() + 1, Denominator: BitsPerStorageElt));
675
676 for (const Record *Elt : *Elements) {
677 const RuntimeLibcallImpl *LibCallImpl =
678 Libcalls.getRuntimeLibcallImpl(Def: Elt);
679 if (!LibCallImpl) {
680 PrintError(Rec: R, Msg: "entry for SystemLibrary is not a RuntimeLibcallImpl");
681 PrintNote(NoteLoc: Elt->getLoc(), Msg: "invalid entry `" + Elt->getName() + "`");
682 continue;
683 }
684
685 size_t BitIdx = LibCallImpl->getEnumVal();
686 uint64_t BitmaskVal = uint64_t(1) << (BitIdx % BitsPerStorageElt);
687 size_t BitsetIdx = BitIdx / BitsPerStorageElt;
688
689 auto It = Func2Preds.find(Val: LibCallImpl);
690 if (It == Func2Preds.end()) {
691 BitsetValues[BitsetIdx] |= BitmaskVal;
692 Pred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: LibCallImpl);
693 continue;
694 }
695
696 for (const Record *Pred : It->second.first) {
697 const Record *CC = It->second.second;
698 AvailabilityPredicate SubsetPredicate(Pred);
699 if (SubsetPredicate.isAlwaysAvailable())
700 BitsetValues[BitsetIdx] |= BitmaskVal;
701
702 PredicateWithCC Key(Pred, CC);
703 auto &Entry = Pred2Funcs[Key];
704 Entry.LibcallImpls.push_back(x: LibCallImpl);
705 Entry.CallingConv = It->second.second;
706 PredicateSorter.insert(X: Key);
707 }
708 }
709
710 OS << " static constexpr LibcallImplBitset SystemAvailableImpls({\n"
711 << indent(6);
712
713 ListSeparator LS;
714 unsigned EntryCount = 0;
715 for (uint64_t Bits : BitsetValues) {
716 if (EntryCount++ == 4) {
717 EntryCount = 1;
718 OS << ",\n" << indent(6);
719 } else
720 OS << LS;
721 OS << format_hex(N: Bits, Width: 16);
722 }
723 OS << "\n });\n"
724 " AvailableLibcallImpls = SystemAvailableImpls;\n\n";
725
726 emitPredicateGroups(OS, R, Pred2Funcs, PredicateSorter, /*BaseIndent=*/2,
727 /*Receiver=*/"");
728
729 OS << indent(4) << "return;\n" << indent(2);
730 TopLevelPredicate.emitEndIf(OS);
731 }
732
733 // FIXME: This should be a fatal error. A few contexts are improperly relying
734 // on RuntimeLibcalls constructed with fully unknown triples.
735 OS << " LLVM_DEBUG(dbgs() << \"no system runtime library applied to target "
736 "\\'\" << TT.str() << \"\\'\\n\");\n"
737 "}\n\n";
738}
739
740// Scalar FP type suffixes in the argument order of RTLIB::getFPLibCall, paired
741// with the llvm::Type predicate used by the IR-level mapping.
742static constexpr std::pair<StringRef, StringRef> ScalarFPSuffixes[] = {
743 {"F32", "isFloatTy()"}, {"F64", "isDoubleTy()"},
744 {"F80", "isX86_FP80Ty()"}, {"F128", "isFP128Ty()"},
745 {"PPCF128", "isPPC_FP128Ty()"},
746};
747
748static std::vector<FPLibcallFamily>
749collectFPLibcallFamilies(const RecordKeeper &Records) {
750 std::vector<FPLibcallFamily> Families;
751 for (const Record *R :
752 Records.getAllDerivedDefinitions(ClassName: "RuntimeLibcallFamily"))
753 Families.emplace_back(args&: R);
754 llvm::sort(C&: Families, Comp: [](const FPLibcallFamily &A, const FPLibcallFamily &B) {
755 return A.Base < B.Base;
756 });
757 return Families;
758}
759
760DenseSet<StringRef> RuntimeLibcallEmitter::collectLibcallNames() const {
761 DenseSet<StringRef> LibcallNames;
762 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList())
763 LibcallNames.insert(V: LC.getName());
764 return LibcallNames;
765}
766
767void RuntimeLibcallEmitter::checkFPLibcallFamilies(
768 ArrayRef<FPLibcallFamily> Families,
769 const DenseSet<StringRef> &LibcallNames) const {
770 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
771 for (const FPLibcallFamily &Family : Families)
772 for (StringRef Intrinsic : Family.Intrinsics)
773 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
774 llvm::sort(C&: IntrinsicToBase);
775
776 for (size_t I = 1, E = IntrinsicToBase.size(); I < E; ++I) {
777 if (IntrinsicToBase[I].first == IntrinsicToBase[I - 1].first)
778 PrintFatalError(
779 Msg: "intrinsic '" + IntrinsicToBase[I].first +
780 "' is mapped by multiple RuntimeLibcallFamily records ('" +
781 IntrinsicToBase[I - 1].second + "' and '" +
782 IntrinsicToBase[I].second + "')");
783 }
784
785 for (const FPLibcallFamily &Family : Families) {
786 bool AnyScalarLibcall = any_of(
787 Range: ScalarFPSuffixes, P: [&](const std::pair<StringRef, StringRef> &Entry) {
788 return LibcallNames.contains(V: (Family.Base + "_" + Entry.first).str());
789 });
790 if (!AnyScalarLibcall)
791 PrintFatalError(Msg: "no runtime libcall found for base name '" + Family.Base +
792 "'");
793 }
794}
795
796/// Generate the declarations for the RTLIB::get<base>(EVT) selectors.
797void RuntimeLibcallEmitter::emitFPLibcallSelectorDecls(
798 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families) const {
799 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTOR_DECLS");
800 for (const FPLibcallFamily &Family : Families)
801 OS << "LLVM_ABI Libcall get" << Family.Base << "(EVT VT);\n";
802}
803
804/// Generate the backend RTLIB::get<base>(EVT) selectors from the floating-point
805/// libcall families.
806void RuntimeLibcallEmitter::emitFPLibcallSelectors(
807 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
808 const DenseSet<StringRef> &LibcallNames) const {
809 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTORS");
810
811 // Only emit a libcall enumerator if it actually exists in the declared set.
812 auto scalarEnum = [&](StringRef Base, StringRef Suffix) -> std::string {
813 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
814 return ("RTLIB::" + Base + "_" + Suffix).str();
815 return "RTLIB::UNKNOWN_LIBCALL";
816 };
817
818 for (const FPLibcallFamily &Family : Families) {
819 StringRef Base = Family.Base;
820 OS << "RTLIB::Libcall llvm::RTLIB::get" << Base << "(EVT VT) {\n";
821
822 if (!Family.VectorSuffixes.empty()) {
823 OS << " if (VT.isVector()) {\n"
824 " if (!VT.isSimple())\n"
825 " return RTLIB::UNKNOWN_LIBCALL;\n"
826 " switch (VT.getSimpleVT().SimpleTy) {\n";
827 for (StringRef Suffix : Family.VectorSuffixes) {
828 OS << " case MVT::" << Suffix.lower()
829 << ":\n return RTLIB::" << Base << "_" << Suffix << ";\n";
830 }
831 OS << " default:\n"
832 " return RTLIB::UNKNOWN_LIBCALL;\n"
833 " }\n"
834 " }\n";
835 }
836
837 OS << " return getFPLibCall(VT";
838 for (auto [Suffix, Pred] : ScalarFPSuffixes)
839 OS << ", " << scalarEnum(Base, Suffix);
840 OS << ");\n}\n\n";
841 }
842}
843
844/// Emit the mapping from floating-point math intrinsics to the runtime libcall
845/// they may lower to, keyed by intrinsic ID and floating-point type. This is
846/// the IR-level counterpart to the backend's RTLIB::getXXX(EVT) selectors.
847void RuntimeLibcallEmitter::emitGetLibcallForIntrinsic(
848 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
849 const DenseSet<StringRef> &LibcallNames) const {
850 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_INTRINSIC_TO_LIBCALL");
851
852 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
853 for (const FPLibcallFamily &Family : Families)
854 for (StringRef Intrinsic : Family.Intrinsics)
855 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
856 llvm::sort(C&: IntrinsicToBase);
857
858 MapVector<StringRef, SmallVector<StringRef, 2>> BaseToIntrinsics;
859 for (auto [Intrinsic, Base] : IntrinsicToBase)
860 BaseToIntrinsics[Base].push_back(Elt: Intrinsic);
861
862 OS << "RTLIB::Libcall "
863 "llvm::RTLIB::RuntimeLibcallsInfo::getLibcallForIntrinsic("
864 "Intrinsic::ID ID, FunctionType *FTy) {\n"
865 " Type *Ty = FTy->getReturnType();\n"
866 " if (!Ty->isFloatingPointTy()) {\n"
867 " for (Type *ParamTy : FTy->params()) {\n"
868 " if (ParamTy->isFloatingPointTy()) {\n"
869 " Ty = ParamTy;\n"
870 " break;\n"
871 " }\n"
872 " }\n"
873 " }\n"
874 " if (!Ty->isFloatingPointTy())\n"
875 " return RTLIB::UNKNOWN_LIBCALL;\n"
876 " switch (ID) {\n";
877
878 for (const auto &[Base, Intrinsics] : BaseToIntrinsics) {
879 SmallVector<std::pair<StringRef, StringRef>, 5> Arms;
880 for (auto [Suffix, Pred] : ScalarFPSuffixes)
881 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
882 Arms.emplace_back(Args&: Suffix, Args&: Pred);
883
884 for (StringRef Intrinsic : Intrinsics)
885 OS << " case Intrinsic::" << Intrinsic << ":\n";
886 for (auto [Suffix, Pred] : Arms)
887 OS << " if (Ty->" << Pred << ")\n return RTLIB::" << Base << "_"
888 << Suffix << ";\n";
889 OS << " return RTLIB::UNKNOWN_LIBCALL;\n";
890 }
891
892 OS << " default:\n"
893 " return RTLIB::UNKNOWN_LIBCALL;\n"
894 " }\n"
895 "}\n";
896}
897
898void RuntimeLibcallEmitter::run(raw_ostream &OS) {
899 emitSourceFileHeader(Desc: "Runtime LibCalls Source Fragment", OS, Record: Records);
900 emitGetRuntimeLibcallEnum(OS);
901
902 emitGetInitRuntimeLibcallNames(OS);
903
904 {
905 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALLS_INFO");
906 emitSystemRuntimeLibrarySetCalls(OS);
907 }
908
909 std::vector<FPLibcallFamily> FPFamilies = collectFPLibcallFamilies(Records);
910 DenseSet<StringRef> LibcallNames = collectLibcallNames();
911 checkFPLibcallFamilies(Families: FPFamilies, LibcallNames);
912 emitFPLibcallSelectorDecls(OS, Families: FPFamilies);
913 emitFPLibcallSelectors(OS, Families: FPFamilies, LibcallNames);
914 emitGetLibcallForIntrinsic(OS, Families: FPFamilies, LibcallNames);
915}
916
917static TableGen::Emitter::OptClass<RuntimeLibcallEmitter>
918 X("gen-runtime-libcalls", "Generate RuntimeLibcalls");
919