1//===- RuntimeLibcallEmitter.cpp - Properties from RuntimeLibcalls.td -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#define DEBUG_TYPE "runtime-libcall-emitter"
10
11#include "AttributeTableEmitter.h"
12#include "RuntimeLibcalls.h"
13
14#include "SequenceToOffsetTable.h"
15#include "llvm/ADT/DenseSet.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/Support/Debug.h"
21#include "llvm/Support/FormatVariadic.h"
22#include "llvm/Support/ModRef.h"
23#include "llvm/Support/raw_ostream.h"
24#include "llvm/TableGen/CodeGenHelpers.h"
25#include "llvm/TableGen/Error.h"
26#include "llvm/TableGen/Record.h"
27#include "llvm/TableGen/SetTheory.h"
28#include "llvm/TableGen/StringToOffsetTable.h"
29#include "llvm/TableGen/TableGenBackend.h"
30
31using namespace llvm;
32
33namespace {
34// Pair of a RuntimeLibcallAvailability and LibcallCallingConv to use as a map
35// key.
36struct PredicateWithCC {
37 const Record *Availability = nullptr;
38 const Record *CallingConv = nullptr;
39
40 PredicateWithCC() = default;
41 PredicateWithCC(std::pair<const Record *, const Record *> P)
42 : Availability(P.first), CallingConv(P.second) {}
43
44 PredicateWithCC(const Record *P, const Record *C)
45 : Availability(P), CallingConv(C) {}
46};
47
48inline bool operator==(PredicateWithCC LHS, PredicateWithCC RHS) {
49 return LHS.Availability == RHS.Availability &&
50 LHS.CallingConv == RHS.CallingConv;
51}
52} // namespace
53
54namespace {
55/// A floating-point libcall family parsed from a RuntimeLibcallFamily record.
56struct FPLibcallFamily {
57 StringRef Base;
58 std::vector<StringRef> Intrinsics;
59 std::vector<StringRef> VectorSuffixes;
60
61 explicit FPLibcallFamily(const Record *R)
62 : Base(R->getValueAsString(FieldName: "LibcallBase")),
63 Intrinsics(R->getValueAsListOfStrings(FieldName: "Intrinsics")),
64 VectorSuffixes(R->getValueAsListOfStrings(FieldName: "VectorSuffixes")) {}
65};
66} // namespace
67
68namespace llvm {
69template <> struct DenseMapInfo<PredicateWithCC, void> {
70 static unsigned getHashValue(const PredicateWithCC Val) {
71 auto Pair = std::make_pair(x: Val.Availability, y: Val.CallingConv);
72 return DenseMapInfo<
73 std::pair<const Record *, const Record *>>::getHashValue(PairVal: Pair);
74 }
75
76 static bool isEqual(PredicateWithCC LHS, PredicateWithCC RHS) {
77 return LHS == RHS;
78 }
79};
80
81class RuntimeLibcallEmitter {
82private:
83 const RecordKeeper &Records;
84 RuntimeLibcalls Libcalls;
85
86 void emitGetRuntimeLibcallEnum(raw_ostream &OS) const;
87
88 void emitNameMatchHashTable(raw_ostream &OS,
89 StringToOffsetTable &OffsetTable) const;
90
91 void emitGetInitRuntimeLibcallNames(raw_ostream &OS) const;
92 void emitGetInitRuntimeLibcallSignatures(raw_ostream &OS) const;
93 void emitRuntimeLibcallAttributes(raw_ostream &OS) const;
94 void emitMaskedVectorLibcallMapping(raw_ostream &OS) const;
95
96 // Emit the sorted per-predicate `setAvailable` tables/loops. The
97 // always-available bucket emits at \p BaseIndent; each predicated bucket is
98 // wrapped in `if (pred)`. All calls are emitted in member context (no
99 // receiver prefix).
100 void
101 emitPredicateGroups(raw_ostream &OS, const Record *R,
102 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
103 SetVector<PredicateWithCC> &PredicateSorter,
104 unsigned BaseIndent) const;
105
106 // A LibraryExclusion: the impls dropped from a shared library on triples
107 // matching TriplePred.
108 struct LibraryExclusion {
109 const Record *TriplePred;
110 std::vector<const RuntimeLibcallImpl *> Impls;
111 };
112
113 // A SystemRuntimeLibrary's default calling convention and default library
114 // names.
115 struct SystemLibraryDispatch {
116 const Record *System;
117 StringRef DefaultCC;
118 SmallVector<StringRef> Libs;
119 };
120
121 // A single LibcallLibrary variant, expanded into its per-predicate impl
122 // groups. Unconditional impls are tracked separately for cross-variant
123 // deduplication. A variant is Deferred when it re-adds an impl its own
124 // consumer excludes; deferred variants are emitted after the
125 // LibraryExclusions so the re-add wins over the opt-out.
126 struct ExpandedLibrary {
127 const Record *Lib;
128 DenseMap<PredicateWithCC, LibcallsWithCC> Pred2Funcs;
129 SetVector<PredicateWithCC> PredicateSorter;
130 SetVector<const RuntimeLibcallImpl *> Unconditional;
131 bool Deferred = false;
132 };
133
134 // Emit one variant's guarded `setAvailable` block into the enclosing
135 // `setAvailableLibFuncs_<name>` function.
136 void emitLibraryVariant(raw_ostream &OS, ExpandedLibrary &EL) const;
137
138 // Emit a `setAvailableLibFuncs_<name>` member function for all LibcallLibrary
139 // defs sharing \p Name, each gated by its own availability predicate. \p
140 // Exclusions are emitted as guarded setUnavailable calls at the end.
141 void emitLibraryFunction(raw_ostream &OS, StringRef Name,
142 ArrayRef<const Record *> Libs,
143 ArrayRef<LibraryExclusion> Exclusions) const;
144
145 // Group all LibcallLibrary defs by their shared LibraryName, preserving
146 // definition order. Both the member-declaration fragment and the definitions
147 // iterate this to stay in lockstep.
148 MapVector<StringRef, std::vector<const Record *>>
149 collectLibrariesByName() const;
150
151 // Get the sorted, unique list of every defined LibraryName.
152 std::vector<StringRef> collectLibraryNames() const;
153
154 void emitRuntimeLibcallsInfoMemberDecls(raw_ostream &OS) const;
155
156 std::vector<SystemLibraryDispatch> collectSystemLibraryDispatch() const;
157
158 void emitLibraryFunctions(raw_ostream &OS) const;
159
160 // Emit getDefaultLibcallCallingConv, getDefaultRuntimeLibraries, and
161 // setAvailableLibraryLibcalls, which enables a library by name.
162 void emitLibraryByNameSetCalls(raw_ostream &OS,
163 ArrayRef<SystemLibraryDispatch> Systems) const;
164
165 DenseSet<StringRef> collectLibcallNames() const;
166
167 void checkFPLibcallFamilies(ArrayRef<FPLibcallFamily> Families,
168 const DenseSet<StringRef> &LibcallNames) const;
169
170 void emitFPLibcallSelectorDecls(raw_ostream &OS,
171 ArrayRef<FPLibcallFamily> Families) const;
172
173 void emitFPLibcallSelectors(raw_ostream &OS,
174 ArrayRef<FPLibcallFamily> Families,
175 const DenseSet<StringRef> &LibcallNames) const;
176
177 void
178 emitGetLibcallForIntrinsic(raw_ostream &OS,
179 ArrayRef<FPLibcallFamily> Families,
180 const DenseSet<StringRef> &LibcallNames) const;
181
182public:
183 RuntimeLibcallEmitter(const RecordKeeper &R) : Records(R), Libcalls(R) {}
184
185 void run(raw_ostream &OS);
186};
187
188} // namespace llvm
189
190// Emit the linker name \p Name as a C++ identifier suffix, replacing characters
191// invalid in an identifier (e.g. the '-' in "compiler-rt") with '_'.
192static void emitLibFuncSuffix(raw_ostream &OS, StringRef Name) {
193 for (char C : Name)
194 OS << (isAlnum(C) || C == '_' ? C : '_');
195}
196
197static void emitRuntimeLibraryEnumerator(raw_ostream &OS, StringRef Name) {
198 OS << "RTLIB::RuntimeLibrary::";
199 emitLibFuncSuffix(OS, Name);
200}
201
202void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
203 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_ENUM");
204
205 OS << "namespace llvm {\n"
206 "namespace RTLIB {\n"
207 "enum Libcall : unsigned short {\n";
208
209 for (const RuntimeLibcall &LibCall : Libcalls.getRuntimeLibcallDefList()) {
210 StringRef Name = LibCall.getName();
211 OS << " " << Name << " = " << LibCall.getEnumVal() << ",\n";
212 }
213
214 OS << " UNKNOWN_LIBCALL = " << Libcalls.getRuntimeLibcallDefList().size()
215 << "\n};\n\n"
216 "enum LibcallImpl : unsigned short {\n"
217 " Unsupported = 0,\n";
218
219 for (const RuntimeLibcallImpl &LibCall :
220 Libcalls.getRuntimeLibcallImplDefList()) {
221 OS << " impl_" << LibCall.getName() << " = " << LibCall.getEnumVal()
222 << ", // " << LibCall.getLibcallFuncName() << '\n';
223 }
224
225 OS << "};\n"
226 << "constexpr size_t NumLibcallImpls = "
227 << Libcalls.getRuntimeLibcallImplDefList().size() + 1
228 << ";\n\n"
229 "enum FuncArgTypeID : char {\n"
230 " NoFuncArgType = 0,\n";
231
232 for (const auto *R : Libcalls.getFuncArgTypeList()) {
233 if (R->getName() == "NoneType")
234 continue;
235 OS << " " << R->getName() << ",\n";
236 }
237
238 OS << "};\n\n"
239 "enum class RuntimeLibrary : unsigned char {\n"
240 " UnknownLibrary = 0,\n";
241 for (StringRef Name : collectLibraryNames()) {
242 OS << " ";
243 emitLibFuncSuffix(OS, Name);
244 OS << ",\n";
245 }
246
247 OS << "};\n"
248 "} // End namespace RTLIB\n"
249 "} // End namespace llvm\n";
250}
251
252// StringMap uses xxh3_64bits, truncated to uint32_t.
253static uint64_t hash(StringRef Str) {
254 return static_cast<uint32_t>(xxh3_64bits(data: Str));
255}
256
257static void emitHashFunction(raw_ostream &OS) {
258 OS << "static inline uint64_t hash(StringRef Str) {\n"
259 " return static_cast<uint32_t>(xxh3_64bits(Str));\n"
260 "}\n\n";
261}
262
263/// Return the table size, maximum number of collisions for the set of hashes
264static std::pair<int, int>
265computePerfectHashParameters(ArrayRef<uint64_t> Hashes) {
266 // Chosen based on experimentation with llvm/benchmarks/RuntimeLibcalls.cpp
267 const int SizeOverhead = 4;
268
269 // Index derived from hash -> number of collisions.
270 DenseMap<uint64_t, int> Table;
271
272 unsigned NumHashes = Hashes.size();
273
274 for (int MaxCollisions = 1;; ++MaxCollisions) {
275 for (unsigned N = NextPowerOf2(A: NumHashes - 1); N < SizeOverhead * NumHashes;
276 N <<= 1) {
277 Table.clear();
278
279 bool NeedResize = false;
280 for (uint64_t H : Hashes) {
281 uint64_t Idx = H % static_cast<uint64_t>(N);
282 if (++Table[Idx] > MaxCollisions) {
283 // Need to resize the final table if we increased the collision count.
284 NeedResize = true;
285 break;
286 }
287 }
288
289 if (!NeedResize)
290 return {N, MaxCollisions};
291 }
292 }
293}
294
295static std::vector<unsigned>
296constructPerfectHashTable(ArrayRef<RuntimeLibcallImpl> Keywords,
297 ArrayRef<uint64_t> Hashes,
298 ArrayRef<unsigned> TableValues, int Size,
299 int Collisions, StringToOffsetTable &OffsetTable) {
300 std::vector<unsigned> Lookup(Size * Collisions);
301
302 for (auto [HashValue, TableValue] : zip(t&: Hashes, u&: TableValues)) {
303 uint64_t Idx = (HashValue % static_cast<uint64_t>(Size)) *
304 static_cast<uint64_t>(Collisions);
305
306 bool Found = false;
307 for (int J = 0; J < Collisions; ++J) {
308 unsigned &Entry = Lookup[Idx + J];
309 if (Entry == 0) {
310 Entry = TableValue;
311 Found = true;
312 break;
313 }
314 }
315
316 if (!Found)
317 reportFatalInternalError(reason: "failure to hash");
318 }
319
320 return Lookup;
321}
322
323/// Generate hash table based lookup by name.
324void RuntimeLibcallEmitter::emitNameMatchHashTable(
325 raw_ostream &OS, StringToOffsetTable &OffsetTable) const {
326 ArrayRef<RuntimeLibcallImpl> RuntimeLibcallImplDefList =
327 Libcalls.getRuntimeLibcallImplDefList();
328 std::vector<uint64_t> Hashes(RuntimeLibcallImplDefList.size());
329 std::vector<unsigned> TableValues(RuntimeLibcallImplDefList.size());
330 DenseSet<StringRef> SeenFuncNames;
331
332 size_t MaxFuncNameSize = 0;
333 size_t Index = 0;
334
335 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
336 StringRef ImplName = LibCallImpl.getLibcallFuncName();
337 if (SeenFuncNames.insert(V: ImplName).second) {
338 MaxFuncNameSize = std::max(a: MaxFuncNameSize, b: ImplName.size());
339 TableValues[Index] = LibCallImpl.getEnumVal();
340 Hashes[Index++] = hash(Str: ImplName);
341 }
342 }
343
344 // Trim excess elements from non-unique entries.
345 Hashes.resize(new_size: SeenFuncNames.size());
346 TableValues.resize(new_size: SeenFuncNames.size());
347
348 LLVM_DEBUG({
349 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
350 StringRef ImplName = LibCallImpl.getLibcallFuncName();
351 if (ImplName.size() == MaxFuncNameSize) {
352 dbgs() << "Maximum runtime libcall name size: " << ImplName << '('
353 << MaxFuncNameSize << ")\n";
354 }
355 }
356 });
357
358 // Early exiting on the symbol name provides a significant speedup in the miss
359 // case on the set of symbols in a clang binary. Emit this as an inlinable
360 // precondition in the header.
361 //
362 // The empty check is also used to get sensible behavior on anonymous
363 // functions.
364 //
365 // TODO: It may make more sense to split the search by string size more. There
366 // are a few outliers, most call names are small.
367 {
368 IfDefEmitter IfDef(OS, "GET_LOOKUP_LIBCALL_IMPL_NAME_BODY");
369
370 OS << " size_t Size = Name.size();\n"
371 " if (Size == 0 || Size > "
372 << MaxFuncNameSize
373 << ")\n"
374 " return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);\n"
375 " return lookupLibcallImplNameImpl(Name);\n";
376 }
377
378 auto [Size, Collisions] = computePerfectHashParameters(Hashes);
379 std::vector<unsigned> Lookup =
380 constructPerfectHashTable(Keywords: RuntimeLibcallImplDefList, Hashes, TableValues,
381 Size, Collisions, OffsetTable);
382
383 LLVM_DEBUG(dbgs() << "Runtime libcall perfect hashing parameters: Size = "
384 << Size << ", maximum collisions = " << Collisions << '\n');
385
386 IfDefEmitter IfDef(OS, "DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME");
387 emitHashFunction(OS);
388
389 OS << "iota_range<RTLIB::LibcallImpl> RTLIB::RuntimeLibcallsInfo::"
390 "lookupLibcallImplNameImpl(StringRef Name) {\n";
391
392 // Emit RTLIB::LibcallImpl values
393 OS << " static constexpr uint16_t HashTableNameToEnum[" << Lookup.size()
394 << "] = {\n";
395
396 for (unsigned TableVal : Lookup)
397 OS << " " << TableVal << ",\n";
398
399 OS << " };\n\n";
400
401 OS << " unsigned Idx = (hash(Name) % " << Size << ") * " << Collisions
402 << ";\n\n"
403 " for (int I = 0; I != "
404 << Collisions << R"(; ++I) {
405 const uint16_t Entry = HashTableNameToEnum[Idx + I];
406 const uint16_t StrOffset = RuntimeLibcallNameOffsetTable[Entry];
407 const uint8_t StrSize = RuntimeLibcallNameSizeTable[Entry];
408 StringRef Str(
409 &RTLIB::RuntimeLibcallsInfo::RuntimeLibcallImplNameTableStorage[StrOffset],
410 StrSize);
411 if (Str == Name)
412 return libcallImplNameHit(Entry, StrOffset);
413 }
414
415 return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);
416}
417)";
418}
419
420void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
421 raw_ostream &OS) const {
422 // Emit the implementation names
423 StringToOffsetTable Table(/*AppendZero=*/true,
424 "RTLIB::RuntimeLibcallsInfo::");
425
426 {
427 IfDefEmitter IfDef(OS, "GET_INIT_RUNTIME_LIBCALL_NAMES");
428
429 for (const RuntimeLibcallImpl &LibCallImpl :
430 Libcalls.getRuntimeLibcallImplDefList())
431 Table.GetOrAddStringOffset(Str: LibCallImpl.getLibcallFuncName());
432
433 Table.EmitStringTableDef(OS, Name: "RuntimeLibcallImplNameTable");
434 OS << R"(
435const uint16_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameOffsetTable[] = {
436)";
437
438 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ""),
439 Vals: ""); // Unsupported entry
440 for (const RuntimeLibcallImpl &LibCallImpl :
441 Libcalls.getRuntimeLibcallImplDefList()) {
442 StringRef ImplName = LibCallImpl.getLibcallFuncName();
443 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ImplName), Vals&: ImplName);
444 }
445 OS << "};\n";
446
447 OS << R"(
448const uint8_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameSizeTable[] = {
449)";
450
451 OS << " 0,\n";
452 for (const RuntimeLibcallImpl &LibCallImpl :
453 Libcalls.getRuntimeLibcallImplDefList())
454 OS << " " << LibCallImpl.getLibcallFuncName().size() << ",\n";
455 OS << "};\n\n";
456
457 // Emit the reverse mapping from implementation libraries to RTLIB::Libcall
458 OS << "const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::"
459 "ImplToLibcall[RTLIB::NumLibcallImpls] = {\n"
460 " RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported\n";
461
462 for (const RuntimeLibcallImpl &LibCallImpl :
463 Libcalls.getRuntimeLibcallImplDefList()) {
464 OS << " ";
465 if (const RuntimeLibcall *Provides = LibCallImpl.getProvides())
466 Provides->emitEnumEntry(OS);
467 else
468 OS << "RTLIB::UNKNOWN_LIBCALL";
469 OS << ", // ";
470 LibCallImpl.emitEnumEntry(OS);
471 OS << '\n';
472 }
473
474 OS << "};\n\n";
475
476 OS << "RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::getLibcallFromImpl("
477 "RTLIB::LibcallImpl Impl) const {\n"
478 " switch (Impl) {\n";
479 for (const RuntimeLibcallImpl &LibCallImpl :
480 Libcalls.getRuntimeLibcallImplDefList()) {
481 if (!LibCallImpl.isLongDouble())
482 continue;
483
484 OS << " case ";
485 LibCallImpl.emitEnumEntry(OS);
486 OS << ":\n return selectLongDoubleLibcall(";
487 LibCallImpl.getProvidesF80()->emitEnumEntry(OS);
488 OS << ", ";
489 LibCallImpl.getProvidesF128()->emitEnumEntry(OS);
490 OS << ", ";
491 LibCallImpl.getProvidesPPCF128()->emitEnumEntry(OS);
492 OS << ");\n";
493 }
494
495 OS << " default:\n"
496 " return ImplToLibcall[Impl];\n"
497 " }\n"
498 "}\n\n";
499 }
500
501 emitNameMatchHashTable(OS, OffsetTable&: Table);
502}
503
504using Signature = std::vector<StringRef>;
505
506static Signature getSignature(const Record *R) {
507 const auto *Tys = R->getValueAsListInit(FieldName: "ArgumentTypes");
508 Signature Sig;
509 Sig.reserve(n: Tys->size() + 1);
510 const Record *RetType = R->getValueAsOptionalDef(FieldName: "ReturnType");
511 if (RetType && RetType->getName() != "NoneType")
512 Sig.push_back(x: RetType->getName());
513 for (unsigned I = 0, E = Tys->size(); I < E; ++I)
514 Sig.push_back(x: Tys->getElementAsRecord(Idx: I)->getName());
515 return Sig;
516}
517
518void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallSignatures(
519 raw_ostream &OS) const {
520 SequenceToOffsetTable<Signature> SignatureTable("NoFuncArgType");
521
522 const Signature NoSignature;
523 SignatureTable.add(Seq: NoSignature);
524 for (const RuntimeLibcallImpl &LibCallImpl :
525 Libcalls.getRuntimeLibcallImplDefList())
526 SignatureTable.add(Seq: getSignature(R: LibCallImpl.getDef()));
527 SignatureTable.layout();
528
529 IfDefEmitter IfDef(OS, "GET_INIT_RUNTIME_LIBCALL_SIGNATURES");
530
531 OS << R"(
532const FuncArgTypeID RTLIB::RuntimeLibcallsInfo::SignatureTable[] = {
533)";
534 SignatureTable.emit(OS, Print: [](raw_ostream &OS, StringRef E) { OS << E; });
535 OS << "};\n";
536
537 OS << R"(
538const uint16_t RTLIB::RuntimeLibcallsInfo::SignatureOffset[] = {
539)";
540 OS << formatv(Fmt: " {}, // RTLIB::Unsupported\n",
541 Vals: SignatureTable.get(Seq: NoSignature));
542 for (const RuntimeLibcallImpl &LibCallImpl :
543 Libcalls.getRuntimeLibcallImplDefList()) {
544 OS << formatv(Fmt: " {}, // ",
545 Vals: SignatureTable.get(Seq: getSignature(R: LibCallImpl.getDef())));
546 LibCallImpl.emitEnumEntry(OS);
547 OS << '\n';
548 }
549
550 OS << "};\n";
551
552 OS << "\nstatic ArrayRef<RTLIB::FuncArgTypeID>\n"
553 "getStructElementTypes(RTLIB::FuncArgTypeID ID) {\n"
554 " switch (ID) {\n";
555
556 for (const Record *R : Libcalls.getFuncArgTypeList()) {
557 if (!R->isSubClassOf(Name: "StructFuncArgType"))
558 continue;
559
560 OS << " case RTLIB::" << R->getName() << ": {\n"
561 << " static constexpr RTLIB::FuncArgTypeID Elts[] = {";
562
563 ListSeparator LS;
564 for (const Record *Elt : R->getValueAsListOfDefs(FieldName: "ElementTypes"))
565 OS << LS << "RTLIB::" << Elt->getName();
566
567 OS << "};\n"
568 " return Elts;\n"
569 " }\n";
570 }
571
572 OS << " default:\n"
573 " return {};\n"
574 " }\n"
575 "}\n";
576
577 OS << "\nstatic bool isInRegStruct(RTLIB::FuncArgTypeID ID) {\n"
578 " switch (ID) {\n";
579
580 bool HasInRegStruct = false;
581 for (const Record *R : Libcalls.getFuncArgTypeList()) {
582 if (R->isSubClassOf(Name: "StructFuncArgType") && R->getValueAsBit(FieldName: "InReg")) {
583 OS << " case RTLIB::" << R->getName() << ":\n";
584 HasInRegStruct = true;
585 }
586 }
587
588 if (HasInRegStruct)
589 OS << " return true;\n";
590
591 OS << " default:\n"
592 " return false;\n"
593 " }\n"
594 "}\n";
595}
596
597using AttrExprs = std::vector<std::string>;
598using ArgAttrEntry = std::tuple<unsigned, unsigned, StringRef, bool>;
599
600static AttrExprs getAttrExprs(const Record *R, const ListInit *L,
601 StringRef Kind) {
602 const Record *KindDef = R->getRecords().getDef(Name: Kind);
603 AttrExprs Exprs;
604 for (const Init *I : L->getElements()) {
605 const Record *A = cast<DefInit>(Val: I)->getDef();
606 if (!is_contained(Range: A->getValueAsListOfDefs(FieldName: "Properties"), Element: KindDef))
607 PrintFatalError(Rec: R, Msg: "'" + A->getName() + "' is not a valid " + Kind);
608
609 if (A->isSubClassOf(Name: "ValueAttr")) {
610 Exprs.push_back(x: A->getValueAsString(FieldName: "Expr").str());
611 } else if (A->isSubClassOf(Name: "EnumAttr")) {
612 Exprs.push_back(
613 x: formatv(Fmt: "Attribute::get(C, Attribute::{})", Vals: A->getName()).str());
614 } else {
615 PrintFatalError(Rec: R, Msg: "'" + A->getName() +
616 "' requires a value; use a ValueAttr");
617 }
618 }
619
620 sort(C&: Exprs);
621 return Exprs;
622}
623
624static ModRefInfo getModRefInfo(const Record *R, StringRef Field) {
625 StringRef Name = R->getValueAsDef(FieldName: Field)->getName();
626 return StringSwitch<ModRefInfo>(Name)
627 .Case(S: "NoModRef", Value: ModRefInfo::NoModRef)
628 .Case(S: "Ref", Value: ModRefInfo::Ref)
629 .Case(S: "Mod", Value: ModRefInfo::Mod)
630 .Case(S: "ModRef", Value: ModRefInfo::ModRef);
631}
632
633static std::optional<uint32_t> getMemoryEffects(const Record *R) {
634 const Record *Memory = R->getValueAsDef(FieldName: "Memory");
635 static constexpr std::pair<IRMemLocation, StringRef> Locations[] = {
636 {IRMemLocation::ArgMem, "ArgMem"},
637 {IRMemLocation::InaccessibleMem, "InaccessibleMem"},
638 {IRMemLocation::ErrnoMem, "ErrnoMem"},
639 {IRMemLocation::Other, "Other"},
640 {IRMemLocation::TargetMem0, "TargetMem0"},
641 {IRMemLocation::TargetMem1, "TargetMem1"},
642 };
643
644 MemoryEffects ME = MemoryEffects::none();
645 for (auto [Loc, Field] : Locations)
646 ME = ME.getWithModRef(Loc, MR: getModRefInfo(R: Memory, Field));
647 if (ME == MemoryEffects::unknown())
648 return std::nullopt;
649 return ME.toIntValue();
650}
651
652static StringRef getMinimalUIntType(size_t MaxValue) {
653 if (MaxValue <= std::numeric_limits<uint8_t>::max())
654 return "uint8_t";
655 if (MaxValue <= std::numeric_limits<uint16_t>::max())
656 return "uint16_t";
657 return "uint32_t";
658}
659
660namespace {
661struct FnAttrSet {
662 AttrExprs Attrs;
663 std::optional<uint32_t> Memory;
664
665 bool operator<(const FnAttrSet &RHS) const {
666 return std::tie(args: Attrs, args: Memory) < std::tie(args: RHS.Attrs, args: RHS.Memory);
667 }
668
669 bool empty() const { return Attrs.empty() && !Memory; }
670};
671} // namespace
672
673static void printAttributes(raw_ostream &OS, const AttrExprs &Exprs) {
674 for (StringRef Expr : Exprs)
675 OS.indent(NumSpaces: 6) << Expr << ",\n";
676}
677
678static void printFnAttributes(raw_ostream &OS, const FnAttrSet &FnAttrs) {
679 printAttributes(OS, Exprs: FnAttrs.Attrs);
680 if (FnAttrs.Memory) {
681 OS.indent(NumSpaces: 6) << formatv(Fmt: "Attribute::getWithMemoryEffects(C, "
682 "MemoryEffects::createFromIntValue({})),\n",
683 Vals: *FnAttrs.Memory);
684 }
685}
686
687void RuntimeLibcallEmitter::emitRuntimeLibcallAttributes(
688 raw_ostream &OS) const {
689 AttributeSetTable<FnAttrSet> FnAttrSets;
690 AttributeSetTable<AttrExprs> ArgAttrSets;
691 AttributeSequenceTable<ArgAttrEntry> ArgAttrSeqs;
692 std::vector<std::tuple<unsigned, unsigned, unsigned>> ImplAttrIDs;
693
694 FnAttrSets.add(Attrs: {});
695 ArgAttrSets.add(Attrs: {});
696 ArgAttrSeqs.add(Seq: {});
697
698 for (const RuntimeLibcallImpl &LibCallImpl :
699 Libcalls.getRuntimeLibcallImplDefList()) {
700 const Record *R = LibCallImpl.getDef();
701 std::vector<const Record *> ArgTypes =
702 R->getValueAsListOfDefs(FieldName: "ArgumentTypes");
703 const ListInit *ParamAttrs = R->getValueAsListInit(FieldName: "ParamAttrs");
704 std::vector<const Record *> ParamPointeeTypes =
705 R->getValueAsListOfDefs(FieldName: "ParamPointeeTypes");
706 bool AlignToElement = R->getValueAsBit(FieldName: "AlignPointeesToElementType");
707
708 if (ParamAttrs->size() > ArgTypes.size()) {
709 PrintFatalError(Rec: R, Msg: "ParamAttrs has " + Twine(ParamAttrs->size()) +
710 " entries but the function has " +
711 Twine(ArgTypes.size()) + " arguments");
712 }
713
714 if (ParamPointeeTypes.size() > ArgTypes.size()) {
715 PrintFatalError(Rec: R, Msg: "ParamPointeeTypes has " +
716 Twine(ParamPointeeTypes.size()) +
717 " entries but the function has " +
718 Twine(ArgTypes.size()) + " arguments");
719 }
720
721 unsigned FnAttrSetID = FnAttrSets.add(
722 Attrs: {.Attrs: getAttrExprs(R, L: R->getValueAsListInit(FieldName: "FnAttrs"), Kind: "FnAttr"),
723 .Memory: getMemoryEffects(R)});
724 unsigned RetAttrSetID = ArgAttrSets.add(
725 Attrs: getAttrExprs(R, L: R->getValueAsListInit(FieldName: "RetAttrs"), Kind: "RetAttr"));
726
727 AttributeSequenceTable<ArgAttrEntry>::Sequence ArgAttrSeq;
728 for (unsigned ArgNo = 0, E = ArgTypes.size(); ArgNo != E; ++ArgNo) {
729 AttrExprs Attrs;
730 if (ArgNo < ParamAttrs->size()) {
731 Attrs = getAttrExprs(R, L: cast<ListInit>(Val: ParamAttrs->getElement(Idx: ArgNo)),
732 Kind: "ParamAttr");
733 }
734
735 StringRef PointeeType;
736 if (ArgNo < ParamPointeeTypes.size() &&
737 ParamPointeeTypes[ArgNo]->getName() != "NoneType") {
738 if (ArgTypes[ArgNo]->getName() != "Ptr") {
739 PrintFatalError(Rec: R,
740 Msg: "ParamPointeeTypes entry for non-pointer argument " +
741 Twine(ArgNo));
742 }
743
744 PointeeType = ParamPointeeTypes[ArgNo]->getName();
745 }
746
747 if (Attrs.empty() && PointeeType.empty())
748 continue;
749
750 ArgAttrSeq.emplace_back(Args&: ArgNo, Args: ArgAttrSets.add(Attrs: std::move(Attrs)),
751 Args&: PointeeType,
752 Args: !PointeeType.empty() && AlignToElement);
753 }
754
755 ImplAttrIDs.emplace_back(args&: FnAttrSetID, args&: RetAttrSetID,
756 args: ArgAttrSeqs.add(Seq: std::move(ArgAttrSeq)));
757 }
758
759 ArgAttrSeqs.layout();
760
761 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_ATTRIBUTES");
762
763 OS << "static AttributeSet getLibcallFnAttributeSet(LLVMContext &C, "
764 "unsigned ID) {\n"
765 " switch (ID) {\n"
766 " default:\n"
767 " llvm_unreachable(\"invalid libcall attribute set\");";
768 FnAttrSets.emitCases(OS, Print: printFnAttributes);
769 OS << "\n }\n"
770 "}\n\n";
771
772 OS << "static AttributeSet getLibcallArgAttributeSet(LLVMContext &C, "
773 "unsigned ID) {\n"
774 " switch (ID) {\n"
775 " default:\n"
776 " llvm_unreachable(\"invalid libcall attribute set\");";
777 ArgAttrSets.emitCases(OS, Print: printAttributes);
778 OS << "\n }\n"
779 "}\n\n";
780
781 OS << "namespace {\n"
782 "struct LibcallArgAttrs {\n"
783 " uint8_t ArgNo;\n"
784 << " " << getMinimalUIntType(MaxValue: ArgAttrSets.size()) << " AttrSetID;\n"
785 << " RTLIB::FuncArgTypeID PointeeType;\n"
786 " bool AlignToElement;\n"
787 "};\n"
788 "} // namespace\n\n"
789 "static constexpr LibcallArgAttrs LibcallArgAttrTable[] = {\n";
790 ArgAttrSeqs.emitEntries(OS, Print: [](raw_ostream &OS, ArgAttrEntry Entry) {
791 auto [ArgNo, SetID, PointeeType, AlignToElement] = Entry;
792 OS << formatv(Fmt: "{{{}, {}, RTLIB::{}, {}}", Vals&: ArgNo, Vals&: SetID,
793 Vals: PointeeType.empty() ? "NoFuncArgType" : PointeeType,
794 Vals&: AlignToElement);
795 });
796 OS << "};\n\n";
797
798 OS << "static constexpr struct {\n"
799 << " " << getMinimalUIntType(MaxValue: ArgAttrSeqs.size()) << " Start;\n"
800 << " uint8_t Size;\n"
801 "} LibcallArgAttrSeqs[] = {\n";
802 ArgAttrSeqs.emitInfo(OS);
803 OS << "};\n\n";
804
805 OS << "static constexpr struct {\n"
806 << " " << getMinimalUIntType(MaxValue: FnAttrSets.size()) << " FnAttrSetID;\n"
807 << " " << getMinimalUIntType(MaxValue: ArgAttrSets.size()) << " RetAttrSetID;\n"
808 << " " << getMinimalUIntType(MaxValue: ArgAttrSeqs.numSequences())
809 << " ArgAttrSeqID;\n"
810 << "} LibcallImplAttrIDs[] = {\n"
811 " {0, 0, 0}, // RTLIB::Unsupported\n";
812 for (auto [LibCallImpl, IDs] :
813 zip_equal(t: Libcalls.getRuntimeLibcallImplDefList(), u&: ImplAttrIDs)) {
814 auto [FnAttrSetID, RetAttrSetID, ArgAttrSeqID] = IDs;
815 OS << formatv(Fmt: " {{{}, {}, {}}, // ", Vals&: FnAttrSetID, Vals&: RetAttrSetID,
816 Vals&: ArgAttrSeqID);
817 LibCallImpl.emitEnumEntry(OS);
818 OS << '\n';
819 }
820
821 OS << "};\n\n";
822
823 OS << "static AttributeSet getLibcallImplFnAttributes(LLVMContext &C,\n"
824 " RTLIB::LibcallImpl Impl) {\n"
825 " return getLibcallFnAttributeSet(C, "
826 "LibcallImplAttrIDs[Impl].FnAttrSetID);\n"
827 "}\n\n"
828 "static AttributeSet getLibcallImplRetAttributes(LLVMContext &C,\n"
829 " RTLIB::LibcallImpl Impl) {\n"
830 " return getLibcallArgAttributeSet(C, "
831 "LibcallImplAttrIDs[Impl].RetAttrSetID);\n"
832 "}\n\n"
833 "static ArrayRef<LibcallArgAttrs>\n"
834 "getLibcallImplArgAttributes(RTLIB::LibcallImpl Impl) {\n"
835 " auto [Start, Size] =\n"
836 " LibcallArgAttrSeqs[LibcallImplAttrIDs[Impl].ArgAttrSeqID];\n"
837 " return ArrayRef(LibcallArgAttrTable).slice(Start, Size);\n"
838 "}\n";
839}
840
841static void emitMaskedVectorLibcallSwitch(
842 raw_ostream &OS, StringRef FuncName, bool ToUnmasked,
843 ArrayRef<std::pair<const RuntimeLibcall *, const RuntimeLibcall *>>
844 MaskedToUnmasked) {
845 OS << "RTLIB::Libcall RTLIB::RuntimeLibcallsInfo::" << FuncName
846 << "(RTLIB::Libcall LC) {\n"
847 " switch (LC) {\n";
848 for (auto [Masked, Unmasked] : MaskedToUnmasked) {
849 OS << " case ";
850 if (ToUnmasked) {
851 Masked->emitEnumEntry(OS);
852 OS << ":\n return ";
853 Unmasked->emitEnumEntry(OS);
854 } else {
855 Unmasked->emitEnumEntry(OS);
856 OS << ":\n return ";
857 Masked->emitEnumEntry(OS);
858 }
859
860 OS << ";\n";
861 }
862
863 OS << " default:\n"
864 " return RTLIB::UNKNOWN_LIBCALL;\n"
865 " }\n"
866 "}\n";
867}
868
869void RuntimeLibcallEmitter::emitMaskedVectorLibcallMapping(
870 raw_ostream &OS) const {
871 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_MASKED_VECTOR_LIBCALL_MAPPING");
872
873 SmallVector<std::pair<const RuntimeLibcall *, const RuntimeLibcall *>, 0>
874 MaskedToUnmasked;
875 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList()) {
876 const Record *Def = LC.getDef();
877 if (Def->isSubClassOf(Name: "MaskedRuntimeLibcall")) {
878 MaskedToUnmasked.emplace_back(
879 Args: &LC, Args: Libcalls.getRuntimeLibcall(Def: Def->getValueAsDef(FieldName: "Unmasked")));
880 }
881 }
882
883 emitMaskedVectorLibcallSwitch(OS, FuncName: "getUnmaskedVectorLibcall",
884 /*ToUnmasked=*/true, MaskedToUnmasked);
885 OS << '\n';
886 emitMaskedVectorLibcallSwitch(OS, FuncName: "getMaskedVectorLibcall",
887 /*ToUnmasked=*/false, MaskedToUnmasked);
888}
889
890void RuntimeLibcallEmitter::emitPredicateGroups(
891 raw_ostream &OS, const Record *R,
892 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
893 SetVector<PredicateWithCC> &PredicateSorter, unsigned BaseIndent) const {
894 SmallVector<PredicateWithCC, 0> SortedPredicates =
895 PredicateSorter.takeVector();
896
897 llvm::sort(C&: SortedPredicates, Comp: [](PredicateWithCC A, PredicateWithCC B) {
898 StringRef AName = A.Availability ? A.Availability->getName() : "";
899 StringRef BName = B.Availability ? B.Availability->getName() : "";
900 if (AName != BName)
901 return AName < BName;
902 // Break name ties on the calling convention for a deterministic order.
903 StringRef ACC = A.CallingConv ? A.CallingConv->getName() : "";
904 StringRef BCC = B.CallingConv ? B.CallingConv->getName() : "";
905 return ACC < BCC;
906 });
907
908 for (PredicateWithCC Entry : SortedPredicates) {
909 AvailabilityPredicate SubsetPredicate(Entry.Availability);
910 unsigned IndentDepth = BaseIndent;
911
912 auto It = Pred2Funcs.find(Val: Entry);
913 if (It == Pred2Funcs.end())
914 continue;
915
916 // Shared-core deduplication can empty a bucket.
917 if (It->second.LibcallImpls.empty())
918 continue;
919
920 if (!SubsetPredicate.isAlwaysAvailable()) {
921 IndentDepth = BaseIndent + 2;
922
923 OS << indent(IndentDepth);
924 SubsetPredicate.emitIf(OS);
925 }
926
927 LibcallsWithCC &FuncsWithCC = It->second;
928
929 std::vector<const RuntimeLibcallImpl *> &Funcs = FuncsWithCC.LibcallImpls;
930
931 // Records which impls are available, not which is selected, so a libcall
932 // may have more than one. Order is irrelevant (each entry is a setAvailable
933 // call); sort by the impl enum for a deterministic order.
934 llvm::sort(C&: Funcs,
935 Comp: [](const RuntimeLibcallImpl *A, const RuntimeLibcallImpl *B) {
936 return A->getEnumVal() < B->getEnumVal();
937 });
938
939 OS << indent(IndentDepth + 2)
940 << "static const RTLIB::LibcallImpl LibraryCalls";
941 SubsetPredicate.emitTableVariableNameSuffix(OS);
942 if (FuncsWithCC.CallingConv)
943 OS << '_' << FuncsWithCC.CallingConv->getName();
944
945 OS << "[] = {\n";
946 for (const RuntimeLibcallImpl *LibCallImpl : Funcs) {
947 OS << indent(IndentDepth + 6);
948 LibCallImpl->emitEnumEntry(OS);
949 OS << ", // " << LibCallImpl->getLibcallFuncName() << '\n';
950 }
951
952 OS << indent(IndentDepth + 2) << "};\n\n"
953 << indent(IndentDepth + 2)
954 << "for (const RTLIB::LibcallImpl Impl : LibraryCalls";
955 SubsetPredicate.emitTableVariableNameSuffix(OS);
956 if (FuncsWithCC.CallingConv)
957 OS << '_' << FuncsWithCC.CallingConv->getName();
958
959 OS << ") {\n" << indent(IndentDepth + 4) << "setAvailable(Impl);\n";
960
961 if (FuncsWithCC.CallingConv) {
962 StringRef CCEnum =
963 FuncsWithCC.CallingConv->getValueAsString(FieldName: "CallingConv");
964 OS << indent(IndentDepth + 4) << "setLibcallImplCallingConv(Impl, "
965 << CCEnum << ");\n";
966 }
967
968 OS << indent(IndentDepth + 2) << "}\n";
969 OS << '\n';
970
971 if (!SubsetPredicate.isAlwaysAvailable()) {
972 OS << indent(IndentDepth);
973 SubsetPredicate.emitEndIf(OS);
974 OS << '\n';
975 }
976 }
977}
978
979void RuntimeLibcallEmitter::emitLibraryVariant(raw_ostream &OS,
980 ExpandedLibrary &EL) const {
981 AvailabilityPredicate LibPred(EL.Lib->getValueAsDef(FieldName: "Pred"));
982
983 if (!LibPred.isAlwaysAvailable()) {
984 OS << indent(2);
985 LibPred.emitIf(OS);
986 } else {
987 // Own block scope so per-variant `LibraryCalls` tables do not collide.
988 OS << indent(2) << "{\n";
989 }
990
991 emitPredicateGroups(OS, R: EL.Lib, Pred2Funcs&: EL.Pred2Funcs, PredicateSorter&: EL.PredicateSorter,
992 /*BaseIndent=*/2);
993
994 if (!LibPred.isAlwaysAvailable()) {
995 OS << indent(2);
996 LibPred.emitEndIf(OS);
997 } else {
998 OS << indent(2) << "}\n";
999 }
1000}
1001
1002void RuntimeLibcallEmitter::emitLibraryFunction(
1003 raw_ostream &OS, StringRef Name, ArrayRef<const Record *> Libs,
1004 ArrayRef<LibraryExclusion> Exclusions) const {
1005 OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibFuncs_";
1006 emitLibFuncSuffix(OS, Name);
1007 OS << "(const llvm::Triple &TT, "
1008 "ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, "
1009 "StringRef ABIName, "
1010 "LongDoubleFormat LongDoubleFormat, CallingConv::ID DefaultCC) {\n";
1011
1012 SmallVector<ExpandedLibrary, 2> Expanded;
1013 for (const Record *Lib : Libs) {
1014 ExpandedLibrary EL;
1015 EL.Lib = Lib;
1016
1017 // Expand this library's members with a library-local Func2Preds.
1018 SetTheory Sets;
1019 DenseMap<const RuntimeLibcallImpl *,
1020 std::pair<std::vector<const Record *>, const Record *>>
1021 Func2Preds;
1022 Sets.addExpander(ClassName: "LibcallImpls", std::make_unique<LibcallPredicateExpander>(
1023 args: Libcalls, args&: Func2Preds));
1024
1025 SetTheory::RecSet Elements;
1026 Sets.evaluate(Expr: Lib->getValueInit(FieldName: "Impls"), Elts&: Elements, Loc: Lib->getLoc());
1027
1028 EL.PredicateSorter.insert(
1029 X: PredicateWithCC()); // No predicate or CC override first.
1030
1031 for (const Record *Elt : Elements) {
1032 const RuntimeLibcallImpl *LibCallImpl =
1033 Libcalls.getRuntimeLibcallImpl(Def: Elt);
1034 if (!LibCallImpl) {
1035 PrintError(Rec: Lib, Msg: "entry for LibcallLibrary is not a RuntimeLibcallImpl");
1036 PrintNote(NoteLoc: Elt->getLoc(), Msg: "invalid entry `" + Elt->getName() + "`");
1037 continue;
1038 }
1039
1040 auto It = Func2Preds.find(Val: LibCallImpl);
1041 if (It == Func2Preds.end()) {
1042 EL.Pred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: LibCallImpl);
1043 EL.Unconditional.insert(X: LibCallImpl);
1044 continue;
1045 }
1046
1047 for (const Record *Pred : It->second.first) {
1048 const Record *CC = It->second.second;
1049 PredicateWithCC Key(Pred, CC);
1050 auto &Entry = EL.Pred2Funcs[Key];
1051 Entry.LibcallImpls.push_back(x: LibCallImpl);
1052 Entry.CallingConv = CC;
1053 EL.PredicateSorter.insert(X: Key);
1054 }
1055 }
1056
1057 Expanded.push_back(Elt: std::move(EL));
1058 }
1059
1060 // Impls unconditional in every variant are emitted once and stripped from
1061 // each variant, so the shared core is not repeated.
1062 SetVector<const RuntimeLibcallImpl *> SharedCore;
1063 if (Expanded.size() > 1) {
1064 for (const RuntimeLibcallImpl *Impl : Expanded.front().Unconditional) {
1065 if (all_of(Range: drop_begin(RangeOrContainer&: Expanded), P: [&](const ExpandedLibrary &EL) {
1066 return EL.Unconditional.contains(key: Impl);
1067 }))
1068 SharedCore.insert(X: Impl);
1069 }
1070 }
1071
1072 if (!SharedCore.empty()) {
1073 // Emit the shared core once, then strip it from every variant.
1074 DenseMap<PredicateWithCC, LibcallsWithCC> CorePred2Funcs;
1075 SetVector<PredicateWithCC> CoreSorter;
1076 CoreSorter.insert(X: PredicateWithCC());
1077 for (const RuntimeLibcallImpl *Impl : SharedCore)
1078 CorePred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: Impl);
1079 emitPredicateGroups(OS, R: Libs.front(), Pred2Funcs&: CorePred2Funcs, PredicateSorter&: CoreSorter,
1080 /*BaseIndent=*/0);
1081
1082 for (ExpandedLibrary &EL : Expanded) {
1083 auto &Funcs = EL.Pred2Funcs[PredicateWithCC()].LibcallImpls;
1084 llvm::erase_if(C&: Funcs, P: [&](const RuntimeLibcallImpl *Impl) {
1085 return SharedCore.contains(key: Impl);
1086 });
1087 }
1088 }
1089
1090 // Mark a variant deferred when it re-adds an impl its own consumer excludes
1091 // (same predicate, via a LibraryExclusion). Such a variant must be emitted
1092 // after the exclusion so the re-add wins while the exclusion still suppresses
1093 // every other variant's contribution.
1094 for (ExpandedLibrary &EL : Expanded) {
1095 const Record *ELPred = EL.Lib->getValueAsDef(FieldName: "Pred");
1096 SetVector<const RuntimeLibcallImpl *> Impls;
1097 for (const auto &[Key, Funcs] : EL.Pred2Funcs)
1098 Impls.insert(Start: Funcs.LibcallImpls.begin(), End: Funcs.LibcallImpls.end());
1099
1100 for (const LibraryExclusion &Excl : Exclusions) {
1101 if (Excl.TriplePred != ELPred)
1102 continue;
1103 if (any_of(Range: Excl.Impls, P: [&](const RuntimeLibcallImpl *Impl) {
1104 return Impls.contains(key: Impl);
1105 })) {
1106 EL.Deferred = true;
1107 break;
1108 }
1109 }
1110 }
1111
1112 // Emit each non-deferred variant under its own Pred.
1113 for (ExpandedLibrary &EL : Expanded) {
1114 if (!EL.Deferred)
1115 emitLibraryVariant(OS, EL);
1116 }
1117
1118 // Emit each LibraryExclusion.
1119 for (const LibraryExclusion &Excl : Exclusions) {
1120 OS << '\n' << indent(2);
1121 AvailabilityPredicate ExcludePred(Excl.TriplePred);
1122 ExcludePred.emitIf(OS);
1123 for (const RuntimeLibcallImpl *Impl : Excl.Impls) {
1124 OS << indent(4) << "setUnavailable(";
1125 Impl->emitEnumEntry(OS);
1126 OS << "); // " << Impl->getLibcallFuncName() << '\n';
1127 }
1128
1129 OS << indent(2);
1130 ExcludePred.emitEndIf(OS);
1131 }
1132
1133 // Deferred variants: emitted after exclusions so a target's own re-adds
1134 // override its own LibraryExclusion opt-outs (the exclusion still applied
1135 // above to every other variant's contributions).
1136 for (ExpandedLibrary &EL : Expanded) {
1137 if (EL.Deferred)
1138 emitLibraryVariant(OS, EL);
1139 }
1140
1141 OS << "}\n\n";
1142}
1143
1144// The setAvailableLibFuncs_ suffix (and merge key) for a library: its shared
1145// LibraryName normally, or its own def name if isolated (so it does not merge).
1146static StringRef libFuncKey(const Record *Lib) {
1147 return Lib->getValueAsBit(FieldName: "Isolated") ? Lib->getName()
1148 : Lib->getValueAsString(FieldName: "LibraryName");
1149}
1150
1151MapVector<StringRef, std::vector<const Record *>>
1152RuntimeLibcallEmitter::collectLibrariesByName() const {
1153 MapVector<StringRef, std::vector<const Record *>> LibsByName;
1154 for (const Record *Lib : Records.getAllDerivedDefinitions(ClassName: "LibcallLibrary"))
1155 LibsByName[libFuncKey(Lib)].push_back(x: Lib);
1156 return LibsByName;
1157}
1158
1159std::vector<StringRef> RuntimeLibcallEmitter::collectLibraryNames() const {
1160 std::vector<StringRef> Names;
1161 for (const Record *Lib : Records.getAllDerivedDefinitions(ClassName: "LibcallLibrary"))
1162 Names.push_back(x: Lib->getValueAsString(FieldName: "LibraryName"));
1163 llvm::sort(C&: Names);
1164 Names.erase(first: llvm::unique(R&: Names), last: Names.end());
1165 return Names;
1166}
1167
1168void RuntimeLibcallEmitter::emitRuntimeLibcallsInfoMemberDecls(
1169 raw_ostream &OS) const {
1170 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALLS_INFO_MEMBER_DECLS");
1171 for (const auto &[Name, Libs] : collectLibrariesByName()) {
1172 OS << "void setAvailableLibFuncs_";
1173 emitLibFuncSuffix(OS, Name);
1174 OS << "(const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
1175 "FloatABI::ABIType FloatABI, StringRef ABIName, "
1176 "LongDoubleFormat LongDoubleFormat, CallingConv::ID DefaultCC);\n";
1177 }
1178
1179 OS << "bool setAvailableLibraryLibcalls(RTLIB::RuntimeLibrary Library, "
1180 "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
1181 "FloatABI::ABIType FloatABI, StringRef ABIName, "
1182 "LongDoubleFormat LongDoubleFormat);\n"
1183 "static CallingConv::ID getDefaultLibcallCallingConv("
1184 "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
1185 "FloatABI::ABIType FloatABI, StringRef ABIName, "
1186 "LongDoubleFormat LongDoubleFormat);\n";
1187}
1188
1189std::vector<RuntimeLibcallEmitter::SystemLibraryDispatch>
1190RuntimeLibcallEmitter::collectSystemLibraryDispatch() const {
1191 std::vector<StringRef> LibraryNames = collectLibraryNames();
1192
1193 std::vector<SystemLibraryDispatch> Systems;
1194 for (const Record *R :
1195 Records.getAllDerivedDefinitions(ClassName: "SystemRuntimeLibrary")) {
1196 SystemLibraryDispatch &S = Systems.emplace_back();
1197 S.System = R;
1198 if (const Record *DefaultCCClass =
1199 R->getValueAsDef(FieldName: "DefaultLibcallCallingConv"))
1200 S.DefaultCC = DefaultCCClass->getValueAsString(FieldName: "CallingConv").trim();
1201
1202 for (StringRef Name : R->getValueAsListOfStrings(FieldName: "DefaultLibraries")) {
1203 if (!llvm::binary_search(Range&: LibraryNames, Value&: Name)) {
1204 PrintError(Rec: R, Msg: "SystemRuntimeLibrary default library `" + Name +
1205 "` is not the name of a LibcallLibrary");
1206 continue;
1207 }
1208
1209 S.Libs.push_back(Elt: Name);
1210 }
1211
1212 llvm::sort(C&: S.Libs);
1213 S.Libs.erase(CS: llvm::unique(R&: S.Libs), CE: S.Libs.end());
1214 }
1215
1216 return Systems;
1217}
1218
1219void RuntimeLibcallEmitter::emitLibraryFunctions(raw_ostream &OS) const {
1220 std::vector<StringRef> LibraryNames = collectLibraryNames();
1221
1222 // Collect each shared library's LibraryExclusions, keyed by library name, so
1223 // its library function can emit them.
1224 MapVector<StringRef, std::vector<LibraryExclusion>> ExclusionsByLibName;
1225 for (const Record *R : Records.getAllDerivedDefinitions(ClassName: "LibraryExclusion")) {
1226 StringRef LibName = R->getValueAsString(FieldName: "LibraryName");
1227 if (!llvm::binary_search(Range&: LibraryNames, Value&: LibName)) {
1228 PrintError(Rec: R, Msg: "LibraryExclusion library `" + LibName +
1229 "` is not the name of a LibcallLibrary");
1230 continue;
1231 }
1232
1233 LibraryExclusion Excl{.TriplePred: R->getValueAsDef(FieldName: "Pred"), .Impls: {}};
1234 for (const Record *ExcludeRec : R->getValueAsListOfDefs(FieldName: "Exclude")) {
1235 if (const RuntimeLibcallImpl *Impl =
1236 Libcalls.getRuntimeLibcallImpl(Def: ExcludeRec))
1237 Excl.Impls.push_back(x: Impl);
1238 }
1239
1240 if (!Excl.Impls.empty())
1241 ExclusionsByLibName[LibName].push_back(x: std::move(Excl));
1242 }
1243
1244 for (const auto &[Name, Libs] : collectLibrariesByName())
1245 emitLibraryFunction(OS, Name, Libs, Exclusions: ExclusionsByLibName.lookup(Key: Name));
1246}
1247
1248void RuntimeLibcallEmitter::emitLibraryByNameSetCalls(
1249 raw_ostream &OS, ArrayRef<SystemLibraryDispatch> Systems) const {
1250 constexpr StringLiteral Params =
1251 "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
1252 "FloatABI::ABIType FloatABI, StringRef ABIName, "
1253 "LongDoubleFormat LongDoubleFormat";
1254
1255 OS << "CallingConv::ID "
1256 "llvm::RTLIB::RuntimeLibcallsInfo::getDefaultLibcallCallingConv("
1257 << Params << ") {\n";
1258 for (const SystemLibraryDispatch &S : Systems) {
1259 if (S.DefaultCC.empty())
1260 continue;
1261
1262 AvailabilityPredicate TriplePred(S.System->getValueAsDef(FieldName: "TriplePred"));
1263 OS << indent(2);
1264 TriplePred.emitIf(OS);
1265 OS << indent(4) << "return " << S.DefaultCC << ";\n" << indent(2);
1266 TriplePred.emitEndIf(OS);
1267 }
1268 OS << " return CallingConv::C;\n"
1269 "}\n\n";
1270
1271 OS << "ArrayRef<RTLIB::RuntimeLibrary> "
1272 "llvm::RTLIB::RuntimeLibcallsInfo::getDefaultRuntimeLibraries("
1273 "const llvm::Triple &TT) {\n";
1274 for (const SystemLibraryDispatch &S : Systems) {
1275 if (S.Libs.empty())
1276 continue;
1277
1278 AvailabilityPredicate TriplePred(S.System->getValueAsDef(FieldName: "TriplePred"));
1279 OS << indent(2);
1280 TriplePred.emitIf(OS);
1281 OS << indent(4) << "static constexpr RTLIB::RuntimeLibrary Libs[] = {\n";
1282 for (StringRef Name : S.Libs) {
1283 OS << indent(6);
1284 emitRuntimeLibraryEnumerator(OS, Name);
1285 OS << ",\n";
1286 }
1287 OS << indent(4) << "};\n" << indent(4) << "return Libs;\n" << indent(2);
1288 TriplePred.emitEndIf(OS);
1289 }
1290 OS << " return {};\n"
1291 "}\n\n";
1292
1293 MapVector<StringRef, std::vector<const Record *>> IsolatedByName;
1294 DenseSet<StringRef> MergedNames;
1295 for (const Record *Lib : Records.getAllDerivedDefinitions(ClassName: "LibcallLibrary")) {
1296 StringRef Name = Lib->getValueAsString(FieldName: "LibraryName");
1297 if (!Lib->getValueAsBit(FieldName: "Isolated")) {
1298 MergedNames.insert(V: Name);
1299 continue;
1300 }
1301
1302 if (AvailabilityPredicate(Lib->getValueAsDef(FieldName: "Pred")).isAlwaysAvailable())
1303 PrintError(Rec: Lib, Msg: "isolated LibcallLibrary must have a predicate");
1304 IsolatedByName[Name].push_back(x: Lib);
1305 }
1306
1307 for (const auto &[Name, Libs] : IsolatedByName) {
1308 if (!MergedNames.contains(V: Name)) {
1309 PrintError(Rec: Libs.front(), Msg: "isolated LibcallLibrary has no non-isolated "
1310 "library named `" +
1311 Name + "`");
1312 }
1313 }
1314
1315 OS << "bool llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibraryLibcalls("
1316 "RTLIB::RuntimeLibrary Library, "
1317 << Params
1318 << ") {\n"
1319 " const CallingConv::ID DefaultCC = getDefaultLibcallCallingConv(TT, "
1320 "ExceptionModel, FloatABI, ABIName, LongDoubleFormat);\n"
1321 " switch (Library) {\n";
1322
1323 auto EmitCall = [&OS](unsigned Ind, StringRef FuncSuffix) {
1324 OS << indent(Ind) << "setAvailableLibFuncs_";
1325 emitLibFuncSuffix(OS, Name: FuncSuffix);
1326 OS << "(TT, ExceptionModel, FloatABI, ABIName, LongDoubleFormat, "
1327 "DefaultCC);\n"
1328 << indent(Ind) << "return true;\n";
1329 };
1330
1331 for (const auto &[Name, Libs] : collectLibrariesByName()) {
1332 if (Libs.front()->getValueAsBit(FieldName: "Isolated"))
1333 continue;
1334
1335 OS << " case ";
1336 emitRuntimeLibraryEnumerator(OS, Name);
1337 OS << ":\n";
1338
1339 for (const Record *Isolated : IsolatedByName.lookup(Key: Name)) {
1340 AvailabilityPredicate LibPred(Isolated->getValueAsDef(FieldName: "Pred"));
1341 OS << indent(4);
1342 LibPred.emitIf(OS);
1343 EmitCall(6, Isolated->getName());
1344 OS << indent(4);
1345 LibPred.emitEndIf(OS);
1346 }
1347
1348 EmitCall(4, Name);
1349 }
1350
1351 OS << " default:\n"
1352 " return false;\n"
1353 " }\n"
1354 "}\n\n";
1355}
1356
1357// Scalar FP type suffixes in the argument order of RTLIB::getFPLibCall, paired
1358// with the llvm::Type predicate used by the IR-level mapping.
1359static constexpr std::pair<StringRef, StringRef> ScalarFPSuffixes[] = {
1360 {"F32", "isFloatTy()"}, {"F64", "isDoubleTy()"},
1361 {"F80", "isX86_FP80Ty()"}, {"F128", "isFP128Ty()"},
1362 {"PPCF128", "isPPC_FP128Ty()"},
1363};
1364
1365static std::vector<FPLibcallFamily>
1366collectFPLibcallFamilies(const RecordKeeper &Records) {
1367 std::vector<FPLibcallFamily> Families;
1368 for (const Record *R :
1369 Records.getAllDerivedDefinitions(ClassName: "RuntimeLibcallFamily"))
1370 Families.emplace_back(args&: R);
1371 llvm::sort(C&: Families, Comp: [](const FPLibcallFamily &A, const FPLibcallFamily &B) {
1372 return A.Base < B.Base;
1373 });
1374 return Families;
1375}
1376
1377DenseSet<StringRef> RuntimeLibcallEmitter::collectLibcallNames() const {
1378 DenseSet<StringRef> LibcallNames;
1379 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList())
1380 LibcallNames.insert(V: LC.getName());
1381 return LibcallNames;
1382}
1383
1384void RuntimeLibcallEmitter::checkFPLibcallFamilies(
1385 ArrayRef<FPLibcallFamily> Families,
1386 const DenseSet<StringRef> &LibcallNames) const {
1387 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
1388 for (const FPLibcallFamily &Family : Families)
1389 for (StringRef Intrinsic : Family.Intrinsics)
1390 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
1391 llvm::sort(C&: IntrinsicToBase);
1392
1393 for (size_t I = 1, E = IntrinsicToBase.size(); I < E; ++I) {
1394 if (IntrinsicToBase[I].first == IntrinsicToBase[I - 1].first)
1395 PrintFatalError(
1396 Msg: "intrinsic '" + IntrinsicToBase[I].first +
1397 "' is mapped by multiple RuntimeLibcallFamily records ('" +
1398 IntrinsicToBase[I - 1].second + "' and '" +
1399 IntrinsicToBase[I].second + "')");
1400 }
1401
1402 for (const FPLibcallFamily &Family : Families) {
1403 bool AnyScalarLibcall = any_of(
1404 Range: ScalarFPSuffixes, P: [&](const std::pair<StringRef, StringRef> &Entry) {
1405 return LibcallNames.contains(V: (Family.Base + "_" + Entry.first).str());
1406 });
1407 if (!AnyScalarLibcall)
1408 PrintFatalError(Msg: "no runtime libcall found for base name '" + Family.Base +
1409 "'");
1410 }
1411}
1412
1413/// Generate the declarations for the RTLIB::get<base>(EVT) selectors.
1414void RuntimeLibcallEmitter::emitFPLibcallSelectorDecls(
1415 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families) const {
1416 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTOR_DECLS");
1417 for (const FPLibcallFamily &Family : Families)
1418 OS << "LLVM_ABI Libcall get" << Family.Base << "(EVT VT);\n";
1419}
1420
1421/// Generate the backend RTLIB::get<base>(EVT) selectors from the floating-point
1422/// libcall families.
1423void RuntimeLibcallEmitter::emitFPLibcallSelectors(
1424 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
1425 const DenseSet<StringRef> &LibcallNames) const {
1426 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTORS");
1427
1428 // Only emit a libcall enumerator if it actually exists in the declared set.
1429 auto scalarEnum = [&](StringRef Base, StringRef Suffix) -> std::string {
1430 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
1431 return ("RTLIB::" + Base + "_" + Suffix).str();
1432 return "RTLIB::UNKNOWN_LIBCALL";
1433 };
1434
1435 for (const FPLibcallFamily &Family : Families) {
1436 StringRef Base = Family.Base;
1437 OS << "RTLIB::Libcall llvm::RTLIB::get" << Base << "(EVT VT) {\n";
1438
1439 if (!Family.VectorSuffixes.empty()) {
1440 OS << " if (VT.isVector()) {\n"
1441 " if (!VT.isSimple())\n"
1442 " return RTLIB::UNKNOWN_LIBCALL;\n"
1443 " switch (VT.getSimpleVT().SimpleTy) {\n";
1444 for (StringRef Suffix : Family.VectorSuffixes) {
1445 OS << " case MVT::" << Suffix.lower()
1446 << ":\n return RTLIB::" << Base << "_" << Suffix << ";\n";
1447 }
1448 OS << " default:\n"
1449 " return RTLIB::UNKNOWN_LIBCALL;\n"
1450 " }\n"
1451 " }\n";
1452 }
1453
1454 OS << " return getFPLibCall(VT";
1455 for (auto [Suffix, Pred] : ScalarFPSuffixes)
1456 OS << ", " << scalarEnum(Base, Suffix);
1457 OS << ");\n}\n\n";
1458 }
1459}
1460
1461/// Emit the mapping from floating-point math intrinsics to the runtime libcall
1462/// they may lower to, keyed by intrinsic ID and floating-point type. This is
1463/// the IR-level counterpart to the backend's RTLIB::getXXX(EVT) selectors.
1464void RuntimeLibcallEmitter::emitGetLibcallForIntrinsic(
1465 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
1466 const DenseSet<StringRef> &LibcallNames) const {
1467 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_INTRINSIC_TO_LIBCALL");
1468
1469 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
1470 for (const FPLibcallFamily &Family : Families)
1471 for (StringRef Intrinsic : Family.Intrinsics)
1472 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
1473 llvm::sort(C&: IntrinsicToBase);
1474
1475 MapVector<StringRef, SmallVector<StringRef, 2>> BaseToIntrinsics;
1476 for (auto [Intrinsic, Base] : IntrinsicToBase)
1477 BaseToIntrinsics[Base].push_back(Elt: Intrinsic);
1478
1479 OS << "RTLIB::Libcall "
1480 "llvm::RTLIB::RuntimeLibcallsInfo::getLibcallForIntrinsic("
1481 "Intrinsic::ID ID, FunctionType *FTy) {\n"
1482 " Type *Ty = FTy->getReturnType();\n"
1483 " if (!Ty->isFloatingPointTy()) {\n"
1484 " for (Type *ParamTy : FTy->params()) {\n"
1485 " if (ParamTy->isFloatingPointTy()) {\n"
1486 " Ty = ParamTy;\n"
1487 " break;\n"
1488 " }\n"
1489 " }\n"
1490 " }\n"
1491 " if (!Ty->isFloatingPointTy())\n"
1492 " return RTLIB::UNKNOWN_LIBCALL;\n"
1493 " switch (ID) {\n";
1494
1495 for (const auto &[Base, Intrinsics] : BaseToIntrinsics) {
1496 SmallVector<std::pair<StringRef, StringRef>, 5> Arms;
1497 for (auto [Suffix, Pred] : ScalarFPSuffixes)
1498 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
1499 Arms.emplace_back(Args&: Suffix, Args&: Pred);
1500
1501 for (StringRef Intrinsic : Intrinsics)
1502 OS << " case Intrinsic::" << Intrinsic << ":\n";
1503 for (auto [Suffix, Pred] : Arms)
1504 OS << " if (Ty->" << Pred << ")\n return RTLIB::" << Base << "_"
1505 << Suffix << ";\n";
1506 OS << " return RTLIB::UNKNOWN_LIBCALL;\n";
1507 }
1508
1509 OS << " default:\n"
1510 " return RTLIB::UNKNOWN_LIBCALL;\n"
1511 " }\n"
1512 "}\n";
1513}
1514
1515void RuntimeLibcallEmitter::run(raw_ostream &OS) {
1516 emitSourceFileHeader(Desc: "Runtime LibCalls Source Fragment", OS, Record: Records);
1517 emitGetRuntimeLibcallEnum(OS);
1518
1519 emitGetInitRuntimeLibcallNames(OS);
1520 emitGetInitRuntimeLibcallSignatures(OS);
1521 emitRuntimeLibcallAttributes(OS);
1522 emitMaskedVectorLibcallMapping(OS);
1523
1524 emitRuntimeLibcallsInfoMemberDecls(OS);
1525
1526 {
1527 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALLS_INFO");
1528 std::vector<SystemLibraryDispatch> Systems = collectSystemLibraryDispatch();
1529 emitLibraryFunctions(OS);
1530 emitLibraryByNameSetCalls(OS, Systems);
1531 }
1532
1533 std::vector<FPLibcallFamily> FPFamilies = collectFPLibcallFamilies(Records);
1534 DenseSet<StringRef> LibcallNames = collectLibcallNames();
1535 checkFPLibcallFamilies(Families: FPFamilies, LibcallNames);
1536 emitFPLibcallSelectorDecls(OS, Families: FPFamilies);
1537 emitFPLibcallSelectors(OS, Families: FPFamilies, LibcallNames);
1538 emitGetLibcallForIntrinsic(OS, Families: FPFamilies, LibcallNames);
1539}
1540
1541static TableGen::Emitter::OptClass<RuntimeLibcallEmitter>
1542 X("gen-runtime-libcalls", "Generate RuntimeLibcalls");
1543