| 1 | //===-- BasicBlockSectionsProfileReader.cpp -------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Implementation of the basic block sections profile reader pass. It parses |
| 10 | // and stores the basic block sections profile file (which is specified via the |
| 11 | // `-basic-block-sections` flag). |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h" |
| 16 | #include "llvm/ADT/DenseSet.h" |
| 17 | #include "llvm/ADT/SmallSet.h" |
| 18 | #include "llvm/ADT/SmallString.h" |
| 19 | #include "llvm/ADT/SmallVector.h" |
| 20 | #include "llvm/ADT/StringMap.h" |
| 21 | #include "llvm/ADT/StringRef.h" |
| 22 | #include "llvm/IR/DebugInfoMetadata.h" |
| 23 | #include "llvm/Pass.h" |
| 24 | #include "llvm/Support/Error.h" |
| 25 | #include "llvm/Support/ErrorHandling.h" |
| 26 | #include "llvm/Support/LineIterator.h" |
| 27 | #include "llvm/Support/MemoryBuffer.h" |
| 28 | #include "llvm/Support/Path.h" |
| 29 | #include "llvm/Support/UniqueBBID.h" |
| 30 | #include <llvm/ADT/STLExtras.h> |
| 31 | |
| 32 | using namespace llvm; |
| 33 | |
| 34 | char BasicBlockSectionsProfileReaderWrapperPass::ID = 0; |
| 35 | INITIALIZE_PASS(BasicBlockSectionsProfileReaderWrapperPass, |
| 36 | "bbsections-profile-reader" , |
| 37 | "Reads and parses a basic block sections profile." , false, |
| 38 | false) |
| 39 | |
| 40 | Expected<UniqueBBID> |
| 41 | BasicBlockSectionsProfileReader::parseUniqueBBID(StringRef S) const { |
| 42 | SmallVector<StringRef, 2> Parts; |
| 43 | S.split(A&: Parts, Separator: '.'); |
| 44 | if (Parts.size() > 2) |
| 45 | return createProfileParseError(Message: Twine("unable to parse basic block id: '" ) + |
| 46 | S + "'" ); |
| 47 | unsigned long long BaseBBID; |
| 48 | if (getAsUnsignedInteger(Str: Parts[0], Radix: 10, Result&: BaseBBID)) |
| 49 | return createProfileParseError( |
| 50 | Message: Twine("unable to parse BB id: '" + Parts[0]) + |
| 51 | "': unsigned integer expected" ); |
| 52 | unsigned long long CloneID = 0; |
| 53 | if (Parts.size() > 1 && getAsUnsignedInteger(Str: Parts[1], Radix: 10, Result&: CloneID)) |
| 54 | return createProfileParseError(Message: Twine("unable to parse clone id: '" ) + |
| 55 | Parts[1] + "': unsigned integer expected" ); |
| 56 | return UniqueBBID{.BaseID: static_cast<unsigned>(BaseBBID), |
| 57 | .CloneID: static_cast<unsigned>(CloneID)}; |
| 58 | } |
| 59 | |
| 60 | bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const { |
| 61 | return !getClusterInfoForFunction(FuncName).empty(); |
| 62 | } |
| 63 | |
| 64 | SmallVector<BBClusterInfo> |
| 65 | BasicBlockSectionsProfileReader::getClusterInfoForFunction( |
| 66 | StringRef FuncName) const { |
| 67 | auto R = ProgramOptimizationProfile.find(Key: getAliasName(FuncName)); |
| 68 | return R != ProgramOptimizationProfile.end() ? R->second.ClusterInfo |
| 69 | : SmallVector<BBClusterInfo>(); |
| 70 | } |
| 71 | |
| 72 | SmallVector<SmallVector<unsigned>> |
| 73 | BasicBlockSectionsProfileReader::getClonePathsForFunction( |
| 74 | StringRef FuncName) const { |
| 75 | auto R = ProgramOptimizationProfile.find(Key: getAliasName(FuncName)); |
| 76 | return R != ProgramOptimizationProfile.end() |
| 77 | ? R->second.ClonePaths |
| 78 | : SmallVector<SmallVector<unsigned>>(); |
| 79 | } |
| 80 | |
| 81 | uint64_t BasicBlockSectionsProfileReader::getEdgeCount( |
| 82 | StringRef FuncName, const UniqueBBID &SrcBBID, |
| 83 | const UniqueBBID &SinkBBID) const { |
| 84 | const CFGProfile *CFG = getFunctionCFGProfile(FuncName); |
| 85 | if (CFG == nullptr) |
| 86 | return 0; |
| 87 | auto NodeIt = CFG->EdgeCounts.find(Val: SrcBBID); |
| 88 | if (NodeIt == CFG->EdgeCounts.end()) |
| 89 | return 0; |
| 90 | auto EdgeIt = NodeIt->second.find(Val: SinkBBID); |
| 91 | if (EdgeIt == NodeIt->second.end()) |
| 92 | return 0; |
| 93 | return EdgeIt->second; |
| 94 | } |
| 95 | |
| 96 | SmallVector<CallsiteID> |
| 97 | BasicBlockSectionsProfileReader::getPrefetchTargetsForFunction( |
| 98 | StringRef FuncName) const { |
| 99 | auto R = ProgramOptimizationProfile.find(Key: getAliasName(FuncName)); |
| 100 | return R != ProgramOptimizationProfile.end() ? R->second.PrefetchTargets |
| 101 | : SmallVector<CallsiteID>(); |
| 102 | } |
| 103 | |
| 104 | // Reads the version 1 basic block sections profile. Profile for each function |
| 105 | // is encoded as follows: |
| 106 | // m <module_name> |
| 107 | // f <function_name_1> <function_name_2> ... |
| 108 | // c <bb_id_1> <bb_id_2> <bb_id_3> |
| 109 | // c <bb_id_4> <bb_id_5> |
| 110 | // ... |
| 111 | // Module name specifier (starting with 'm') is optional and allows |
| 112 | // distinguishing profile for internal-linkage functions with the same name. If |
| 113 | // not specified, it will apply to any function with the same name. Function |
| 114 | // name specifier (starting with 'f') can specify multiple function name |
| 115 | // aliases. Basic block clusters are specified by 'c' and specify the cluster of |
| 116 | // basic blocks, and the internal order in which they must be placed in the same |
| 117 | // section. |
| 118 | // This profile can also specify cloning paths which instruct the compiler to |
| 119 | // clone basic blocks along a path. The cloned blocks are then specified in the |
| 120 | // cluster information. |
| 121 | // The following profile lists two cloning paths (starting with 'p') for |
| 122 | // function bar and places the total 9 blocks within two clusters. The first two |
| 123 | // blocks of a cloning path specify the edge along which the path is cloned. For |
| 124 | // instance, path 1 (1 -> 3 -> 4) instructs that 3 and 4 must be cloned along |
| 125 | // the edge 1->3. Within the given clusters, each cloned block is identified by |
| 126 | // "<original block id>.<clone id>". For instance, 3.1 represents the first |
| 127 | // clone of block 3. Original blocks are specified just with their block ids. A |
| 128 | // block cloned multiple times appears with distinct clone ids. The CFG for bar |
| 129 | // is shown below before and after cloning with its final clusters labeled. |
| 130 | // |
| 131 | // f main |
| 132 | // f bar |
| 133 | // p 1 3 4 # cloning path 1 |
| 134 | // p 4 2 # cloning path 2 |
| 135 | // c 1 3.1 4.1 6 # basic block cluster 1 |
| 136 | // c 0 2 3 4 2.1 5 # basic block cluster 2 |
| 137 | // **************************************************************************** |
| 138 | // function bar before and after cloning with basic block clusters shown. |
| 139 | // **************************************************************************** |
| 140 | // .... .............. |
| 141 | // 0 -------+ : 0 :---->: 1 ---> 3.1 : |
| 142 | // | | : | : :........ | : |
| 143 | // v v : v : : v : |
| 144 | // +--> 2 --> 5 1 ~~~~~~> +---: 2 : : 4.1: clsuter 1 |
| 145 | // | | | | : | : : | : |
| 146 | // | v | | : v ....... : v : |
| 147 | // | 3 <------+ | : 3 <--+ : : 6 : |
| 148 | // | | | : | | : :....: |
| 149 | // | v | : v | : |
| 150 | // +--- 4 ---> 6 | : 4 | : |
| 151 | // | : | | : |
| 152 | // | : v | : |
| 153 | // | :2.1---+ : cluster 2 |
| 154 | // | : | ......: |
| 155 | // | : v : |
| 156 | // +-->: 5 : |
| 157 | // .... |
| 158 | // **************************************************************************** |
| 159 | // This profile can also specify prefetch targets (starting with 't') which |
| 160 | // instruct the compiler to emit a prefetch symbol for the given target. |
| 161 | // A prefetch target is specified by a pair "<bbid>,<subblock_index>" where |
| 162 | // bbid specifies the target basic block and subblock_index is a zero-based |
| 163 | // index. Subblock 0 refers to the region at the beginning of the block up to |
| 164 | // the first callsite. Subblock `i > 0` refers to the region immediately after |
| 165 | // the `i`-th callsite up to the `i+1`-th callsite (or the end of the block). |
| 166 | // The prefetch target is always emitted at the beginning of the subblock. |
| 167 | // This is the beginning of the basic block for `i = 0` and immediately after |
| 168 | // the `i`-th call for every `i > 0`. |
| 169 | // |
| 170 | // Example: A basic block in function "foo" with BBID 10 and two call |
| 171 | // instructions (call_A, call_B). This block is conceptually split into |
| 172 | // subblocks, with the prefetch target symbol emitted at the beginning of each |
| 173 | // subblock. |
| 174 | // |
| 175 | // +----------------------------------+ |
| 176 | // | __llvm_prefetch_target_foo_10_0: | <- Subblock 0 (before call_A) |
| 177 | // | Instruction 1 | |
| 178 | // | Instruction 2 | |
| 179 | // | call_A (Callsite 0) | |
| 180 | // | __llvm_prefetch_target_foo_10_1: | <--- Subblock 1 (after call_A, |
| 181 | // | | before call_B) |
| 182 | // | Instruction 3 | |
| 183 | // | call_B (Callsite 1) | |
| 184 | // | __llvm_prefetch_target_foo_10_2: | <--- Subblock 2 (after call_B, |
| 185 | // | | before call_C) |
| 186 | // | Instruction 4 | |
| 187 | // +----------------------------------+ |
| 188 | // |
| 189 | Error BasicBlockSectionsProfileReader::ReadV1Profile() { |
| 190 | auto FI = ProgramOptimizationProfile.end(); |
| 191 | |
| 192 | // Current cluster ID corresponding to this function. |
| 193 | unsigned CurrentCluster = 0; |
| 194 | // Current position in the current cluster. |
| 195 | unsigned CurrentPosition = 0; |
| 196 | |
| 197 | // Temporary set to ensure every basic block ID appears once in the clusters |
| 198 | // of a function. |
| 199 | DenseSet<UniqueBBID> FuncBBIDs; |
| 200 | |
| 201 | // Debug-info-based module filename for the current function. Empty string |
| 202 | // means no filename. |
| 203 | StringRef DIFilename; |
| 204 | |
| 205 | for (; !LineIt.is_at_eof(); ++LineIt) { |
| 206 | StringRef S(*LineIt); |
| 207 | char Specifier = S[0]; |
| 208 | S = S.drop_front().trim(); |
| 209 | SmallVector<StringRef, 4> Values; |
| 210 | S.split(A&: Values, Separator: ' '); |
| 211 | switch (Specifier) { |
| 212 | case '@': |
| 213 | continue; |
| 214 | case 'm': // Module name speicifer. |
| 215 | if (Values.size() != 1) { |
| 216 | return createProfileParseError(Message: Twine("invalid module name value: '" ) + |
| 217 | S + "'" ); |
| 218 | } |
| 219 | DIFilename = sys::path::remove_leading_dotslash(path: Values[0]); |
| 220 | continue; |
| 221 | case 'f': { // Function names specifier. |
| 222 | bool FunctionFound = any_of(Range&: Values, P: [&](StringRef Alias) { |
| 223 | auto It = FunctionNameToDIFilename.find(Key: Alias); |
| 224 | // No match if this function name is not found in this module. |
| 225 | if (It == FunctionNameToDIFilename.end()) |
| 226 | return false; |
| 227 | // Return a match if debug-info-filename is not specified. Otherwise, |
| 228 | // check for equality. |
| 229 | return DIFilename.empty() || It->second == DIFilename; |
| 230 | }); |
| 231 | if (!FunctionFound) { |
| 232 | // Skip the following profile by setting the profile iterator (FI) to |
| 233 | // the past-the-end element. |
| 234 | FI = ProgramOptimizationProfile.end(); |
| 235 | DIFilename = "" ; |
| 236 | continue; |
| 237 | } |
| 238 | for (size_t i = 1; i < Values.size(); ++i) |
| 239 | FuncAliasMap.try_emplace(Key: Values[i], Args&: Values.front()); |
| 240 | |
| 241 | // Prepare for parsing clusters of this function name. |
| 242 | // Start a new cluster map for this function name. |
| 243 | auto R = ProgramOptimizationProfile.try_emplace(Key: Values.front()); |
| 244 | // Report error when multiple profiles have been specified for the same |
| 245 | // function. |
| 246 | if (!R.second) |
| 247 | return createProfileParseError(Message: "duplicate profile for function '" + |
| 248 | Values.front() + "'" ); |
| 249 | FI = R.first; |
| 250 | CurrentCluster = 0; |
| 251 | FuncBBIDs.clear(); |
| 252 | // We won't need DIFilename anymore. Clean it up to avoid its application |
| 253 | // on the next function. |
| 254 | DIFilename = "" ; |
| 255 | continue; |
| 256 | } |
| 257 | case 'c': // Basic block cluster specifier. |
| 258 | // Skip the profile when we the profile iterator (FI) refers to the |
| 259 | // past-the-end element. |
| 260 | if (FI == ProgramOptimizationProfile.end()) |
| 261 | continue; |
| 262 | // Reset current cluster position. |
| 263 | CurrentPosition = 0; |
| 264 | for (auto BasicBlockIDStr : Values) { |
| 265 | auto BasicBlockID = parseUniqueBBID(S: BasicBlockIDStr); |
| 266 | if (!BasicBlockID) |
| 267 | return BasicBlockID.takeError(); |
| 268 | if (!FuncBBIDs.insert(V: *BasicBlockID).second) |
| 269 | return createProfileParseError( |
| 270 | Message: Twine("duplicate basic block id found '" ) + BasicBlockIDStr + |
| 271 | "'" ); |
| 272 | |
| 273 | FI->second.ClusterInfo.emplace_back(Args: BBClusterInfo{ |
| 274 | .BBID: *std::move(BasicBlockID), .ClusterID: CurrentCluster, .PositionInCluster: CurrentPosition++}); |
| 275 | } |
| 276 | CurrentCluster++; |
| 277 | continue; |
| 278 | case 'p': { // Basic block cloning path specifier. |
| 279 | // Skip the profile when we the profile iterator (FI) refers to the |
| 280 | // past-the-end element. |
| 281 | if (FI == ProgramOptimizationProfile.end()) |
| 282 | continue; |
| 283 | SmallSet<unsigned, 5> BBsInPath; |
| 284 | FI->second.ClonePaths.push_back(Elt: {}); |
| 285 | for (size_t I = 0; I < Values.size(); ++I) { |
| 286 | auto BaseBBIDStr = Values[I]; |
| 287 | unsigned long long BaseBBID = 0; |
| 288 | if (getAsUnsignedInteger(Str: BaseBBIDStr, Radix: 10, Result&: BaseBBID)) |
| 289 | return createProfileParseError(Message: Twine("unsigned integer expected: '" ) + |
| 290 | BaseBBIDStr + "'" ); |
| 291 | if (I != 0 && !BBsInPath.insert(V: BaseBBID).second) |
| 292 | return createProfileParseError( |
| 293 | Message: Twine("duplicate cloned block in path: '" ) + BaseBBIDStr + "'" ); |
| 294 | FI->second.ClonePaths.back().push_back(Elt: BaseBBID); |
| 295 | } |
| 296 | continue; |
| 297 | } |
| 298 | case 'g': { // CFG profile specifier. |
| 299 | // Skip the profile when we the profile iterator (FI) refers to the |
| 300 | // past-the-end element. |
| 301 | if (FI == ProgramOptimizationProfile.end()) |
| 302 | continue; |
| 303 | // For each node, its CFG profile is encoded as |
| 304 | // <src>:<count>,<sink_1>:<count_1>,<sink_2>:<count_2>,... |
| 305 | for (auto BasicBlockEdgeProfile : Values) { |
| 306 | if (BasicBlockEdgeProfile.empty()) |
| 307 | continue; |
| 308 | SmallVector<StringRef, 4> NodeEdgeCounts; |
| 309 | BasicBlockEdgeProfile.split(A&: NodeEdgeCounts, Separator: ','); |
| 310 | UniqueBBID SrcBBID; |
| 311 | for (size_t i = 0; i < NodeEdgeCounts.size(); ++i) { |
| 312 | auto [BBIDStr, CountStr] = NodeEdgeCounts[i].split(Separator: ':'); |
| 313 | auto BBID = parseUniqueBBID(S: BBIDStr); |
| 314 | if (!BBID) |
| 315 | return BBID.takeError(); |
| 316 | unsigned long long Count = 0; |
| 317 | if (getAsUnsignedInteger(Str: CountStr, Radix: 10, Result&: Count)) |
| 318 | return createProfileParseError( |
| 319 | Message: Twine("unsigned integer expected: '" ) + CountStr + "'" ); |
| 320 | if (i == 0) { |
| 321 | // The first element represents the source and its total count. |
| 322 | FI->second.CFG.NodeCounts[SrcBBID = *BBID] = Count; |
| 323 | continue; |
| 324 | } |
| 325 | FI->second.CFG.EdgeCounts[SrcBBID][*BBID] = Count; |
| 326 | } |
| 327 | } |
| 328 | continue; |
| 329 | } |
| 330 | case 'h': { // Basic block hash secifier. |
| 331 | // Skip the profile when the profile iterator (FI) refers to the |
| 332 | // past-the-end element. |
| 333 | if (FI == ProgramOptimizationProfile.end()) |
| 334 | continue; |
| 335 | for (auto BBIDHashStr : Values) { |
| 336 | auto [BBIDStr, HashStr] = BBIDHashStr.split(Separator: ':'); |
| 337 | unsigned long long BBID = 0, Hash = 0; |
| 338 | if (getAsUnsignedInteger(Str: BBIDStr, Radix: 10, Result&: BBID)) |
| 339 | return createProfileParseError(Message: Twine("unsigned integer expected: '" ) + |
| 340 | BBIDStr + "'" ); |
| 341 | if (getAsUnsignedInteger(Str: HashStr, Radix: 16, Result&: Hash)) |
| 342 | return createProfileParseError( |
| 343 | Message: Twine("unsigned integer expected in hex format: '" ) + HashStr + |
| 344 | "'" ); |
| 345 | FI->second.CFG.BBHashes[BBID] = Hash; |
| 346 | } |
| 347 | continue; |
| 348 | } |
| 349 | case 't': { // Callsite target specifier. |
| 350 | // Skip the profile when we the profile iterator (FI) refers to the |
| 351 | // past-the-end element. |
| 352 | if (FI == ProgramOptimizationProfile.end()) |
| 353 | continue; |
| 354 | SmallVector<StringRef, 2> PrefetchTargetStr; |
| 355 | Values[0].split(A&: PrefetchTargetStr, Separator: ','); |
| 356 | if (PrefetchTargetStr.size() != 2) |
| 357 | return createProfileParseError(Message: Twine("Callsite target expected: " ) + |
| 358 | Values[0]); |
| 359 | auto TargetBBID = parseUniqueBBID(S: PrefetchTargetStr[0]); |
| 360 | if (!TargetBBID) |
| 361 | return TargetBBID.takeError(); |
| 362 | unsigned long long CallsiteIndex; |
| 363 | if (getAsUnsignedInteger(Str: PrefetchTargetStr[1], Radix: 10, Result&: CallsiteIndex)) |
| 364 | return createProfileParseError(Message: Twine("signed integer expected: '" ) + |
| 365 | PrefetchTargetStr[1]); |
| 366 | FI->second.PrefetchTargets.push_back( |
| 367 | Elt: CallsiteID{.BBID: *TargetBBID, .CallsiteIndex: static_cast<unsigned>(CallsiteIndex)}); |
| 368 | continue; |
| 369 | } |
| 370 | default: |
| 371 | return createProfileParseError(Message: Twine("invalid specifier: '" ) + |
| 372 | Twine(Specifier) + "'" ); |
| 373 | } |
| 374 | llvm_unreachable("should not break from this switch statement" ); |
| 375 | } |
| 376 | return Error::success(); |
| 377 | } |
| 378 | |
| 379 | Error BasicBlockSectionsProfileReader::ReadV0Profile() { |
| 380 | auto FI = ProgramOptimizationProfile.end(); |
| 381 | // Current cluster ID corresponding to this function. |
| 382 | unsigned CurrentCluster = 0; |
| 383 | // Current position in the current cluster. |
| 384 | unsigned CurrentPosition = 0; |
| 385 | |
| 386 | // Temporary set to ensure every basic block ID appears once in the clusters |
| 387 | // of a function. |
| 388 | SmallSet<unsigned, 4> FuncBBIDs; |
| 389 | |
| 390 | for (; !LineIt.is_at_eof(); ++LineIt) { |
| 391 | StringRef S(*LineIt); |
| 392 | if (S[0] == '@') |
| 393 | continue; |
| 394 | // Check for the leading "!" |
| 395 | if (!S.consume_front(Prefix: "!" ) || S.empty()) |
| 396 | break; |
| 397 | // Check for second "!" which indicates a cluster of basic blocks. |
| 398 | if (S.consume_front(Prefix: "!" )) { |
| 399 | // Skip the profile when we the profile iterator (FI) refers to the |
| 400 | // past-the-end element. |
| 401 | if (FI == ProgramOptimizationProfile.end()) |
| 402 | continue; |
| 403 | SmallVector<StringRef, 4> BBIDs; |
| 404 | S.split(A&: BBIDs, Separator: ' '); |
| 405 | // Reset current cluster position. |
| 406 | CurrentPosition = 0; |
| 407 | for (auto BBIDStr : BBIDs) { |
| 408 | unsigned long long BBID; |
| 409 | if (getAsUnsignedInteger(Str: BBIDStr, Radix: 10, Result&: BBID)) |
| 410 | return createProfileParseError(Message: Twine("unsigned integer expected: '" ) + |
| 411 | BBIDStr + "'" ); |
| 412 | if (!FuncBBIDs.insert(V: BBID).second) |
| 413 | return createProfileParseError( |
| 414 | Message: Twine("duplicate basic block id found '" ) + BBIDStr + "'" ); |
| 415 | |
| 416 | FI->second.ClusterInfo.emplace_back( |
| 417 | Args: BBClusterInfo({.BBID: {.BaseID: static_cast<unsigned>(BBID), .CloneID: 0}, |
| 418 | .ClusterID: CurrentCluster, |
| 419 | .PositionInCluster: CurrentPosition++})); |
| 420 | } |
| 421 | CurrentCluster++; |
| 422 | } else { |
| 423 | // This is a function name specifier. It may include a debug info filename |
| 424 | // specifier starting with `M=`. |
| 425 | auto [AliasesStr, DIFilenameStr] = S.split(Separator: ' '); |
| 426 | SmallString<128> DIFilename; |
| 427 | if (DIFilenameStr.starts_with(Prefix: "M=" )) { |
| 428 | DIFilename = |
| 429 | sys::path::remove_leading_dotslash(path: DIFilenameStr.substr(Start: 2)); |
| 430 | if (DIFilename.empty()) |
| 431 | return createProfileParseError(Message: "empty module name specifier" ); |
| 432 | } else if (!DIFilenameStr.empty()) { |
| 433 | return createProfileParseError(Message: "unknown string found: '" + |
| 434 | DIFilenameStr + "'" ); |
| 435 | } |
| 436 | // Function aliases are separated using '/'. We use the first function |
| 437 | // name for the cluster info mapping and delegate all other aliases to |
| 438 | // this one. |
| 439 | SmallVector<StringRef, 4> Aliases; |
| 440 | AliasesStr.split(A&: Aliases, Separator: '/'); |
| 441 | bool FunctionFound = any_of(Range&: Aliases, P: [&](StringRef Alias) { |
| 442 | auto It = FunctionNameToDIFilename.find(Key: Alias); |
| 443 | // No match if this function name is not found in this module. |
| 444 | if (It == FunctionNameToDIFilename.end()) |
| 445 | return false; |
| 446 | // Return a match if debug-info-filename is not specified. Otherwise, |
| 447 | // check for equality. |
| 448 | return DIFilename.empty() || It->second == DIFilename; |
| 449 | }); |
| 450 | if (!FunctionFound) { |
| 451 | // Skip the following profile by setting the profile iterator (FI) to |
| 452 | // the past-the-end element. |
| 453 | FI = ProgramOptimizationProfile.end(); |
| 454 | continue; |
| 455 | } |
| 456 | for (size_t i = 1; i < Aliases.size(); ++i) |
| 457 | FuncAliasMap.try_emplace(Key: Aliases[i], Args&: Aliases.front()); |
| 458 | |
| 459 | // Prepare for parsing clusters of this function name. |
| 460 | // Start a new cluster map for this function name. |
| 461 | auto R = ProgramOptimizationProfile.try_emplace(Key: Aliases.front()); |
| 462 | // Report error when multiple profiles have been specified for the same |
| 463 | // function. |
| 464 | if (!R.second) |
| 465 | return createProfileParseError(Message: "duplicate profile for function '" + |
| 466 | Aliases.front() + "'" ); |
| 467 | FI = R.first; |
| 468 | CurrentCluster = 0; |
| 469 | FuncBBIDs.clear(); |
| 470 | } |
| 471 | } |
| 472 | return Error::success(); |
| 473 | } |
| 474 | |
| 475 | // Basic Block Sections can be enabled for a subset of machine basic blocks. |
| 476 | // This is done by passing a file containing names of functions for which basic |
| 477 | // block sections are desired. Additionally, machine basic block ids of the |
| 478 | // functions can also be specified for a finer granularity. Moreover, a cluster |
| 479 | // of basic blocks could be assigned to the same section. |
| 480 | // Optionally, a debug-info filename can be specified for each function to allow |
| 481 | // distinguishing internal-linkage functions of the same name. |
| 482 | // A file with basic block sections for all of function main and three blocks |
| 483 | // for function foo (of which 1 and 2 are placed in a cluster) looks like this: |
| 484 | // (Profile for function foo is only loaded when its debug-info filename |
| 485 | // matches 'path/to/foo_file.cc'). |
| 486 | // ---------------------------- |
| 487 | // list.txt: |
| 488 | // !main |
| 489 | // !foo M=path/to/foo_file.cc |
| 490 | // !!1 2 |
| 491 | // !!4 |
| 492 | Error BasicBlockSectionsProfileReader::ReadProfile() { |
| 493 | assert(MBuf); |
| 494 | |
| 495 | unsigned long long Version = 0; |
| 496 | StringRef FirstLine(*LineIt); |
| 497 | if (FirstLine.consume_front(Prefix: "v" )) { |
| 498 | if (getAsUnsignedInteger(Str: FirstLine, Radix: 10, Result&: Version)) { |
| 499 | return createProfileParseError(Message: Twine("version number expected: '" ) + |
| 500 | FirstLine + "'" ); |
| 501 | } |
| 502 | if (Version > 1) { |
| 503 | return createProfileParseError(Message: Twine("invalid profile version: " ) + |
| 504 | Twine(Version)); |
| 505 | } |
| 506 | ++LineIt; |
| 507 | } |
| 508 | |
| 509 | switch (Version) { |
| 510 | case 0: |
| 511 | // TODO: Deprecate V0 once V1 is fully integrated downstream. |
| 512 | return ReadV0Profile(); |
| 513 | case 1: |
| 514 | return ReadV1Profile(); |
| 515 | default: |
| 516 | llvm_unreachable("Invalid profile version." ); |
| 517 | } |
| 518 | } |
| 519 | |
| 520 | bool BasicBlockSectionsProfileReaderWrapperPass::doInitialization(Module &M) { |
| 521 | if (!BBSPR.MBuf) |
| 522 | return false; |
| 523 | // Get the function name to debug info filename mapping. |
| 524 | BBSPR.FunctionNameToDIFilename.clear(); |
| 525 | for (const Function &F : M) { |
| 526 | SmallString<128> DIFilename; |
| 527 | if (F.isDeclaration()) |
| 528 | continue; |
| 529 | DISubprogram *Subprogram = F.getSubprogram(); |
| 530 | if (Subprogram) { |
| 531 | llvm::DICompileUnit *CU = Subprogram->getUnit(); |
| 532 | if (CU) |
| 533 | DIFilename = sys::path::remove_leading_dotslash(path: CU->getFilename()); |
| 534 | } |
| 535 | [[maybe_unused]] bool inserted = |
| 536 | BBSPR.FunctionNameToDIFilename.try_emplace(Key: F.getName(), Args&: DIFilename) |
| 537 | .second; |
| 538 | assert(inserted); |
| 539 | } |
| 540 | if (auto Err = BBSPR.ReadProfile()) |
| 541 | report_fatal_error(Err: std::move(Err)); |
| 542 | return false; |
| 543 | } |
| 544 | |
| 545 | AnalysisKey BasicBlockSectionsProfileReaderAnalysis::Key; |
| 546 | |
| 547 | BasicBlockSectionsProfileReader |
| 548 | BasicBlockSectionsProfileReaderAnalysis::run(Function &F, |
| 549 | FunctionAnalysisManager &AM) { |
| 550 | return BasicBlockSectionsProfileReader(TM->getBBSectionsFuncListBuf()); |
| 551 | } |
| 552 | |
| 553 | bool BasicBlockSectionsProfileReaderWrapperPass::isFunctionHot( |
| 554 | StringRef FuncName) const { |
| 555 | return BBSPR.isFunctionHot(FuncName); |
| 556 | } |
| 557 | |
| 558 | SmallVector<BBClusterInfo> |
| 559 | BasicBlockSectionsProfileReaderWrapperPass::getClusterInfoForFunction( |
| 560 | StringRef FuncName) const { |
| 561 | return BBSPR.getClusterInfoForFunction(FuncName); |
| 562 | } |
| 563 | |
| 564 | SmallVector<SmallVector<unsigned>> |
| 565 | BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction( |
| 566 | StringRef FuncName) const { |
| 567 | return BBSPR.getClonePathsForFunction(FuncName); |
| 568 | } |
| 569 | |
| 570 | const CFGProfile * |
| 571 | BasicBlockSectionsProfileReaderWrapperPass::getFunctionCFGProfile( |
| 572 | StringRef FuncName) const { |
| 573 | return BBSPR.getFunctionCFGProfile(FuncName); |
| 574 | } |
| 575 | |
| 576 | uint64_t BasicBlockSectionsProfileReaderWrapperPass::getEdgeCount( |
| 577 | StringRef FuncName, const UniqueBBID &SrcBBID, |
| 578 | const UniqueBBID &SinkBBID) const { |
| 579 | return BBSPR.getEdgeCount(FuncName, SrcBBID, SinkBBID); |
| 580 | } |
| 581 | |
| 582 | SmallVector<CallsiteID> |
| 583 | BasicBlockSectionsProfileReaderWrapperPass::getPrefetchTargetsForFunction( |
| 584 | StringRef FuncName) const { |
| 585 | return BBSPR.getPrefetchTargetsForFunction(FuncName); |
| 586 | } |
| 587 | |
| 588 | BasicBlockSectionsProfileReader & |
| 589 | BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() { |
| 590 | return BBSPR; |
| 591 | } |
| 592 | |
| 593 | ImmutablePass *llvm::createBasicBlockSectionsProfileReaderWrapperPass( |
| 594 | const MemoryBuffer *Buf) { |
| 595 | return new BasicBlockSectionsProfileReaderWrapperPass(Buf); |
| 596 | } |
| 597 | |