| 1 | //===- LoopAnnotationImporter.cpp - Loop annotation import ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "LoopAnnotationImporter.h" |
| 10 | #include "llvm/ADT/ScopeExit.h" |
| 11 | #include "llvm/ADT/SmallPtrSet.h" |
| 12 | #include "llvm/IR/Constants.h" |
| 13 | |
| 14 | using namespace mlir; |
| 15 | using namespace mlir::LLVM; |
| 16 | using namespace mlir::LLVM::detail; |
| 17 | |
| 18 | namespace { |
| 19 | /// Helper class that keeps the state of one metadata to attribute conversion. |
| 20 | struct LoopMetadataConversion { |
| 21 | /// Converts a loop ID, owning the recursion state for its follow-ups. |
| 22 | static LoopAnnotationAttr convert(const llvm::MDNode *node, Location loc, |
| 23 | LoopAnnotationImporter &importer); |
| 24 | |
| 25 | private: |
| 26 | LoopMetadataConversion( |
| 27 | const llvm::MDNode *node, Location loc, DistinctAttr id, |
| 28 | LoopAnnotationImporter &loopAnnotationImporter, |
| 29 | llvm::SmallPtrSetImpl<const llvm::MDNode *> &activeNodes) |
| 30 | : node(node), loc(loc), id(id), |
| 31 | loopAnnotationImporter(loopAnnotationImporter), ctx(loc->getContext()), |
| 32 | activeNodes(activeNodes) {} |
| 33 | |
| 34 | /// Converts a follow-up using the current recursion state. |
| 35 | LoopAnnotationAttr convertFollowup(const llvm::MDNode *followup); |
| 36 | |
| 37 | /// Converts the properties after the validated loop ID or follow-up header. |
| 38 | LoopAnnotationAttr convertProperties(); |
| 39 | |
| 40 | /// Initializes the shared state for the conversion member functions. |
| 41 | LogicalResult initConversionState(); |
| 42 | |
| 43 | /// Helper function to get and erase a property. |
| 44 | const llvm::MDNode *lookupAndEraseProperty(StringRef name); |
| 45 | |
| 46 | /// Helper functions to lookup and convert MDNodes into a specifc attribute |
| 47 | /// kind. These functions return null-attributes if there is no node with the |
| 48 | /// specified name, or failure, if the node is ill-formatted. |
| 49 | FailureOr<BoolAttr> lookupUnitNode(StringRef name); |
| 50 | FailureOr<BoolAttr> lookupBoolNode(StringRef name, bool negated = false); |
| 51 | FailureOr<BoolAttr> lookupIntNodeAsBoolAttr(StringRef name); |
| 52 | FailureOr<IntegerAttr> lookupIntNode(StringRef name); |
| 53 | FailureOr<uint32_t> lookupUInt32Node(StringRef name); |
| 54 | FailureOr<SmallVector<llvm::MDNode *>> lookupMDNodes(StringRef name); |
| 55 | FailureOr<LoopAnnotationAttr> lookupFollowupNode(StringRef name); |
| 56 | FailureOr<BoolAttr> lookupBooleanUnitNode(StringRef enableName, |
| 57 | StringRef disableName, |
| 58 | bool negated = false); |
| 59 | |
| 60 | /// Conversion functions for sub-attributes. |
| 61 | FailureOr<LoopVectorizeAttr> convertVectorizeAttr(); |
| 62 | FailureOr<LoopInterleaveAttr> convertInterleaveAttr(); |
| 63 | FailureOr<LoopUnrollAttr> convertUnrollAttr(); |
| 64 | FailureOr<LoopUnrollAndJamAttr> convertUnrollAndJamAttr(); |
| 65 | FailureOr<LoopLICMAttr> convertLICMAttr(); |
| 66 | FailureOr<LoopDistributeAttr> convertDistributeAttr(); |
| 67 | FailureOr<LoopPipelineAttr> convertPipelineAttr(); |
| 68 | FailureOr<LoopPeeledAttr> convertPeeledAttr(); |
| 69 | FailureOr<LoopUnswitchAttr> convertUnswitchAttr(); |
| 70 | FailureOr<SmallVector<AccessGroupAttr>> convertParallelAccesses(); |
| 71 | FusedLoc convertStartLoc(); |
| 72 | FailureOr<FusedLoc> convertEndLoc(); |
| 73 | |
| 74 | llvm::SmallVector<llvm::DILocation *, 2> locations; |
| 75 | llvm::StringMap<const llvm::MDNode *> propertyMap; |
| 76 | const llvm::MDNode *node; |
| 77 | Location loc; |
| 78 | DistinctAttr id; |
| 79 | LoopAnnotationImporter &loopAnnotationImporter; |
| 80 | MLIRContext *ctx; |
| 81 | llvm::SmallPtrSetImpl<const llvm::MDNode *> &activeNodes; |
| 82 | }; |
| 83 | } // namespace |
| 84 | |
| 85 | LogicalResult LoopMetadataConversion::initConversionState() { |
| 86 | for (const llvm::MDOperand &operand : llvm::drop_begin(RangeOrContainer: node->operands())) { |
| 87 | if (auto *diLoc = dyn_cast<llvm::DILocation>(Val: operand)) { |
| 88 | locations.push_back(Elt: diLoc); |
| 89 | continue; |
| 90 | } |
| 91 | |
| 92 | auto *property = dyn_cast<llvm::MDNode>(Val: operand); |
| 93 | if (!property) |
| 94 | return emitWarning(loc) << "expected all loop properties to be either " |
| 95 | "debug locations or metadata nodes" ; |
| 96 | |
| 97 | if (property->getNumOperands() == 0) |
| 98 | return emitWarning(loc) << "cannot import empty loop property" ; |
| 99 | |
| 100 | auto *nameNode = dyn_cast<llvm::MDString>(Val: property->getOperand(I: 0)); |
| 101 | if (!nameNode) |
| 102 | return emitWarning(loc) << "cannot import loop property without a name" ; |
| 103 | StringRef name = nameNode->getString(); |
| 104 | |
| 105 | bool succ = propertyMap.try_emplace(Key: name, Args&: property).second; |
| 106 | if (!succ) |
| 107 | return emitWarning(loc) |
| 108 | << "cannot import loop properties with duplicated names " << name; |
| 109 | } |
| 110 | |
| 111 | return success(); |
| 112 | } |
| 113 | |
| 114 | const llvm::MDNode * |
| 115 | LoopMetadataConversion::lookupAndEraseProperty(StringRef name) { |
| 116 | auto it = propertyMap.find(Key: name); |
| 117 | if (it == propertyMap.end()) |
| 118 | return nullptr; |
| 119 | const llvm::MDNode *property = it->getValue(); |
| 120 | propertyMap.erase(I: it); |
| 121 | return property; |
| 122 | } |
| 123 | |
| 124 | FailureOr<BoolAttr> LoopMetadataConversion::lookupUnitNode(StringRef name) { |
| 125 | const llvm::MDNode *property = lookupAndEraseProperty(name); |
| 126 | if (!property) |
| 127 | return BoolAttr(nullptr); |
| 128 | |
| 129 | if (property->getNumOperands() != 1) |
| 130 | return emitWarning(loc) |
| 131 | << "expected metadata node " << name << " to hold no value" ; |
| 132 | |
| 133 | return BoolAttr::get(context: ctx, value: true); |
| 134 | } |
| 135 | |
| 136 | FailureOr<BoolAttr> LoopMetadataConversion::lookupBooleanUnitNode( |
| 137 | StringRef enableName, StringRef disableName, bool negated) { |
| 138 | auto enable = lookupUnitNode(name: enableName); |
| 139 | auto disable = lookupUnitNode(name: disableName); |
| 140 | if (failed(Result: enable) || failed(Result: disable)) |
| 141 | return failure(); |
| 142 | |
| 143 | if (*enable && *disable) |
| 144 | return emitWarning(loc) |
| 145 | << "expected metadata nodes " << enableName << " and " << disableName |
| 146 | << " to be mutually exclusive." ; |
| 147 | |
| 148 | if (*enable) |
| 149 | return BoolAttr::get(context: ctx, value: !negated); |
| 150 | |
| 151 | if (*disable) |
| 152 | return BoolAttr::get(context: ctx, value: negated); |
| 153 | return BoolAttr(nullptr); |
| 154 | } |
| 155 | |
| 156 | FailureOr<BoolAttr> LoopMetadataConversion::lookupBoolNode(StringRef name, |
| 157 | bool negated) { |
| 158 | const llvm::MDNode *property = lookupAndEraseProperty(name); |
| 159 | if (!property) |
| 160 | return BoolAttr(nullptr); |
| 161 | |
| 162 | auto emitNodeWarning = [&]() { |
| 163 | return emitWarning(loc) |
| 164 | << "expected metadata node " << name << " to hold a boolean value" ; |
| 165 | }; |
| 166 | |
| 167 | if (property->getNumOperands() != 2) |
| 168 | return emitNodeWarning(); |
| 169 | llvm::ConstantInt *val = |
| 170 | llvm::mdconst::dyn_extract<llvm::ConstantInt>(MD: property->getOperand(I: 1)); |
| 171 | if (!val || val->getBitWidth() != 1) |
| 172 | return emitNodeWarning(); |
| 173 | |
| 174 | return BoolAttr::get(context: ctx, value: val->getValue().getLimitedValue(Limit: 1) ^ negated); |
| 175 | } |
| 176 | |
| 177 | FailureOr<BoolAttr> |
| 178 | LoopMetadataConversion::lookupIntNodeAsBoolAttr(StringRef name) { |
| 179 | const llvm::MDNode *property = lookupAndEraseProperty(name); |
| 180 | if (!property) |
| 181 | return BoolAttr(nullptr); |
| 182 | |
| 183 | auto emitNodeWarning = [&]() { |
| 184 | return emitWarning(loc) |
| 185 | << "expected metadata node " << name << " to hold an integer value" ; |
| 186 | }; |
| 187 | |
| 188 | if (property->getNumOperands() != 2) |
| 189 | return emitNodeWarning(); |
| 190 | llvm::ConstantInt *val = |
| 191 | llvm::mdconst::dyn_extract<llvm::ConstantInt>(MD: property->getOperand(I: 1)); |
| 192 | if (!val || val->getBitWidth() != 32) |
| 193 | return emitNodeWarning(); |
| 194 | |
| 195 | return BoolAttr::get(context: ctx, value: val->getValue().getLimitedValue(Limit: 1)); |
| 196 | } |
| 197 | |
| 198 | /// Like lookupIntNode, but yields the value itself, with 0 for a missing node. |
| 199 | FailureOr<uint32_t> LoopMetadataConversion::lookupUInt32Node(StringRef name) { |
| 200 | FailureOr<IntegerAttr> attr = lookupIntNode(name); |
| 201 | if (failed(Result: attr)) |
| 202 | return failure(); |
| 203 | return *attr ? uint32_t(attr->getValue().getZExtValue()) : uint32_t(0); |
| 204 | } |
| 205 | |
| 206 | FailureOr<IntegerAttr> LoopMetadataConversion::lookupIntNode(StringRef name) { |
| 207 | const llvm::MDNode *property = lookupAndEraseProperty(name); |
| 208 | if (!property) |
| 209 | return IntegerAttr(nullptr); |
| 210 | |
| 211 | auto emitNodeWarning = [&]() { |
| 212 | return emitWarning(loc) |
| 213 | << "expected metadata node " << name << " to hold an i32 value" ; |
| 214 | }; |
| 215 | |
| 216 | if (property->getNumOperands() != 2) |
| 217 | return emitNodeWarning(); |
| 218 | |
| 219 | llvm::ConstantInt *val = |
| 220 | llvm::mdconst::dyn_extract<llvm::ConstantInt>(MD: property->getOperand(I: 1)); |
| 221 | if (!val || val->getBitWidth() != 32) |
| 222 | return emitNodeWarning(); |
| 223 | |
| 224 | return IntegerAttr::get(type: IntegerType::get(context: ctx, width: 32), |
| 225 | value: val->getValue().getLimitedValue()); |
| 226 | } |
| 227 | |
| 228 | FailureOr<SmallVector<llvm::MDNode *>> |
| 229 | LoopMetadataConversion::lookupMDNodes(StringRef name) { |
| 230 | const llvm::MDNode *property = lookupAndEraseProperty(name); |
| 231 | SmallVector<llvm::MDNode *> res; |
| 232 | if (!property) |
| 233 | return res; |
| 234 | |
| 235 | auto emitNodeWarning = [&]() { |
| 236 | return emitWarning(loc) << "expected metadata node " << name |
| 237 | << " to hold one or multiple MDNodes" ; |
| 238 | }; |
| 239 | |
| 240 | if (property->getNumOperands() < 2) |
| 241 | return emitNodeWarning(); |
| 242 | |
| 243 | for (unsigned i = 1, e = property->getNumOperands(); i < e; ++i) { |
| 244 | auto *node = dyn_cast<llvm::MDNode>(Val: property->getOperand(I: i)); |
| 245 | if (!node) |
| 246 | return emitNodeWarning(); |
| 247 | res.push_back(Elt: node); |
| 248 | } |
| 249 | |
| 250 | return res; |
| 251 | } |
| 252 | |
| 253 | FailureOr<LoopAnnotationAttr> |
| 254 | LoopMetadataConversion::lookupFollowupNode(StringRef name) { |
| 255 | const llvm::MDNode *followup = lookupAndEraseProperty(name); |
| 256 | if (!followup) |
| 257 | return LoopAnnotationAttr(nullptr); |
| 258 | |
| 259 | LoopAnnotationAttr attr = convertFollowup(followup); |
| 260 | if (!attr) |
| 261 | return failure(); |
| 262 | return attr; |
| 263 | } |
| 264 | |
| 265 | static bool isEmptyOrNull(const Attribute attr) { return !attr; } |
| 266 | |
| 267 | static bool isEmptyOrNull(uint32_t value) { return !value; } |
| 268 | |
| 269 | template <typename T> |
| 270 | static bool isEmptyOrNull(const SmallVectorImpl<T> &vec) { |
| 271 | return vec.empty(); |
| 272 | } |
| 273 | |
| 274 | /// Helper function that returns true if any of the argument conversions failed |
| 275 | /// or if all of them are null or empty. |
| 276 | template <typename... P> |
| 277 | static bool anyFailedOrAllEmpty(const P &...args) { |
| 278 | bool anyFailed = (failed(args) || ...); |
| 279 | if (anyFailed) |
| 280 | return true; |
| 281 | |
| 282 | return (isEmptyOrNull(*args) && ...); |
| 283 | } |
| 284 | |
| 285 | /// Helper function that only creates and attribute of type T if all argument |
| 286 | /// conversion were successfull and at least one of them holds a non-null value. |
| 287 | template <typename T, typename... P> |
| 288 | static T createIfNonNull(MLIRContext *ctx, const P &...args) { |
| 289 | if (anyFailedOrAllEmpty(args...)) |
| 290 | return {}; |
| 291 | |
| 292 | return T::get(ctx, *args...); |
| 293 | } |
| 294 | |
| 295 | FailureOr<LoopVectorizeAttr> LoopMetadataConversion::convertVectorizeAttr() { |
| 296 | FailureOr<BoolAttr> enable = lookupBooleanUnitNode( |
| 297 | enableName: "llvm.loop.vectorize.enable" , disableName: "llvm.loop.vectorize.disable" , |
| 298 | /*negated=*/true); |
| 299 | FailureOr<BoolAttr> predicateEnable = |
| 300 | lookupBooleanUnitNode(enableName: "llvm.loop.vectorize.predicate.enable" , |
| 301 | disableName: "llvm.loop.vectorize.predicate.disable" ); |
| 302 | FailureOr<BoolAttr> scalableEnable = |
| 303 | lookupBooleanUnitNode(enableName: "llvm.loop.vectorize.scalable.enable" , |
| 304 | disableName: "llvm.loop.vectorize.scalable.disable" ); |
| 305 | FailureOr<IntegerAttr> width = lookupIntNode(name: "llvm.loop.vectorize.width" ); |
| 306 | FailureOr<LoopAnnotationAttr> followupVec = |
| 307 | lookupFollowupNode(name: "llvm.loop.vectorize.followup_vectorized" ); |
| 308 | FailureOr<LoopAnnotationAttr> followupEpi = |
| 309 | lookupFollowupNode(name: "llvm.loop.vectorize.followup_epilogue" ); |
| 310 | FailureOr<LoopAnnotationAttr> followupAll = |
| 311 | lookupFollowupNode(name: "llvm.loop.vectorize.followup_all" ); |
| 312 | |
| 313 | return createIfNonNull<LoopVectorizeAttr>(ctx, args: enable, args: predicateEnable, |
| 314 | args: scalableEnable, args: width, args: followupVec, |
| 315 | args: followupEpi, args: followupAll); |
| 316 | } |
| 317 | |
| 318 | FailureOr<LoopInterleaveAttr> LoopMetadataConversion::convertInterleaveAttr() { |
| 319 | FailureOr<IntegerAttr> count = lookupIntNode(name: "llvm.loop.interleave.count" ); |
| 320 | return createIfNonNull<LoopInterleaveAttr>(ctx, args: count); |
| 321 | } |
| 322 | |
| 323 | FailureOr<LoopUnrollAttr> LoopMetadataConversion::convertUnrollAttr() { |
| 324 | FailureOr<BoolAttr> disable = lookupBooleanUnitNode( |
| 325 | enableName: "llvm.loop.unroll.enable" , disableName: "llvm.loop.unroll.disable" , /*negated=*/true); |
| 326 | FailureOr<IntegerAttr> count = lookupIntNode(name: "llvm.loop.unroll.count" ); |
| 327 | FailureOr<BoolAttr> runtimeDisable = |
| 328 | lookupUnitNode(name: "llvm.loop.unroll.runtime.disable" ); |
| 329 | FailureOr<BoolAttr> full = lookupUnitNode(name: "llvm.loop.unroll.full" ); |
| 330 | FailureOr<LoopAnnotationAttr> followupUnrolled = |
| 331 | lookupFollowupNode(name: "llvm.loop.unroll.followup_unrolled" ); |
| 332 | FailureOr<LoopAnnotationAttr> followupRemainder = |
| 333 | lookupFollowupNode(name: "llvm.loop.unroll.followup_remainder" ); |
| 334 | FailureOr<LoopAnnotationAttr> followupAll = |
| 335 | lookupFollowupNode(name: "llvm.loop.unroll.followup_all" ); |
| 336 | |
| 337 | return createIfNonNull<LoopUnrollAttr>(ctx, args: disable, args: count, args: runtimeDisable, |
| 338 | args: full, args: followupUnrolled, |
| 339 | args: followupRemainder, args: followupAll); |
| 340 | } |
| 341 | |
| 342 | FailureOr<LoopUnrollAndJamAttr> |
| 343 | LoopMetadataConversion::convertUnrollAndJamAttr() { |
| 344 | FailureOr<BoolAttr> disable = lookupBooleanUnitNode( |
| 345 | enableName: "llvm.loop.unroll_and_jam.enable" , disableName: "llvm.loop.unroll_and_jam.disable" , |
| 346 | /*negated=*/true); |
| 347 | FailureOr<IntegerAttr> count = |
| 348 | lookupIntNode(name: "llvm.loop.unroll_and_jam.count" ); |
| 349 | FailureOr<LoopAnnotationAttr> followupOuter = |
| 350 | lookupFollowupNode(name: "llvm.loop.unroll_and_jam.followup_outer" ); |
| 351 | FailureOr<LoopAnnotationAttr> followupInner = |
| 352 | lookupFollowupNode(name: "llvm.loop.unroll_and_jam.followup_inner" ); |
| 353 | FailureOr<LoopAnnotationAttr> followupRemainderOuter = |
| 354 | lookupFollowupNode(name: "llvm.loop.unroll_and_jam.followup_remainder_outer" ); |
| 355 | FailureOr<LoopAnnotationAttr> followupRemainderInner = |
| 356 | lookupFollowupNode(name: "llvm.loop.unroll_and_jam.followup_remainder_inner" ); |
| 357 | FailureOr<LoopAnnotationAttr> followupAll = |
| 358 | lookupFollowupNode(name: "llvm.loop.unroll_and_jam.followup_all" ); |
| 359 | return createIfNonNull<LoopUnrollAndJamAttr>( |
| 360 | ctx, args: disable, args: count, args: followupOuter, args: followupInner, args: followupRemainderOuter, |
| 361 | args: followupRemainderInner, args: followupAll); |
| 362 | } |
| 363 | |
| 364 | FailureOr<LoopLICMAttr> LoopMetadataConversion::convertLICMAttr() { |
| 365 | FailureOr<BoolAttr> disable = lookupUnitNode(name: "llvm.licm.disable" ); |
| 366 | FailureOr<BoolAttr> versioningDisable = |
| 367 | lookupUnitNode(name: "llvm.loop.licm_versioning.disable" ); |
| 368 | return createIfNonNull<LoopLICMAttr>(ctx, args: disable, args: versioningDisable); |
| 369 | } |
| 370 | |
| 371 | FailureOr<LoopDistributeAttr> LoopMetadataConversion::convertDistributeAttr() { |
| 372 | FailureOr<BoolAttr> disable = lookupBooleanUnitNode( |
| 373 | enableName: "llvm.loop.distribute.enable" , disableName: "llvm.loop.distribute.disable" , |
| 374 | /*negated=*/true); |
| 375 | FailureOr<LoopAnnotationAttr> followupCoincident = |
| 376 | lookupFollowupNode(name: "llvm.loop.distribute.followup_coincident" ); |
| 377 | FailureOr<LoopAnnotationAttr> followupSequential = |
| 378 | lookupFollowupNode(name: "llvm.loop.distribute.followup_sequential" ); |
| 379 | FailureOr<LoopAnnotationAttr> followupFallback = |
| 380 | lookupFollowupNode(name: "llvm.loop.distribute.followup_fallback" ); |
| 381 | FailureOr<LoopAnnotationAttr> followupAll = |
| 382 | lookupFollowupNode(name: "llvm.loop.distribute.followup_all" ); |
| 383 | return createIfNonNull<LoopDistributeAttr>(ctx, args: disable, args: followupCoincident, |
| 384 | args: followupSequential, |
| 385 | args: followupFallback, args: followupAll); |
| 386 | } |
| 387 | |
| 388 | FailureOr<LoopPipelineAttr> LoopMetadataConversion::convertPipelineAttr() { |
| 389 | FailureOr<BoolAttr> disable = lookupBoolNode(name: "llvm.loop.pipeline.disable" ); |
| 390 | FailureOr<IntegerAttr> initiationinterval = |
| 391 | lookupIntNode(name: "llvm.loop.pipeline.initiationinterval" ); |
| 392 | return createIfNonNull<LoopPipelineAttr>(ctx, args: disable, args: initiationinterval); |
| 393 | } |
| 394 | |
| 395 | FailureOr<LoopPeeledAttr> LoopMetadataConversion::convertPeeledAttr() { |
| 396 | FailureOr<IntegerAttr> count = lookupIntNode(name: "llvm.loop.peeled.count" ); |
| 397 | return createIfNonNull<LoopPeeledAttr>(ctx, args: count); |
| 398 | } |
| 399 | |
| 400 | FailureOr<LoopUnswitchAttr> LoopMetadataConversion::convertUnswitchAttr() { |
| 401 | FailureOr<BoolAttr> partialDisable = |
| 402 | lookupUnitNode(name: "llvm.loop.unswitch.partial.disable" ); |
| 403 | return createIfNonNull<LoopUnswitchAttr>(ctx, args: partialDisable); |
| 404 | } |
| 405 | |
| 406 | FailureOr<SmallVector<AccessGroupAttr>> |
| 407 | LoopMetadataConversion::convertParallelAccesses() { |
| 408 | FailureOr<SmallVector<llvm::MDNode *>> nodes = |
| 409 | lookupMDNodes(name: "llvm.loop.parallel_accesses" ); |
| 410 | if (failed(Result: nodes)) |
| 411 | return failure(); |
| 412 | SmallVector<AccessGroupAttr> refs; |
| 413 | for (llvm::MDNode *node : *nodes) { |
| 414 | FailureOr<SmallVector<AccessGroupAttr>> accessGroups = |
| 415 | loopAnnotationImporter.lookupAccessGroupAttrs(node); |
| 416 | if (failed(Result: accessGroups)) { |
| 417 | emitWarning(loc) << "could not lookup access group" ; |
| 418 | continue; |
| 419 | } |
| 420 | llvm::append_range(C&: refs, R&: *accessGroups); |
| 421 | } |
| 422 | return refs; |
| 423 | } |
| 424 | |
| 425 | FusedLoc LoopMetadataConversion::convertStartLoc() { |
| 426 | if (locations.empty()) |
| 427 | return {}; |
| 428 | return dyn_cast<FusedLoc>( |
| 429 | Val: loopAnnotationImporter.moduleImport.translateLoc(loc: locations[0])); |
| 430 | } |
| 431 | |
| 432 | FailureOr<FusedLoc> LoopMetadataConversion::convertEndLoc() { |
| 433 | if (locations.size() < 2) |
| 434 | return FusedLoc(); |
| 435 | if (locations.size() > 2) |
| 436 | return emitError(loc) |
| 437 | << "expected loop metadata to have at most two DILocations" ; |
| 438 | return dyn_cast<FusedLoc>( |
| 439 | Val: loopAnnotationImporter.moduleImport.translateLoc(loc: locations[1])); |
| 440 | } |
| 441 | |
| 442 | LoopAnnotationAttr |
| 443 | LoopMetadataConversion::convert(const llvm::MDNode *node, Location loc, |
| 444 | LoopAnnotationImporter &importer) { |
| 445 | if (node->getNumOperands() == 0 || |
| 446 | dyn_cast<llvm::MDNode>(Val: node->getOperand(I: 0)) != node) { |
| 447 | emitWarning(loc) << "invalid loop node" ; |
| 448 | return {}; |
| 449 | } |
| 450 | |
| 451 | llvm::SmallPtrSet<const llvm::MDNode *, 4> activeNodes; |
| 452 | DistinctAttr id = DistinctAttr::create(referencedAttr: UnitAttr::get(context: loc->getContext())); |
| 453 | return LoopMetadataConversion(node, loc, id, importer, activeNodes) |
| 454 | .convertProperties(); |
| 455 | } |
| 456 | |
| 457 | LoopAnnotationAttr |
| 458 | LoopMetadataConversion::convertFollowup(const llvm::MDNode *followup) { |
| 459 | if (followup->getNumOperands() == 0 || |
| 460 | !isa<llvm::MDString>(Val: followup->getOperand(I: 0))) { |
| 461 | emitWarning(loc) << "invalid loop followup node" ; |
| 462 | return {}; |
| 463 | } |
| 464 | |
| 465 | // An explicit empty follow-up is different from an absent follow-up. |
| 466 | if (followup->getNumOperands() == 1) |
| 467 | return LoopAnnotationAttr::get(context: ctx, id: {}, disableNonforced: {}, vectorize: {}, interleave: {}, unroll: {}, unrollAndJam: {}, licm: {}, distribute: {}, pipeline: {}, peeled: {}, |
| 468 | unswitch: {}, mustProgress: {}, isVectorized: {}, align: {}, startLoc: {}, endLoc: {}, parallelAccesses: {}); |
| 469 | |
| 470 | return LoopMetadataConversion(followup, loc, /*id=*/{}, |
| 471 | loopAnnotationImporter, activeNodes) |
| 472 | .convertProperties(); |
| 473 | } |
| 474 | |
| 475 | LoopAnnotationAttr LoopMetadataConversion::convertProperties() { |
| 476 | if (!activeNodes.insert(Ptr: node).second) { |
| 477 | emitWarning(loc) << "cannot import cyclic loop annotation" ; |
| 478 | return {}; |
| 479 | } |
| 480 | llvm::scope_exit guard([&] { activeNodes.erase(Ptr: node); }); |
| 481 | |
| 482 | if (failed(Result: initConversionState())) |
| 483 | return {}; |
| 484 | |
| 485 | FailureOr<BoolAttr> disableNonForced = |
| 486 | lookupUnitNode(name: "llvm.loop.disable_nonforced" ); |
| 487 | FailureOr<LoopVectorizeAttr> vecAttr = convertVectorizeAttr(); |
| 488 | FailureOr<LoopInterleaveAttr> interleaveAttr = convertInterleaveAttr(); |
| 489 | FailureOr<LoopUnrollAttr> unrollAttr = convertUnrollAttr(); |
| 490 | FailureOr<LoopUnrollAndJamAttr> unrollAndJamAttr = convertUnrollAndJamAttr(); |
| 491 | FailureOr<LoopLICMAttr> licmAttr = convertLICMAttr(); |
| 492 | FailureOr<LoopDistributeAttr> distributeAttr = convertDistributeAttr(); |
| 493 | FailureOr<LoopPipelineAttr> pipelineAttr = convertPipelineAttr(); |
| 494 | FailureOr<LoopPeeledAttr> peeledAttr = convertPeeledAttr(); |
| 495 | FailureOr<LoopUnswitchAttr> unswitchAttr = convertUnswitchAttr(); |
| 496 | FailureOr<BoolAttr> mustProgress = lookupUnitNode(name: "llvm.loop.mustprogress" ); |
| 497 | FailureOr<BoolAttr> isVectorized = |
| 498 | lookupIntNodeAsBoolAttr(name: "llvm.loop.isvectorized" ); |
| 499 | FailureOr<uint32_t> align = lookupUInt32Node(name: "llvm.loop.align" ); |
| 500 | FailureOr<SmallVector<AccessGroupAttr>> parallelAccesses = |
| 501 | convertParallelAccesses(); |
| 502 | |
| 503 | // Drop the metadata if there are parts that cannot be imported. |
| 504 | if (!propertyMap.empty()) { |
| 505 | for (auto name : propertyMap.keys()) |
| 506 | emitWarning(loc) << "unknown loop annotation " << name; |
| 507 | return {}; |
| 508 | } |
| 509 | |
| 510 | FailureOr<FusedLoc> startLoc = convertStartLoc(); |
| 511 | FailureOr<FusedLoc> endLoc = convertEndLoc(); |
| 512 | |
| 513 | // LoopID doesn't participate in 'failure', so check everything except it. |
| 514 | if (anyFailedOrAllEmpty(args: disableNonForced, args: vecAttr, args: interleaveAttr, args: unrollAttr, |
| 515 | args: unrollAndJamAttr, args: licmAttr, args: distributeAttr, |
| 516 | args: pipelineAttr, args: peeledAttr, args: unswitchAttr, args: mustProgress, |
| 517 | args: isVectorized, args: align, args: startLoc, args: endLoc, |
| 518 | args: parallelAccesses)) |
| 519 | return {}; |
| 520 | |
| 521 | return LoopAnnotationAttr::get( |
| 522 | context: ctx, id, disableNonforced: *disableNonForced, vectorize: *vecAttr, interleave: *interleaveAttr, unroll: *unrollAttr, |
| 523 | unrollAndJam: *unrollAndJamAttr, licm: *licmAttr, distribute: *distributeAttr, pipeline: *pipelineAttr, peeled: *peeledAttr, |
| 524 | unswitch: *unswitchAttr, mustProgress: *mustProgress, isVectorized: *isVectorized, align: *align, startLoc: *startLoc, endLoc: *endLoc, |
| 525 | parallelAccesses: *parallelAccesses); |
| 526 | } |
| 527 | |
| 528 | LoopAnnotationAttr |
| 529 | LoopAnnotationImporter::translateLoopAnnotation(const llvm::MDNode *node, |
| 530 | Location loc) { |
| 531 | if (!node) |
| 532 | return {}; |
| 533 | |
| 534 | // Note: This check is necessary to distinguish between failed translations |
| 535 | // and not yet attempted translations. |
| 536 | auto it = loopMetadataMapping.find(Val: node); |
| 537 | if (it != loopMetadataMapping.end()) |
| 538 | return it->getSecond(); |
| 539 | |
| 540 | LoopAnnotationAttr attr = LoopMetadataConversion::convert(node, loc, importer&: *this); |
| 541 | |
| 542 | mapLoopMetadata(metadata: node, attr); |
| 543 | return attr; |
| 544 | } |
| 545 | |
| 546 | LogicalResult |
| 547 | LoopAnnotationImporter::translateAccessGroup(const llvm::MDNode *node, |
| 548 | Location loc) { |
| 549 | SmallVector<const llvm::MDNode *> accessGroups; |
| 550 | if (!node->getNumOperands()) |
| 551 | accessGroups.push_back(Elt: node); |
| 552 | for (const llvm::MDOperand &operand : node->operands()) { |
| 553 | auto *childNode = dyn_cast<llvm::MDNode>(Val: operand); |
| 554 | if (!childNode) |
| 555 | return failure(); |
| 556 | accessGroups.push_back(Elt: cast<llvm::MDNode>(Val: operand.get())); |
| 557 | } |
| 558 | |
| 559 | // Convert all entries of the access group list to access group operations. |
| 560 | for (const llvm::MDNode *accessGroup : accessGroups) { |
| 561 | if (accessGroupMapping.count(Val: accessGroup)) |
| 562 | continue; |
| 563 | // Verify the access group node is distinct and empty. |
| 564 | if (accessGroup->getNumOperands() != 0 || !accessGroup->isDistinct()) |
| 565 | return emitWarning(loc) |
| 566 | << "expected an access group node to be empty and distinct" ; |
| 567 | |
| 568 | // Add a mapping from the access group node to the newly created attribute. |
| 569 | accessGroupMapping[accessGroup] = builder.getAttr<AccessGroupAttr>(); |
| 570 | } |
| 571 | return success(); |
| 572 | } |
| 573 | |
| 574 | FailureOr<SmallVector<AccessGroupAttr>> |
| 575 | LoopAnnotationImporter::lookupAccessGroupAttrs(const llvm::MDNode *node) const { |
| 576 | // An access group node is either a single access group or an access group |
| 577 | // list. |
| 578 | SmallVector<AccessGroupAttr> accessGroups; |
| 579 | if (!node->getNumOperands()) |
| 580 | accessGroups.push_back(Elt: accessGroupMapping.lookup(Val: node)); |
| 581 | for (const llvm::MDOperand &operand : node->operands()) { |
| 582 | auto *node = cast<llvm::MDNode>(Val: operand.get()); |
| 583 | accessGroups.push_back(Elt: accessGroupMapping.lookup(Val: node)); |
| 584 | } |
| 585 | // Exit if one of the access group node lookups failed. |
| 586 | if (llvm::is_contained(Range&: accessGroups, Element: nullptr)) |
| 587 | return failure(); |
| 588 | return accessGroups; |
| 589 | } |
| 590 | |