| 1 | //===- LoopAnnotationTranslation.cpp - Loop annotation export -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "LoopAnnotationTranslation.h" |
| 10 | #include "llvm/IR/DebugInfoMetadata.h" |
| 11 | |
| 12 | using namespace mlir; |
| 13 | using namespace mlir::LLVM; |
| 14 | using namespace mlir::LLVM::detail; |
| 15 | |
| 16 | namespace { |
| 17 | /// Helper class that keeps the state of one attribute to metadata conversion. |
| 18 | struct LoopAnnotationConversion { |
| 19 | LoopAnnotationConversion(LoopAnnotationAttr attr, Operation *op, |
| 20 | LoopAnnotationTranslation &loopAnnotationTranslation, |
| 21 | llvm::LLVMContext &ctx) |
| 22 | : attr(attr), op(op), |
| 23 | loopAnnotationTranslation(loopAnnotationTranslation), ctx(ctx) {} |
| 24 | |
| 25 | /// Converts this struct's loop annotation into a corresponding LLVMIR |
| 26 | /// metadata representation. |
| 27 | llvm::MDNode *convert(); |
| 28 | |
| 29 | /// Conversion functions for different payload attribute kinds. |
| 30 | void addUnitNode(StringRef name); |
| 31 | void addUnitNode(StringRef name, BoolAttr attr); |
| 32 | void addI32NodeWithVal(StringRef name, uint32_t val); |
| 33 | void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false); |
| 34 | void convertBooleanUnitNode(StringRef enableName, StringRef disableName, |
| 35 | BoolAttr attr, bool negated = false); |
| 36 | void convertI32Node(StringRef name, IntegerAttr attr); |
| 37 | void convertFollowupNode(StringRef name, LoopAnnotationAttr attr); |
| 38 | void convertLocation(FusedLoc attr); |
| 39 | |
| 40 | /// Conversion functions for each for each loop annotation sub-attribute. |
| 41 | void convertLoopOptions(LoopVectorizeAttr options); |
| 42 | void convertLoopOptions(LoopInterleaveAttr options); |
| 43 | void convertLoopOptions(LoopUnrollAttr options); |
| 44 | void convertLoopOptions(LoopUnrollAndJamAttr options); |
| 45 | void convertLoopOptions(LoopLICMAttr options); |
| 46 | void convertLoopOptions(LoopDistributeAttr options); |
| 47 | void convertLoopOptions(LoopPipelineAttr options); |
| 48 | void convertLoopOptions(LoopPeeledAttr options); |
| 49 | void convertLoopOptions(LoopUnswitchAttr options); |
| 50 | |
| 51 | LoopAnnotationAttr attr; |
| 52 | Operation *op; |
| 53 | LoopAnnotationTranslation &loopAnnotationTranslation; |
| 54 | llvm::LLVMContext &ctx; |
| 55 | llvm::SmallVector<llvm::Metadata *> metadataNodes; |
| 56 | }; |
| 57 | } // namespace |
| 58 | |
| 59 | void LoopAnnotationConversion::addUnitNode(StringRef name) { |
| 60 | metadataNodes.push_back( |
| 61 | Elt: llvm::MDNode::get(Context&: ctx, MDs: {llvm::MDString::get(Context&: ctx, Str: name)})); |
| 62 | } |
| 63 | |
| 64 | void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) { |
| 65 | if (attr && attr.getValue()) |
| 66 | addUnitNode(name); |
| 67 | } |
| 68 | |
| 69 | void LoopAnnotationConversion::addI32NodeWithVal(StringRef name, uint32_t val) { |
| 70 | llvm::Constant *cstValue = llvm::ConstantInt::get( |
| 71 | Ty: llvm::IntegerType::get(C&: ctx, /*NumBits=*/32), V: val, /*isSigned=*/IsSigned: false); |
| 72 | metadataNodes.push_back( |
| 73 | Elt: llvm::MDNode::get(Context&: ctx, MDs: {llvm::MDString::get(Context&: ctx, Str: name), |
| 74 | llvm::ConstantAsMetadata::get(C: cstValue)})); |
| 75 | } |
| 76 | |
| 77 | void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr, |
| 78 | bool negated) { |
| 79 | if (!attr) |
| 80 | return; |
| 81 | bool val = negated ^ attr.getValue(); |
| 82 | llvm::Constant *cstValue = llvm::ConstantInt::getBool(Context&: ctx, V: val); |
| 83 | metadataNodes.push_back( |
| 84 | Elt: llvm::MDNode::get(Context&: ctx, MDs: {llvm::MDString::get(Context&: ctx, Str: name), |
| 85 | llvm::ConstantAsMetadata::get(C: cstValue)})); |
| 86 | } |
| 87 | |
| 88 | /// Emits the single-operand node of an enable/disable pair. As in |
| 89 | /// convertBoolNode, the attribute is tri-state: a null \p attr emits no node |
| 90 | /// at all, otherwise \p negated ^ the attribute value is the enable bit and |
| 91 | /// selects which of the two names is emitted. |
| 92 | void LoopAnnotationConversion::convertBooleanUnitNode(StringRef enableName, |
| 93 | StringRef disableName, |
| 94 | BoolAttr attr, |
| 95 | bool negated) { |
| 96 | if (!attr) |
| 97 | return; |
| 98 | addUnitNode(name: (negated ^ attr.getValue()) ? enableName : disableName); |
| 99 | } |
| 100 | |
| 101 | void LoopAnnotationConversion::convertI32Node(StringRef name, |
| 102 | IntegerAttr attr) { |
| 103 | if (!attr) |
| 104 | return; |
| 105 | addI32NodeWithVal(name, val: attr.getInt()); |
| 106 | } |
| 107 | |
| 108 | void LoopAnnotationConversion::convertFollowupNode(StringRef name, |
| 109 | LoopAnnotationAttr attr) { |
| 110 | if (!attr) |
| 111 | return; |
| 112 | |
| 113 | llvm::MDNode *loopID = |
| 114 | loopAnnotationTranslation.translateLoopAnnotation(attr, op); |
| 115 | |
| 116 | // Follow-ups contain properties directly, without the LoopID self-reference. |
| 117 | SmallVector<llvm::Metadata *> operands; |
| 118 | operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: name)); |
| 119 | for (const llvm::MDOperand &operand : llvm::drop_begin(RangeOrContainer: loopID->operands())) |
| 120 | operands.push_back(Elt: operand.get()); |
| 121 | metadataNodes.push_back(Elt: llvm::MDNode::get(Context&: ctx, MDs: operands)); |
| 122 | } |
| 123 | |
| 124 | void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) { |
| 125 | convertBooleanUnitNode(enableName: "llvm.loop.vectorize.enable" , |
| 126 | disableName: "llvm.loop.vectorize.disable" , attr: options.getDisable(), |
| 127 | /*negated=*/true); |
| 128 | convertBooleanUnitNode(enableName: "llvm.loop.vectorize.predicate.enable" , |
| 129 | disableName: "llvm.loop.vectorize.predicate.disable" , |
| 130 | attr: options.getPredicateEnable()); |
| 131 | convertBooleanUnitNode(enableName: "llvm.loop.vectorize.scalable.enable" , |
| 132 | disableName: "llvm.loop.vectorize.scalable.disable" , |
| 133 | attr: options.getScalableEnable()); |
| 134 | convertI32Node(name: "llvm.loop.vectorize.width" , attr: options.getWidth()); |
| 135 | convertFollowupNode(name: "llvm.loop.vectorize.followup_vectorized" , |
| 136 | attr: options.getFollowupVectorized()); |
| 137 | convertFollowupNode(name: "llvm.loop.vectorize.followup_epilogue" , |
| 138 | attr: options.getFollowupEpilogue()); |
| 139 | convertFollowupNode(name: "llvm.loop.vectorize.followup_all" , |
| 140 | attr: options.getFollowupAll()); |
| 141 | } |
| 142 | |
| 143 | void LoopAnnotationConversion::convertLoopOptions(LoopInterleaveAttr options) { |
| 144 | convertI32Node(name: "llvm.loop.interleave.count" , attr: options.getCount()); |
| 145 | } |
| 146 | |
| 147 | void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) { |
| 148 | if (auto disable = options.getDisable()) |
| 149 | addUnitNode(name: disable.getValue() ? "llvm.loop.unroll.disable" |
| 150 | : "llvm.loop.unroll.enable" ); |
| 151 | convertI32Node(name: "llvm.loop.unroll.count" , attr: options.getCount()); |
| 152 | convertBoolNode(name: "llvm.loop.unroll.runtime.disable" , |
| 153 | attr: options.getRuntimeDisable()); |
| 154 | addUnitNode(name: "llvm.loop.unroll.full" , attr: options.getFull()); |
| 155 | convertFollowupNode(name: "llvm.loop.unroll.followup_unrolled" , |
| 156 | attr: options.getFollowupUnrolled()); |
| 157 | convertFollowupNode(name: "llvm.loop.unroll.followup_remainder" , |
| 158 | attr: options.getFollowupRemainder()); |
| 159 | convertFollowupNode(name: "llvm.loop.unroll.followup_all" , |
| 160 | attr: options.getFollowupAll()); |
| 161 | } |
| 162 | |
| 163 | void LoopAnnotationConversion::convertLoopOptions( |
| 164 | LoopUnrollAndJamAttr options) { |
| 165 | if (auto disable = options.getDisable()) |
| 166 | addUnitNode(name: disable.getValue() ? "llvm.loop.unroll_and_jam.disable" |
| 167 | : "llvm.loop.unroll_and_jam.enable" ); |
| 168 | convertI32Node(name: "llvm.loop.unroll_and_jam.count" , attr: options.getCount()); |
| 169 | convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_outer" , |
| 170 | attr: options.getFollowupOuter()); |
| 171 | convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_inner" , |
| 172 | attr: options.getFollowupInner()); |
| 173 | convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_remainder_outer" , |
| 174 | attr: options.getFollowupRemainderOuter()); |
| 175 | convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_remainder_inner" , |
| 176 | attr: options.getFollowupRemainderInner()); |
| 177 | convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_all" , |
| 178 | attr: options.getFollowupAll()); |
| 179 | } |
| 180 | |
| 181 | void LoopAnnotationConversion::convertLoopOptions(LoopLICMAttr options) { |
| 182 | addUnitNode(name: "llvm.licm.disable" , attr: options.getDisable()); |
| 183 | addUnitNode(name: "llvm.loop.licm_versioning.disable" , |
| 184 | attr: options.getVersioningDisable()); |
| 185 | } |
| 186 | |
| 187 | void LoopAnnotationConversion::convertLoopOptions(LoopDistributeAttr options) { |
| 188 | convertBooleanUnitNode(enableName: "llvm.loop.distribute.enable" , |
| 189 | disableName: "llvm.loop.distribute.disable" , attr: options.getDisable(), |
| 190 | /*negated=*/true); |
| 191 | convertFollowupNode(name: "llvm.loop.distribute.followup_coincident" , |
| 192 | attr: options.getFollowupCoincident()); |
| 193 | convertFollowupNode(name: "llvm.loop.distribute.followup_sequential" , |
| 194 | attr: options.getFollowupSequential()); |
| 195 | convertFollowupNode(name: "llvm.loop.distribute.followup_fallback" , |
| 196 | attr: options.getFollowupFallback()); |
| 197 | convertFollowupNode(name: "llvm.loop.distribute.followup_all" , |
| 198 | attr: options.getFollowupAll()); |
| 199 | } |
| 200 | |
| 201 | void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) { |
| 202 | convertBoolNode(name: "llvm.loop.pipeline.disable" , attr: options.getDisable()); |
| 203 | convertI32Node(name: "llvm.loop.pipeline.initiationinterval" , |
| 204 | attr: options.getInitiationinterval()); |
| 205 | } |
| 206 | |
| 207 | void LoopAnnotationConversion::convertLoopOptions(LoopPeeledAttr options) { |
| 208 | convertI32Node(name: "llvm.loop.peeled.count" , attr: options.getCount()); |
| 209 | } |
| 210 | |
| 211 | void LoopAnnotationConversion::convertLoopOptions(LoopUnswitchAttr options) { |
| 212 | addUnitNode(name: "llvm.loop.unswitch.partial.disable" , |
| 213 | attr: options.getPartialDisable()); |
| 214 | } |
| 215 | |
| 216 | void LoopAnnotationConversion::convertLocation(FusedLoc location) { |
| 217 | auto localScopeAttr = |
| 218 | dyn_cast_or_null<DILocalScopeAttr>(Val: location.getMetadata()); |
| 219 | if (!localScopeAttr) |
| 220 | return; |
| 221 | auto *localScope = dyn_cast<llvm::DILocalScope>( |
| 222 | Val: loopAnnotationTranslation.moduleTranslation.translateDebugInfo( |
| 223 | attr: localScopeAttr)); |
| 224 | if (!localScope) |
| 225 | return; |
| 226 | llvm::Metadata *loc = |
| 227 | loopAnnotationTranslation.moduleTranslation.translateLoc(loc: location, |
| 228 | scope: localScope); |
| 229 | metadataNodes.push_back(Elt: loc); |
| 230 | } |
| 231 | |
| 232 | llvm::MDNode *LoopAnnotationConversion::convert() { |
| 233 | // Reserve operand 0 for loop id self reference. |
| 234 | auto dummy = llvm::MDNode::getTemporary(Context&: ctx, MDs: {}); |
| 235 | metadataNodes.push_back(Elt: dummy.get()); |
| 236 | |
| 237 | if (FusedLoc startLoc = attr.getStartLoc()) |
| 238 | convertLocation(location: startLoc); |
| 239 | |
| 240 | if (FusedLoc endLoc = attr.getEndLoc()) |
| 241 | convertLocation(location: endLoc); |
| 242 | |
| 243 | addUnitNode(name: "llvm.loop.disable_nonforced" , attr: attr.getDisableNonforced()); |
| 244 | addUnitNode(name: "llvm.loop.mustprogress" , attr: attr.getMustProgress()); |
| 245 | // "isvectorized" is encoded as an i32 value. |
| 246 | if (BoolAttr isVectorized = attr.getIsVectorized()) |
| 247 | addI32NodeWithVal(name: "llvm.loop.isvectorized" , val: isVectorized.getValue()); |
| 248 | if (uint32_t align = attr.getAlign()) |
| 249 | addI32NodeWithVal(name: "llvm.loop.align" , val: align); |
| 250 | |
| 251 | if (auto options = attr.getVectorize()) |
| 252 | convertLoopOptions(options); |
| 253 | if (auto options = attr.getInterleave()) |
| 254 | convertLoopOptions(options); |
| 255 | if (auto options = attr.getUnroll()) |
| 256 | convertLoopOptions(options); |
| 257 | if (auto options = attr.getUnrollAndJam()) |
| 258 | convertLoopOptions(options); |
| 259 | if (auto options = attr.getLicm()) |
| 260 | convertLoopOptions(options); |
| 261 | if (auto options = attr.getDistribute()) |
| 262 | convertLoopOptions(options); |
| 263 | if (auto options = attr.getPipeline()) |
| 264 | convertLoopOptions(options); |
| 265 | if (auto options = attr.getPeeled()) |
| 266 | convertLoopOptions(options); |
| 267 | if (auto options = attr.getUnswitch()) |
| 268 | convertLoopOptions(options); |
| 269 | |
| 270 | ArrayRef<AccessGroupAttr> parallelAccessGroups = attr.getParallelAccesses(); |
| 271 | if (!parallelAccessGroups.empty()) { |
| 272 | SmallVector<llvm::Metadata *> parallelAccess; |
| 273 | parallelAccess.push_back( |
| 274 | Elt: llvm::MDString::get(Context&: ctx, Str: "llvm.loop.parallel_accesses" )); |
| 275 | for (AccessGroupAttr accessGroupAttr : parallelAccessGroups) |
| 276 | parallelAccess.push_back( |
| 277 | Elt: loopAnnotationTranslation.getAccessGroup(accessGroupAttr)); |
| 278 | metadataNodes.push_back(Elt: llvm::MDNode::get(Context&: ctx, MDs: parallelAccess)); |
| 279 | } |
| 280 | |
| 281 | // Create loop options and set the first operand to itself. |
| 282 | llvm::MDNode *loopMD = llvm::MDNode::get(Context&: ctx, MDs: metadataNodes); |
| 283 | loopMD->replaceOperandWith(I: 0, New: loopMD); |
| 284 | |
| 285 | return loopMD; |
| 286 | } |
| 287 | |
| 288 | llvm::MDNode * |
| 289 | LoopAnnotationTranslation::translateLoopAnnotation(LoopAnnotationAttr attr, |
| 290 | Operation *op) { |
| 291 | if (!attr) |
| 292 | return nullptr; |
| 293 | |
| 294 | llvm::MDNode *loopMD = lookupLoopMetadata(options: attr); |
| 295 | if (loopMD) |
| 296 | return loopMD; |
| 297 | |
| 298 | loopMD = |
| 299 | LoopAnnotationConversion(attr, op, *this, this->llvmModule.getContext()) |
| 300 | .convert(); |
| 301 | // Store a map from this Attribute to the LLVM metadata in case we |
| 302 | // encounter it again. |
| 303 | mapLoopMetadata(options: attr, metadata: loopMD); |
| 304 | return loopMD; |
| 305 | } |
| 306 | |
| 307 | llvm::MDNode * |
| 308 | LoopAnnotationTranslation::getAccessGroup(AccessGroupAttr accessGroupAttr) { |
| 309 | auto [result, inserted] = |
| 310 | accessGroupMetadataMapping.try_emplace(Key: accessGroupAttr); |
| 311 | if (inserted) |
| 312 | result->second = llvm::MDNode::getDistinct(Context&: llvmModule.getContext(), MDs: {}); |
| 313 | return result->second; |
| 314 | } |
| 315 | |
| 316 | llvm::MDNode * |
| 317 | LoopAnnotationTranslation::getAccessGroups(AccessGroupOpInterface op) { |
| 318 | ArrayAttr accessGroups = op.getAccessGroupsOrNull(); |
| 319 | if (!accessGroups || accessGroups.empty()) |
| 320 | return nullptr; |
| 321 | |
| 322 | SmallVector<llvm::Metadata *> groupMDs; |
| 323 | for (AccessGroupAttr group : accessGroups.getAsRange<AccessGroupAttr>()) |
| 324 | groupMDs.push_back(Elt: getAccessGroup(accessGroupAttr: group)); |
| 325 | if (groupMDs.size() == 1) |
| 326 | return llvm::cast<llvm::MDNode>(Val: groupMDs.front()); |
| 327 | return llvm::MDNode::get(Context&: llvmModule.getContext(), MDs: groupMDs); |
| 328 | } |
| 329 | |