1//===- LoopAnnotationTranslation.cpp - Loop annotation export -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "LoopAnnotationTranslation.h"
10#include "llvm/IR/DebugInfoMetadata.h"
11
12using namespace mlir;
13using namespace mlir::LLVM;
14using namespace mlir::LLVM::detail;
15
16namespace {
17/// Helper class that keeps the state of one attribute to metadata conversion.
18struct LoopAnnotationConversion {
19 LoopAnnotationConversion(LoopAnnotationAttr attr, Operation *op,
20 LoopAnnotationTranslation &loopAnnotationTranslation,
21 llvm::LLVMContext &ctx)
22 : attr(attr), op(op),
23 loopAnnotationTranslation(loopAnnotationTranslation), ctx(ctx) {}
24
25 /// Converts this struct's loop annotation into a corresponding LLVMIR
26 /// metadata representation.
27 llvm::MDNode *convert();
28
29 /// Conversion functions for different payload attribute kinds.
30 void addUnitNode(StringRef name);
31 void addUnitNode(StringRef name, BoolAttr attr);
32 void addI32NodeWithVal(StringRef name, uint32_t val);
33 void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false);
34 void convertBooleanUnitNode(StringRef enableName, StringRef disableName,
35 BoolAttr attr, bool negated = false);
36 void convertI32Node(StringRef name, IntegerAttr attr);
37 void convertFollowupNode(StringRef name, LoopAnnotationAttr attr);
38 void convertLocation(FusedLoc attr);
39
40 /// Conversion functions for each for each loop annotation sub-attribute.
41 void convertLoopOptions(LoopVectorizeAttr options);
42 void convertLoopOptions(LoopInterleaveAttr options);
43 void convertLoopOptions(LoopUnrollAttr options);
44 void convertLoopOptions(LoopUnrollAndJamAttr options);
45 void convertLoopOptions(LoopLICMAttr options);
46 void convertLoopOptions(LoopDistributeAttr options);
47 void convertLoopOptions(LoopPipelineAttr options);
48 void convertLoopOptions(LoopPeeledAttr options);
49 void convertLoopOptions(LoopUnswitchAttr options);
50
51 LoopAnnotationAttr attr;
52 Operation *op;
53 LoopAnnotationTranslation &loopAnnotationTranslation;
54 llvm::LLVMContext &ctx;
55 llvm::SmallVector<llvm::Metadata *> metadataNodes;
56};
57} // namespace
58
59void LoopAnnotationConversion::addUnitNode(StringRef name) {
60 metadataNodes.push_back(
61 Elt: llvm::MDNode::get(Context&: ctx, MDs: {llvm::MDString::get(Context&: ctx, Str: name)}));
62}
63
64void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) {
65 if (attr && attr.getValue())
66 addUnitNode(name);
67}
68
69void LoopAnnotationConversion::addI32NodeWithVal(StringRef name, uint32_t val) {
70 llvm::Constant *cstValue = llvm::ConstantInt::get(
71 Ty: llvm::IntegerType::get(C&: ctx, /*NumBits=*/32), V: val, /*isSigned=*/IsSigned: false);
72 metadataNodes.push_back(
73 Elt: llvm::MDNode::get(Context&: ctx, MDs: {llvm::MDString::get(Context&: ctx, Str: name),
74 llvm::ConstantAsMetadata::get(C: cstValue)}));
75}
76
77void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr,
78 bool negated) {
79 if (!attr)
80 return;
81 bool val = negated ^ attr.getValue();
82 llvm::Constant *cstValue = llvm::ConstantInt::getBool(Context&: ctx, V: val);
83 metadataNodes.push_back(
84 Elt: llvm::MDNode::get(Context&: ctx, MDs: {llvm::MDString::get(Context&: ctx, Str: name),
85 llvm::ConstantAsMetadata::get(C: cstValue)}));
86}
87
88/// Emits the single-operand node of an enable/disable pair. As in
89/// convertBoolNode, the attribute is tri-state: a null \p attr emits no node
90/// at all, otherwise \p negated ^ the attribute value is the enable bit and
91/// selects which of the two names is emitted.
92void LoopAnnotationConversion::convertBooleanUnitNode(StringRef enableName,
93 StringRef disableName,
94 BoolAttr attr,
95 bool negated) {
96 if (!attr)
97 return;
98 addUnitNode(name: (negated ^ attr.getValue()) ? enableName : disableName);
99}
100
101void LoopAnnotationConversion::convertI32Node(StringRef name,
102 IntegerAttr attr) {
103 if (!attr)
104 return;
105 addI32NodeWithVal(name, val: attr.getInt());
106}
107
108void LoopAnnotationConversion::convertFollowupNode(StringRef name,
109 LoopAnnotationAttr attr) {
110 if (!attr)
111 return;
112
113 llvm::MDNode *loopID =
114 loopAnnotationTranslation.translateLoopAnnotation(attr, op);
115
116 // Follow-ups contain properties directly, without the LoopID self-reference.
117 SmallVector<llvm::Metadata *> operands;
118 operands.push_back(Elt: llvm::MDString::get(Context&: ctx, Str: name));
119 for (const llvm::MDOperand &operand : llvm::drop_begin(RangeOrContainer: loopID->operands()))
120 operands.push_back(Elt: operand.get());
121 metadataNodes.push_back(Elt: llvm::MDNode::get(Context&: ctx, MDs: operands));
122}
123
124void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) {
125 convertBooleanUnitNode(enableName: "llvm.loop.vectorize.enable",
126 disableName: "llvm.loop.vectorize.disable", attr: options.getDisable(),
127 /*negated=*/true);
128 convertBooleanUnitNode(enableName: "llvm.loop.vectorize.predicate.enable",
129 disableName: "llvm.loop.vectorize.predicate.disable",
130 attr: options.getPredicateEnable());
131 convertBooleanUnitNode(enableName: "llvm.loop.vectorize.scalable.enable",
132 disableName: "llvm.loop.vectorize.scalable.disable",
133 attr: options.getScalableEnable());
134 convertI32Node(name: "llvm.loop.vectorize.width", attr: options.getWidth());
135 convertFollowupNode(name: "llvm.loop.vectorize.followup_vectorized",
136 attr: options.getFollowupVectorized());
137 convertFollowupNode(name: "llvm.loop.vectorize.followup_epilogue",
138 attr: options.getFollowupEpilogue());
139 convertFollowupNode(name: "llvm.loop.vectorize.followup_all",
140 attr: options.getFollowupAll());
141}
142
143void LoopAnnotationConversion::convertLoopOptions(LoopInterleaveAttr options) {
144 convertI32Node(name: "llvm.loop.interleave.count", attr: options.getCount());
145}
146
147void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) {
148 if (auto disable = options.getDisable())
149 addUnitNode(name: disable.getValue() ? "llvm.loop.unroll.disable"
150 : "llvm.loop.unroll.enable");
151 convertI32Node(name: "llvm.loop.unroll.count", attr: options.getCount());
152 convertBoolNode(name: "llvm.loop.unroll.runtime.disable",
153 attr: options.getRuntimeDisable());
154 addUnitNode(name: "llvm.loop.unroll.full", attr: options.getFull());
155 convertFollowupNode(name: "llvm.loop.unroll.followup_unrolled",
156 attr: options.getFollowupUnrolled());
157 convertFollowupNode(name: "llvm.loop.unroll.followup_remainder",
158 attr: options.getFollowupRemainder());
159 convertFollowupNode(name: "llvm.loop.unroll.followup_all",
160 attr: options.getFollowupAll());
161}
162
163void LoopAnnotationConversion::convertLoopOptions(
164 LoopUnrollAndJamAttr options) {
165 if (auto disable = options.getDisable())
166 addUnitNode(name: disable.getValue() ? "llvm.loop.unroll_and_jam.disable"
167 : "llvm.loop.unroll_and_jam.enable");
168 convertI32Node(name: "llvm.loop.unroll_and_jam.count", attr: options.getCount());
169 convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_outer",
170 attr: options.getFollowupOuter());
171 convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_inner",
172 attr: options.getFollowupInner());
173 convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_remainder_outer",
174 attr: options.getFollowupRemainderOuter());
175 convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_remainder_inner",
176 attr: options.getFollowupRemainderInner());
177 convertFollowupNode(name: "llvm.loop.unroll_and_jam.followup_all",
178 attr: options.getFollowupAll());
179}
180
181void LoopAnnotationConversion::convertLoopOptions(LoopLICMAttr options) {
182 addUnitNode(name: "llvm.licm.disable", attr: options.getDisable());
183 addUnitNode(name: "llvm.loop.licm_versioning.disable",
184 attr: options.getVersioningDisable());
185}
186
187void LoopAnnotationConversion::convertLoopOptions(LoopDistributeAttr options) {
188 convertBooleanUnitNode(enableName: "llvm.loop.distribute.enable",
189 disableName: "llvm.loop.distribute.disable", attr: options.getDisable(),
190 /*negated=*/true);
191 convertFollowupNode(name: "llvm.loop.distribute.followup_coincident",
192 attr: options.getFollowupCoincident());
193 convertFollowupNode(name: "llvm.loop.distribute.followup_sequential",
194 attr: options.getFollowupSequential());
195 convertFollowupNode(name: "llvm.loop.distribute.followup_fallback",
196 attr: options.getFollowupFallback());
197 convertFollowupNode(name: "llvm.loop.distribute.followup_all",
198 attr: options.getFollowupAll());
199}
200
201void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) {
202 convertBoolNode(name: "llvm.loop.pipeline.disable", attr: options.getDisable());
203 convertI32Node(name: "llvm.loop.pipeline.initiationinterval",
204 attr: options.getInitiationinterval());
205}
206
207void LoopAnnotationConversion::convertLoopOptions(LoopPeeledAttr options) {
208 convertI32Node(name: "llvm.loop.peeled.count", attr: options.getCount());
209}
210
211void LoopAnnotationConversion::convertLoopOptions(LoopUnswitchAttr options) {
212 addUnitNode(name: "llvm.loop.unswitch.partial.disable",
213 attr: options.getPartialDisable());
214}
215
216void LoopAnnotationConversion::convertLocation(FusedLoc location) {
217 auto localScopeAttr =
218 dyn_cast_or_null<DILocalScopeAttr>(Val: location.getMetadata());
219 if (!localScopeAttr)
220 return;
221 auto *localScope = dyn_cast<llvm::DILocalScope>(
222 Val: loopAnnotationTranslation.moduleTranslation.translateDebugInfo(
223 attr: localScopeAttr));
224 if (!localScope)
225 return;
226 llvm::Metadata *loc =
227 loopAnnotationTranslation.moduleTranslation.translateLoc(loc: location,
228 scope: localScope);
229 metadataNodes.push_back(Elt: loc);
230}
231
232llvm::MDNode *LoopAnnotationConversion::convert() {
233 // Reserve operand 0 for loop id self reference.
234 auto dummy = llvm::MDNode::getTemporary(Context&: ctx, MDs: {});
235 metadataNodes.push_back(Elt: dummy.get());
236
237 if (FusedLoc startLoc = attr.getStartLoc())
238 convertLocation(location: startLoc);
239
240 if (FusedLoc endLoc = attr.getEndLoc())
241 convertLocation(location: endLoc);
242
243 addUnitNode(name: "llvm.loop.disable_nonforced", attr: attr.getDisableNonforced());
244 addUnitNode(name: "llvm.loop.mustprogress", attr: attr.getMustProgress());
245 // "isvectorized" is encoded as an i32 value.
246 if (BoolAttr isVectorized = attr.getIsVectorized())
247 addI32NodeWithVal(name: "llvm.loop.isvectorized", val: isVectorized.getValue());
248 if (uint32_t align = attr.getAlign())
249 addI32NodeWithVal(name: "llvm.loop.align", val: align);
250
251 if (auto options = attr.getVectorize())
252 convertLoopOptions(options);
253 if (auto options = attr.getInterleave())
254 convertLoopOptions(options);
255 if (auto options = attr.getUnroll())
256 convertLoopOptions(options);
257 if (auto options = attr.getUnrollAndJam())
258 convertLoopOptions(options);
259 if (auto options = attr.getLicm())
260 convertLoopOptions(options);
261 if (auto options = attr.getDistribute())
262 convertLoopOptions(options);
263 if (auto options = attr.getPipeline())
264 convertLoopOptions(options);
265 if (auto options = attr.getPeeled())
266 convertLoopOptions(options);
267 if (auto options = attr.getUnswitch())
268 convertLoopOptions(options);
269
270 ArrayRef<AccessGroupAttr> parallelAccessGroups = attr.getParallelAccesses();
271 if (!parallelAccessGroups.empty()) {
272 SmallVector<llvm::Metadata *> parallelAccess;
273 parallelAccess.push_back(
274 Elt: llvm::MDString::get(Context&: ctx, Str: "llvm.loop.parallel_accesses"));
275 for (AccessGroupAttr accessGroupAttr : parallelAccessGroups)
276 parallelAccess.push_back(
277 Elt: loopAnnotationTranslation.getAccessGroup(accessGroupAttr));
278 metadataNodes.push_back(Elt: llvm::MDNode::get(Context&: ctx, MDs: parallelAccess));
279 }
280
281 // Create loop options and set the first operand to itself.
282 llvm::MDNode *loopMD = llvm::MDNode::get(Context&: ctx, MDs: metadataNodes);
283 loopMD->replaceOperandWith(I: 0, New: loopMD);
284
285 return loopMD;
286}
287
288llvm::MDNode *
289LoopAnnotationTranslation::translateLoopAnnotation(LoopAnnotationAttr attr,
290 Operation *op) {
291 if (!attr)
292 return nullptr;
293
294 llvm::MDNode *loopMD = lookupLoopMetadata(options: attr);
295 if (loopMD)
296 return loopMD;
297
298 loopMD =
299 LoopAnnotationConversion(attr, op, *this, this->llvmModule.getContext())
300 .convert();
301 // Store a map from this Attribute to the LLVM metadata in case we
302 // encounter it again.
303 mapLoopMetadata(options: attr, metadata: loopMD);
304 return loopMD;
305}
306
307llvm::MDNode *
308LoopAnnotationTranslation::getAccessGroup(AccessGroupAttr accessGroupAttr) {
309 auto [result, inserted] =
310 accessGroupMetadataMapping.try_emplace(Key: accessGroupAttr);
311 if (inserted)
312 result->second = llvm::MDNode::getDistinct(Context&: llvmModule.getContext(), MDs: {});
313 return result->second;
314}
315
316llvm::MDNode *
317LoopAnnotationTranslation::getAccessGroups(AccessGroupOpInterface op) {
318 ArrayAttr accessGroups = op.getAccessGroupsOrNull();
319 if (!accessGroups || accessGroups.empty())
320 return nullptr;
321
322 SmallVector<llvm::Metadata *> groupMDs;
323 for (AccessGroupAttr group : accessGroups.getAsRange<AccessGroupAttr>())
324 groupMDs.push_back(Elt: getAccessGroup(accessGroupAttr: group));
325 if (groupMDs.size() == 1)
326 return llvm::cast<llvm::MDNode>(Val: groupMDs.front());
327 return llvm::MDNode::get(Context&: llvmModule.getContext(), MDs: groupMDs);
328}
329