1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "clang/AST/Expr.h"
13#include "clang/Basic/CodeGenOptions.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
20#include <optional>
21using namespace clang::CodeGen;
22using namespace llvm;
23
24MDNode *
25LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
26 LLVMContext &Ctx = Header->getContext();
27 SmallVector<Metadata *, 4> NewLoopProperties;
28 NewLoopProperties.push_back(Elt: nullptr);
29 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
30
31 MDNode *LoopID = MDNode::getDistinct(Context&: Ctx, MDs: NewLoopProperties);
32 LoopID->replaceOperandWith(I: 0, New: LoopID);
33 return LoopID;
34}
35
36MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37 ArrayRef<Metadata *> LoopProperties,
38 bool &HasUserTransforms) {
39 LLVMContext &Ctx = Header->getContext();
40
41 std::optional<bool> Enabled;
42 if (Attrs.PipelineDisabled)
43 Enabled = false;
44 else if (Attrs.PipelineInitiationInterval != 0)
45 Enabled = true;
46
47 if (Enabled != true) {
48 SmallVector<Metadata *, 4> NewLoopProperties;
49 if (Enabled == false) {
50 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
51 NewLoopProperties.push_back(
52 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.disable"),
53 ConstantAsMetadata::get(C: ConstantInt::get(
54 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 1))}));
55 LoopProperties = NewLoopProperties;
56 }
57 return createLoopPropertiesMetadata(LoopProperties);
58 }
59
60 SmallVector<Metadata *, 4> Args;
61 Args.push_back(Elt: nullptr);
62 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
63
64 if (Attrs.PipelineInitiationInterval > 0) {
65 Metadata *Vals[] = {
66 MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.initiationinterval"),
67 ConstantAsMetadata::get(C: ConstantInt::get(
68 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.PipelineInitiationInterval))};
69 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
70 }
71
72 // No follow-up: This is the last transformation.
73
74 MDNode *LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
75 LoopID->replaceOperandWith(I: 0, New: LoopID);
76 HasUserTransforms = true;
77 return LoopID;
78}
79
80MDNode *
81LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
82 ArrayRef<Metadata *> LoopProperties,
83 bool &HasUserTransforms) {
84 LLVMContext &Ctx = Header->getContext();
85
86 std::optional<bool> Enabled;
87 if (Attrs.UnrollEnable == LoopAttributes::Disable)
88 Enabled = false;
89 else if (Attrs.UnrollEnable == LoopAttributes::Full)
90 Enabled = std::nullopt;
91 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
92 Attrs.UnrollCount != 0)
93 Enabled = true;
94
95 if (Enabled != true) {
96 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
97 // if unrolling is disabled.
98 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
99 }
100
101 SmallVector<Metadata *, 4> FollowupLoopProperties;
102
103 // Apply all loop properties to the unrolled loop.
104 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
105
106 // Don't unroll an already unrolled loop.
107 FollowupLoopProperties.push_back(
108 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
109
110 bool FollowupHasTransforms = false;
111 MDNode *Followup = createPipeliningMetadata(Attrs, LoopProperties: FollowupLoopProperties,
112 HasUserTransforms&: FollowupHasTransforms);
113
114 SmallVector<Metadata *, 4> Args;
115 Args.push_back(Elt: nullptr);
116 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
117
118 // Setting unroll.count
119 if (Attrs.UnrollCount > 0) {
120 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"),
121 ConstantAsMetadata::get(C: ConstantInt::get(
122 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.UnrollCount))};
123 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
124 }
125
126 // Setting unroll.full or unroll.disable
127 if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")};
129 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
130 }
131
132 if (FollowupHasTransforms)
133 Args.push_back(Elt: MDNode::get(
134 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.followup_all"), Followup}));
135
136 MDNode *LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
137 LoopID->replaceOperandWith(I: 0, New: LoopID);
138 HasUserTransforms = true;
139 return LoopID;
140}
141
142MDNode *
143LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144 ArrayRef<Metadata *> LoopProperties,
145 bool &HasUserTransforms) {
146 LLVMContext &Ctx = Header->getContext();
147
148 std::optional<bool> Enabled;
149 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150 Enabled = false;
151 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152 Attrs.UnrollAndJamCount != 0)
153 Enabled = true;
154
155 if (Enabled != true) {
156 SmallVector<Metadata *, 4> NewLoopProperties;
157 if (Enabled == false) {
158 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
159 NewLoopProperties.push_back(Elt: MDNode::get(
160 Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
161 LoopProperties = NewLoopProperties;
162 }
163 return createPartialUnrollMetadata(Attrs, LoopProperties,
164 HasUserTransforms);
165 }
166
167 SmallVector<Metadata *, 4> FollowupLoopProperties;
168 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
169 FollowupLoopProperties.push_back(
170 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
171
172 bool FollowupHasTransforms = false;
173 MDNode *Followup = createPartialUnrollMetadata(Attrs, LoopProperties: FollowupLoopProperties,
174 HasUserTransforms&: FollowupHasTransforms);
175
176 SmallVector<Metadata *, 4> Args;
177 Args.push_back(Elt: nullptr);
178 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
179
180 // Setting unroll_and_jam.count
181 if (Attrs.UnrollAndJamCount > 0) {
182 Metadata *Vals[] = {
183 MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.count"),
184 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
185 V: Attrs.UnrollAndJamCount))};
186 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
187 }
188
189 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
190 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.enable")};
191 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
192 }
193
194 if (FollowupHasTransforms)
195 Args.push_back(Elt: MDNode::get(
196 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.followup_outer"),
197 Followup}));
198
199 if (UnrollAndJamInnerFollowup)
200 Args.push_back(Elt: MDNode::get(
201 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.followup_inner"),
202 UnrollAndJamInnerFollowup}));
203
204 MDNode *LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
205 LoopID->replaceOperandWith(I: 0, New: LoopID);
206 HasUserTransforms = true;
207 return LoopID;
208}
209
210MDNode *
211LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
212 ArrayRef<Metadata *> LoopProperties,
213 bool &HasUserTransforms) {
214 LLVMContext &Ctx = Header->getContext();
215
216 std::optional<bool> Enabled;
217 if (Attrs.VectorizeEnable == LoopAttributes::Disable)
218 Enabled = false;
219 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
220 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
221 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
222 Attrs.VectorizeScalable != LoopAttributes::Unspecified)
223 Enabled = true;
224
225 if (Enabled != true) {
226 SmallVector<Metadata *, 4> NewLoopProperties;
227 if (Enabled == false) {
228 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
229 NewLoopProperties.push_back(
230 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
231 ConstantAsMetadata::get(C: ConstantInt::get(
232 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
233 LoopProperties = NewLoopProperties;
234 }
235 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236 }
237
238 // Apply all loop properties to the vectorized loop.
239 SmallVector<Metadata *, 4> FollowupLoopProperties;
240 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
241
242 // Don't vectorize an already vectorized loop.
243 FollowupLoopProperties.push_back(
244 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
245
246 bool FollowupHasTransforms = false;
247 MDNode *Followup = createUnrollAndJamMetadata(Attrs, LoopProperties: FollowupLoopProperties,
248 HasUserTransforms&: FollowupHasTransforms);
249
250 SmallVector<Metadata *, 4> Args;
251 Args.push_back(Elt: nullptr);
252 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
253
254 // Setting vectorize.predicate when it has been specified and vectorization
255 // has not been disabled.
256 bool IsVectorPredicateEnabled = false;
257 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
258 IsVectorPredicateEnabled =
259 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
260
261 Metadata *Vals[] = {
262 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.predicate.enable"),
263 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx),
264 V: IsVectorPredicateEnabled))};
265 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
266 }
267
268 // Setting vectorize.width
269 if (Attrs.VectorizeWidth > 0) {
270 Metadata *Vals[] = {
271 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
272 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
273 V: Attrs.VectorizeWidth))};
274
275 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
276 }
277
278 if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
279 bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
280 Metadata *Vals[] = {
281 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.scalable.enable"),
282 ConstantAsMetadata::get(
283 C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx), V: IsScalable))};
284 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
285 }
286
287 // Setting interleave.count
288 if (Attrs.InterleaveCount > 0) {
289 Metadata *Vals[] = {
290 MDString::get(Context&: Ctx, Str: "llvm.loop.interleave.count"),
291 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
292 V: Attrs.InterleaveCount))};
293 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
294 }
295
296 // vectorize.enable is set if:
297 // 1) loop hint vectorize.enable is set, or
298 // 2) it is implied when vectorize.predicate is set, or
299 // 3) it is implied when vectorize.width is set to a value > 1
300 // 4) it is implied when vectorize.scalable.enable is true
301 // 5) it is implied when vectorize.width is unset (0) and the user
302 // explicitly requested fixed-width vectorization, i.e.
303 // vectorize.scalable.enable is false.
304 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
305 (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
306 Attrs.VectorizeWidth > 1 ||
307 Attrs.VectorizeScalable == LoopAttributes::Enable ||
308 (Attrs.VectorizeScalable == LoopAttributes::Disable &&
309 Attrs.VectorizeWidth != 1)) {
310 bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
311 Args.push_back(
312 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
313 ConstantAsMetadata::get(C: ConstantInt::get(
314 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: AttrVal))}));
315 }
316
317 if (FollowupHasTransforms)
318 Args.push_back(Elt: MDNode::get(
319 Context&: Ctx,
320 MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.followup_all"), Followup}));
321
322 MDNode *LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
323 LoopID->replaceOperandWith(I: 0, New: LoopID);
324 HasUserTransforms = true;
325 return LoopID;
326}
327
328MDNode *
329LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
330 ArrayRef<Metadata *> LoopProperties,
331 bool &HasUserTransforms) {
332 LLVMContext &Ctx = Header->getContext();
333
334 std::optional<bool> Enabled;
335 if (Attrs.DistributeEnable == LoopAttributes::Disable)
336 Enabled = false;
337 if (Attrs.DistributeEnable == LoopAttributes::Enable)
338 Enabled = true;
339
340 if (Enabled != true) {
341 SmallVector<Metadata *, 4> NewLoopProperties;
342 if (Enabled == false) {
343 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
344 NewLoopProperties.push_back(
345 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
346 ConstantAsMetadata::get(C: ConstantInt::get(
347 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
348 LoopProperties = NewLoopProperties;
349 }
350 return createLoopVectorizeMetadata(Attrs, LoopProperties,
351 HasUserTransforms);
352 }
353
354 bool FollowupHasTransforms = false;
355 MDNode *Followup =
356 createLoopVectorizeMetadata(Attrs, LoopProperties, HasUserTransforms&: FollowupHasTransforms);
357
358 SmallVector<Metadata *, 4> Args;
359 Args.push_back(Elt: nullptr);
360 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
361
362 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
363 ConstantAsMetadata::get(C: ConstantInt::get(
364 Ty: llvm::Type::getInt1Ty(C&: Ctx),
365 V: (Attrs.DistributeEnable == LoopAttributes::Enable)))};
366 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
367
368 if (FollowupHasTransforms)
369 Args.push_back(Elt: MDNode::get(
370 Context&: Ctx,
371 MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.followup_all"), Followup}));
372
373 MDNode *LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
374 LoopID->replaceOperandWith(I: 0, New: LoopID);
375 HasUserTransforms = true;
376 return LoopID;
377}
378
379MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
380 ArrayRef<Metadata *> LoopProperties,
381 bool &HasUserTransforms) {
382 LLVMContext &Ctx = Header->getContext();
383
384 std::optional<bool> Enabled;
385 if (Attrs.UnrollEnable == LoopAttributes::Disable)
386 Enabled = false;
387 else if (Attrs.UnrollEnable == LoopAttributes::Full)
388 Enabled = true;
389
390 if (Enabled != true) {
391 SmallVector<Metadata *, 4> NewLoopProperties;
392 if (Enabled == false) {
393 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
394 NewLoopProperties.push_back(
395 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
396 LoopProperties = NewLoopProperties;
397 }
398 return createLoopDistributeMetadata(Attrs, LoopProperties,
399 HasUserTransforms);
400 }
401
402 SmallVector<Metadata *, 4> Args;
403 Args.push_back(Elt: nullptr);
404 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
405 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full")));
406
407 // No follow-up: there is no loop after full unrolling.
408 // TODO: Warn if there are transformations after full unrolling.
409
410 MDNode *LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
411 LoopID->replaceOperandWith(I: 0, New: LoopID);
412 HasUserTransforms = true;
413 return LoopID;
414}
415
416MDNode *LoopInfo::createMetadata(
417 const LoopAttributes &Attrs,
418 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
419 bool &HasUserTransforms) {
420 SmallVector<Metadata *, 3> LoopProperties;
421
422 // If we have a valid start debug location for the loop, add it.
423 if (StartLoc) {
424 LoopProperties.push_back(Elt: StartLoc.getAsMDNode());
425
426 // If we also have a valid end debug location for the loop, add it.
427 if (EndLoc)
428 LoopProperties.push_back(Elt: EndLoc.getAsMDNode());
429 }
430
431 LLVMContext &Ctx = Header->getContext();
432 if (Attrs.MustProgress)
433 LoopProperties.push_back(
434 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.mustprogress")));
435
436 assert(!!AccGroup == Attrs.IsParallel &&
437 "There must be an access group iff the loop is parallel");
438 if (Attrs.IsParallel) {
439 LoopProperties.push_back(Elt: MDNode::get(
440 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccGroup}));
441 }
442
443 // Setting clang::code_align attribute.
444 if (Attrs.CodeAlign > 0) {
445 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.align"),
446 ConstantAsMetadata::get(C: ConstantInt::get(
447 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.CodeAlign))};
448 LoopProperties.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
449 }
450
451 LoopProperties.insert(I: LoopProperties.end(), From: AdditionalLoopProperties.begin(),
452 To: AdditionalLoopProperties.end());
453 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
454}
455
456LoopAttributes::LoopAttributes(bool IsParallel)
457 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
458 UnrollEnable(LoopAttributes::Unspecified),
459 UnrollAndJamEnable(LoopAttributes::Unspecified),
460 VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
461 VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
462 UnrollCount(0), UnrollAndJamCount(0),
463 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
464 PipelineInitiationInterval(0), CodeAlign(0), MustProgress(false) {}
465
466void LoopAttributes::clear() {
467 IsParallel = false;
468 VectorizeWidth = 0;
469 VectorizeScalable = LoopAttributes::Unspecified;
470 InterleaveCount = 0;
471 UnrollCount = 0;
472 UnrollAndJamCount = 0;
473 VectorizeEnable = LoopAttributes::Unspecified;
474 UnrollEnable = LoopAttributes::Unspecified;
475 UnrollAndJamEnable = LoopAttributes::Unspecified;
476 VectorizePredicateEnable = LoopAttributes::Unspecified;
477 DistributeEnable = LoopAttributes::Unspecified;
478 PipelineDisabled = false;
479 PipelineInitiationInterval = 0;
480 CodeAlign = 0;
481 MustProgress = false;
482}
483
484LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
485 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
486 LoopInfo *Parent)
487 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
488 Parent(Parent) {
489
490 if (Attrs.IsParallel) {
491 // Create an access group for this loop.
492 LLVMContext &Ctx = Header->getContext();
493 AccGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
494 }
495
496 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
497 Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
498 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
499 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
500 Attrs.PipelineInitiationInterval == 0 &&
501 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
502 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
503 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
504 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
505 Attrs.DistributeEnable == LoopAttributes::Unspecified &&
506 Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress)
507 return;
508
509 TempLoopID = MDNode::getTemporary(Context&: Header->getContext(), MDs: std::nullopt);
510}
511
512void LoopInfo::finish() {
513 // We did not annotate the loop body instructions because there are no
514 // attributes for this loop.
515 if (!TempLoopID)
516 return;
517
518 MDNode *LoopID;
519 LoopAttributes CurLoopAttr = Attrs;
520 LLVMContext &Ctx = Header->getContext();
521
522 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
523 Parent->Attrs.UnrollAndJamCount != 0)) {
524 // Parent unroll-and-jams this loop.
525 // Split the transformations in those that happens before the unroll-and-jam
526 // and those after.
527
528 LoopAttributes BeforeJam, AfterJam;
529
530 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
531
532 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
533 BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
534 BeforeJam.InterleaveCount = Attrs.InterleaveCount;
535 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
536 BeforeJam.DistributeEnable = Attrs.DistributeEnable;
537 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
538
539 switch (Attrs.UnrollEnable) {
540 case LoopAttributes::Unspecified:
541 case LoopAttributes::Disable:
542 BeforeJam.UnrollEnable = Attrs.UnrollEnable;
543 AfterJam.UnrollEnable = Attrs.UnrollEnable;
544 break;
545 case LoopAttributes::Full:
546 BeforeJam.UnrollEnable = LoopAttributes::Full;
547 break;
548 case LoopAttributes::Enable:
549 AfterJam.UnrollEnable = LoopAttributes::Enable;
550 break;
551 }
552
553 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
554 AfterJam.UnrollCount = Attrs.UnrollCount;
555 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
556 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
557
558 // If this loop is subject of an unroll-and-jam by the parent loop, and has
559 // an unroll-and-jam annotation itself, we have to decide whether to first
560 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
561 // UnrollAndJam pass processes loops from inner to outer, so we apply the
562 // inner first.
563 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
564 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
565
566 // Set the inner followup metadata to process by the outer loop. Only
567 // consider the first inner loop.
568 if (!Parent->UnrollAndJamInnerFollowup) {
569 // Splitting the attributes into a BeforeJam and an AfterJam part will
570 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
571 // to be forwarded to the AfterJam part. We detect the situation here and
572 // add it manually.
573 SmallVector<Metadata *, 1> BeforeLoopProperties;
574 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
575 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
576 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
577 BeforeJam.VectorizeScalable == LoopAttributes::Enable)
578 BeforeLoopProperties.push_back(
579 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
580
581 bool InnerFollowupHasTransform = false;
582 MDNode *InnerFollowup = createMetadata(Attrs: AfterJam, AdditionalLoopProperties: BeforeLoopProperties,
583 HasUserTransforms&: InnerFollowupHasTransform);
584 if (InnerFollowupHasTransform)
585 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
586 }
587
588 CurLoopAttr = BeforeJam;
589 }
590
591 bool HasUserTransforms = false;
592 LoopID = createMetadata(Attrs: CurLoopAttr, AdditionalLoopProperties: {}, HasUserTransforms);
593 TempLoopID->replaceAllUsesWith(MD: LoopID);
594}
595
596void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
597 const llvm::DebugLoc &EndLoc) {
598 Active.emplace_back(
599 Args: new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
600 Active.empty() ? nullptr : Active.back().get()));
601 // Clear the attributes so nested loops do not inherit them.
602 StagedAttrs.clear();
603}
604
605void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
606 const clang::CodeGenOptions &CGOpts,
607 ArrayRef<const clang::Attr *> Attrs,
608 const llvm::DebugLoc &StartLoc,
609 const llvm::DebugLoc &EndLoc, bool MustProgress) {
610 // Identify loop hint attributes from Attrs.
611 for (const auto *Attr : Attrs) {
612 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Val: Attr);
613 const OpenCLUnrollHintAttr *OpenCLHint =
614 dyn_cast<OpenCLUnrollHintAttr>(Val: Attr);
615 const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Val: Attr);
616 // Skip non loop hint attributes
617 if (!LH && !OpenCLHint && !HLSLLoopHint) {
618 continue;
619 }
620
621 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
622 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
623 unsigned ValueInt = 1;
624 // Translate opencl_unroll_hint attribute argument to
625 // equivalent LoopHintAttr enums.
626 // OpenCL v2.0 s6.11.5:
627 // 0 - enable unroll (no argument).
628 // 1 - disable unroll.
629 // other positive integer n - unroll by n.
630 if (OpenCLHint) {
631 ValueInt = OpenCLHint->getUnrollHint();
632 if (ValueInt == 0) {
633 State = LoopHintAttr::Enable;
634 } else if (ValueInt != 1) {
635 Option = LoopHintAttr::UnrollCount;
636 State = LoopHintAttr::Numeric;
637 }
638 } else if (HLSLLoopHint) {
639 ValueInt = HLSLLoopHint->getDirective();
640 if (HLSLLoopHint->getSemanticSpelling() ==
641 HLSLLoopHintAttr::Spelling::Microsoft_unroll) {
642 if (ValueInt == 0)
643 State = LoopHintAttr::Enable;
644 if (ValueInt > 0) {
645 Option = LoopHintAttr::UnrollCount;
646 State = LoopHintAttr::Numeric;
647 }
648 }
649 } else if (LH) {
650 auto *ValueExpr = LH->getValue();
651 if (ValueExpr) {
652 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
653 ValueInt = ValueAPS.getSExtValue();
654 }
655
656 Option = LH->getOption();
657 State = LH->getState();
658 }
659 switch (State) {
660 case LoopHintAttr::Disable:
661 switch (Option) {
662 case LoopHintAttr::Vectorize:
663 // Disable vectorization by specifying a width of 1.
664 setVectorizeWidth(1);
665 setVectorizeScalable(LoopAttributes::Unspecified);
666 break;
667 case LoopHintAttr::Interleave:
668 // Disable interleaving by speciyfing a count of 1.
669 setInterleaveCount(1);
670 break;
671 case LoopHintAttr::Unroll:
672 setUnrollState(LoopAttributes::Disable);
673 break;
674 case LoopHintAttr::UnrollAndJam:
675 setUnrollAndJamState(LoopAttributes::Disable);
676 break;
677 case LoopHintAttr::VectorizePredicate:
678 setVectorizePredicateState(LoopAttributes::Disable);
679 break;
680 case LoopHintAttr::Distribute:
681 setDistributeState(false);
682 break;
683 case LoopHintAttr::PipelineDisabled:
684 setPipelineDisabled(true);
685 break;
686 case LoopHintAttr::UnrollCount:
687 case LoopHintAttr::UnrollAndJamCount:
688 case LoopHintAttr::VectorizeWidth:
689 case LoopHintAttr::InterleaveCount:
690 case LoopHintAttr::PipelineInitiationInterval:
691 llvm_unreachable("Options cannot be disabled.");
692 break;
693 }
694 break;
695 case LoopHintAttr::Enable:
696 switch (Option) {
697 case LoopHintAttr::Vectorize:
698 case LoopHintAttr::Interleave:
699 setVectorizeEnable(true);
700 break;
701 case LoopHintAttr::Unroll:
702 setUnrollState(LoopAttributes::Enable);
703 break;
704 case LoopHintAttr::UnrollAndJam:
705 setUnrollAndJamState(LoopAttributes::Enable);
706 break;
707 case LoopHintAttr::VectorizePredicate:
708 setVectorizePredicateState(LoopAttributes::Enable);
709 break;
710 case LoopHintAttr::Distribute:
711 setDistributeState(true);
712 break;
713 case LoopHintAttr::UnrollCount:
714 case LoopHintAttr::UnrollAndJamCount:
715 case LoopHintAttr::VectorizeWidth:
716 case LoopHintAttr::InterleaveCount:
717 case LoopHintAttr::PipelineDisabled:
718 case LoopHintAttr::PipelineInitiationInterval:
719 llvm_unreachable("Options cannot enabled.");
720 break;
721 }
722 break;
723 case LoopHintAttr::AssumeSafety:
724 switch (Option) {
725 case LoopHintAttr::Vectorize:
726 case LoopHintAttr::Interleave:
727 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
728 setParallel(true);
729 setVectorizeEnable(true);
730 break;
731 case LoopHintAttr::Unroll:
732 case LoopHintAttr::UnrollAndJam:
733 case LoopHintAttr::VectorizePredicate:
734 case LoopHintAttr::UnrollCount:
735 case LoopHintAttr::UnrollAndJamCount:
736 case LoopHintAttr::VectorizeWidth:
737 case LoopHintAttr::InterleaveCount:
738 case LoopHintAttr::Distribute:
739 case LoopHintAttr::PipelineDisabled:
740 case LoopHintAttr::PipelineInitiationInterval:
741 llvm_unreachable("Options cannot be used to assume mem safety.");
742 break;
743 }
744 break;
745 case LoopHintAttr::Full:
746 switch (Option) {
747 case LoopHintAttr::Unroll:
748 setUnrollState(LoopAttributes::Full);
749 break;
750 case LoopHintAttr::UnrollAndJam:
751 setUnrollAndJamState(LoopAttributes::Full);
752 break;
753 case LoopHintAttr::Vectorize:
754 case LoopHintAttr::Interleave:
755 case LoopHintAttr::UnrollCount:
756 case LoopHintAttr::UnrollAndJamCount:
757 case LoopHintAttr::VectorizeWidth:
758 case LoopHintAttr::InterleaveCount:
759 case LoopHintAttr::Distribute:
760 case LoopHintAttr::PipelineDisabled:
761 case LoopHintAttr::PipelineInitiationInterval:
762 case LoopHintAttr::VectorizePredicate:
763 llvm_unreachable("Options cannot be used with 'full' hint.");
764 break;
765 }
766 break;
767 case LoopHintAttr::FixedWidth:
768 case LoopHintAttr::ScalableWidth:
769 switch (Option) {
770 case LoopHintAttr::VectorizeWidth:
771 setVectorizeScalable(State == LoopHintAttr::ScalableWidth
772 ? LoopAttributes::Enable
773 : LoopAttributes::Disable);
774 if (LH->getValue())
775 setVectorizeWidth(ValueInt);
776 break;
777 default:
778 llvm_unreachable("Options cannot be used with 'scalable' hint.");
779 break;
780 }
781 break;
782 case LoopHintAttr::Numeric:
783 switch (Option) {
784 case LoopHintAttr::InterleaveCount:
785 setInterleaveCount(ValueInt);
786 break;
787 case LoopHintAttr::UnrollCount:
788 setUnrollCount(ValueInt);
789 break;
790 case LoopHintAttr::UnrollAndJamCount:
791 setUnrollAndJamCount(ValueInt);
792 break;
793 case LoopHintAttr::PipelineInitiationInterval:
794 setPipelineInitiationInterval(ValueInt);
795 break;
796 case LoopHintAttr::Unroll:
797 case LoopHintAttr::UnrollAndJam:
798 case LoopHintAttr::VectorizePredicate:
799 case LoopHintAttr::Vectorize:
800 case LoopHintAttr::VectorizeWidth:
801 case LoopHintAttr::Interleave:
802 case LoopHintAttr::Distribute:
803 case LoopHintAttr::PipelineDisabled:
804 llvm_unreachable("Options cannot be assigned a value.");
805 break;
806 }
807 break;
808 }
809 }
810
811 // Identify loop attribute 'code_align' from Attrs.
812 // For attribute code_align:
813 // n - 'llvm.loop.align i32 n' metadata will be emitted.
814 if (const auto *CodeAlign = getSpecificAttr<const CodeAlignAttr>(container: Attrs)) {
815 const auto *CE = cast<ConstantExpr>(Val: CodeAlign->getAlignment());
816 llvm::APSInt ArgVal = CE->getResultAsAPSInt();
817 setCodeAlign(ArgVal.getSExtValue());
818 }
819
820 setMustProgress(MustProgress);
821
822 if (CGOpts.OptimizationLevel > 0)
823 // Disable unrolling for the loop, if unrolling is disabled (via
824 // -fno-unroll-loops) and no pragmas override the decision.
825 if (!CGOpts.UnrollLoops &&
826 (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
827 StagedAttrs.UnrollCount == 0))
828 setUnrollState(LoopAttributes::Disable);
829
830 /// Stage the attributes.
831 push(Header, StartLoc, EndLoc);
832}
833
834void LoopInfoStack::pop() {
835 assert(!Active.empty() && "No active loops to pop");
836 Active.back()->finish();
837 Active.pop_back();
838}
839
840void LoopInfoStack::InsertHelper(Instruction *I) const {
841 if (I->mayReadOrWriteMemory()) {
842 SmallVector<Metadata *, 4> AccessGroups;
843 for (const auto &AL : Active) {
844 // Here we assume that every loop that has an access group is parallel.
845 if (MDNode *Group = AL->getAccessGroup())
846 AccessGroups.push_back(Elt: Group);
847 }
848 MDNode *UnionMD = nullptr;
849 if (AccessGroups.size() == 1)
850 UnionMD = cast<MDNode>(Val: AccessGroups[0]);
851 else if (AccessGroups.size() >= 2)
852 UnionMD = MDNode::get(Context&: I->getContext(), MDs: AccessGroups);
853 I->setMetadata(Kind: "llvm.access.group", Node: UnionMD);
854 }
855
856 if (!hasInfo())
857 return;
858
859 const LoopInfo &L = getInfo();
860 if (!L.getLoopID())
861 return;
862
863 if (I->isTerminator()) {
864 for (BasicBlock *Succ : successors(I))
865 if (Succ == L.getHeader()) {
866 I->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: L.getLoopID());
867 break;
868 }
869 return;
870 }
871}
872