1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "clang/AST/Expr.h"
13#include "clang/Basic/CodeGenOptions.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
20#include <optional>
21using namespace clang::CodeGen;
22using namespace llvm;
23
24MDNode *
25LoopInfo::createFollowupMetadata(const char *FollowupName,
26 ArrayRef<llvm::Metadata *> LoopProperties) {
27 LLVMContext &Ctx = Header->getContext();
28
29 SmallVector<Metadata *, 4> Args;
30 Args.push_back(Elt: MDString::get(Context&: Ctx, Str: FollowupName));
31 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
32 return MDNode::get(Context&: Ctx, MDs: Args);
33}
34
35SmallVector<Metadata *, 4>
36LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37 ArrayRef<Metadata *> LoopProperties,
38 bool &HasUserTransforms) {
39 LLVMContext &Ctx = Header->getContext();
40
41 std::optional<bool> Enabled;
42 if (Attrs.PipelineDisabled)
43 Enabled = false;
44 else if (Attrs.PipelineInitiationInterval != 0)
45 Enabled = true;
46
47 SmallVector<Metadata *, 4> Args;
48 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
49
50 if (Enabled != true) {
51 if (Enabled == false) {
52 Args.push_back(
53 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.disable"),
54 ConstantAsMetadata::get(C: ConstantInt::get(
55 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 1))}));
56 }
57 return Args;
58 }
59
60 if (Attrs.PipelineInitiationInterval > 0) {
61 Metadata *Vals[] = {
62 MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.initiationinterval"),
63 ConstantAsMetadata::get(C: ConstantInt::get(
64 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.PipelineInitiationInterval))};
65 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
66 }
67
68 // No follow-up: This is the last transformation.
69
70 HasUserTransforms = true;
71 return Args;
72}
73
74SmallVector<Metadata *, 4>
75LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
76 ArrayRef<Metadata *> LoopProperties,
77 bool &HasUserTransforms) {
78 LLVMContext &Ctx = Header->getContext();
79
80 std::optional<bool> Enabled;
81 if (Attrs.UnrollEnable == LoopAttributes::Disable)
82 Enabled = false;
83 else if (Attrs.UnrollEnable == LoopAttributes::Full)
84 Enabled = std::nullopt;
85 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
86 Attrs.UnrollCount != 0)
87 Enabled = true;
88
89 if (Enabled != true) {
90 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
91 // if unrolling is disabled.
92 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
93 }
94
95 SmallVector<Metadata *, 4> FollowupLoopProperties;
96
97 // Apply all loop properties to the unrolled loop.
98 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
99
100 // Don't unroll an already unrolled loop.
101 FollowupLoopProperties.push_back(
102 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
103
104 bool FollowupHasTransforms = false;
105 SmallVector<Metadata *, 4> Followup = createPipeliningMetadata(
106 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
107
108 SmallVector<Metadata *, 4> Args;
109 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
110
111 // Setting unroll.count
112 if (Attrs.UnrollCount > 0) {
113 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"),
114 ConstantAsMetadata::get(C: ConstantInt::get(
115 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.UnrollCount))};
116 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
117 }
118
119 // Setting unroll.full or unroll.disable
120 if (Attrs.UnrollEnable == LoopAttributes::Enable) {
121 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")};
122 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
123 }
124
125 if (FollowupHasTransforms)
126 Args.push_back(
127 Elt: createFollowupMetadata(FollowupName: "llvm.loop.unroll.followup_all", LoopProperties: Followup));
128
129 HasUserTransforms = true;
130 return Args;
131}
132
133SmallVector<Metadata *, 4>
134LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
135 ArrayRef<Metadata *> LoopProperties,
136 bool &HasUserTransforms) {
137 LLVMContext &Ctx = Header->getContext();
138
139 std::optional<bool> Enabled;
140 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
141 Enabled = false;
142 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
143 Attrs.UnrollAndJamCount != 0)
144 Enabled = true;
145
146 if (Enabled != true) {
147 SmallVector<Metadata *, 4> NewLoopProperties;
148 if (Enabled == false) {
149 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
150 NewLoopProperties.push_back(Elt: MDNode::get(
151 Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
152 LoopProperties = NewLoopProperties;
153 }
154 return createPartialUnrollMetadata(Attrs, LoopProperties,
155 HasUserTransforms);
156 }
157
158 SmallVector<Metadata *, 4> FollowupLoopProperties;
159 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
160 FollowupLoopProperties.push_back(
161 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
162
163 bool FollowupHasTransforms = false;
164 SmallVector<Metadata *, 4> Followup = createPartialUnrollMetadata(
165 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
166
167 SmallVector<Metadata *, 4> Args;
168 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
169
170 // Setting unroll_and_jam.count
171 if (Attrs.UnrollAndJamCount > 0) {
172 Metadata *Vals[] = {
173 MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.count"),
174 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
175 V: Attrs.UnrollAndJamCount))};
176 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
177 }
178
179 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
180 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.enable")};
181 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
182 }
183
184 if (FollowupHasTransforms)
185 Args.push_back(Elt: createFollowupMetadata(
186 FollowupName: "llvm.loop.unroll_and_jam.followup_outer", LoopProperties: Followup));
187
188 if (UnrollAndJamInnerFollowup.has_value())
189 Args.push_back(Elt: createFollowupMetadata(
190 FollowupName: "llvm.loop.unroll_and_jam.followup_inner", LoopProperties: *UnrollAndJamInnerFollowup));
191
192 HasUserTransforms = true;
193 return Args;
194}
195
196SmallVector<Metadata *, 4>
197LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
198 ArrayRef<Metadata *> LoopProperties,
199 bool &HasUserTransforms) {
200 LLVMContext &Ctx = Header->getContext();
201
202 std::optional<bool> Enabled;
203 if (Attrs.VectorizeEnable == LoopAttributes::Disable)
204 Enabled = false;
205 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
206 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
207 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
208 Attrs.VectorizeScalable != LoopAttributes::Unspecified)
209 Enabled = true;
210
211 if (Enabled != true) {
212 SmallVector<Metadata *, 4> NewLoopProperties;
213 if (Enabled == false) {
214 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
215 NewLoopProperties.push_back(
216 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
217 ConstantAsMetadata::get(C: ConstantInt::get(
218 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
219 LoopProperties = NewLoopProperties;
220 }
221 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
222 }
223
224 // Apply all loop properties to the vectorized loop.
225 SmallVector<Metadata *, 4> FollowupLoopProperties;
226 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
227
228 // Don't vectorize an already vectorized loop.
229 FollowupLoopProperties.push_back(
230 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
231
232 bool FollowupHasTransforms = false;
233 SmallVector<Metadata *, 4> Followup = createUnrollAndJamMetadata(
234 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
235
236 SmallVector<Metadata *, 4> Args;
237 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
238
239 // Setting vectorize.predicate when it has been specified and vectorization
240 // has not been disabled.
241 bool IsVectorPredicateEnabled = false;
242 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
243 IsVectorPredicateEnabled =
244 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
245
246 Metadata *Vals[] = {
247 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.predicate.enable"),
248 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx),
249 V: IsVectorPredicateEnabled))};
250 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
251 }
252
253 // Setting vectorize.width
254 if (Attrs.VectorizeWidth > 0) {
255 Metadata *Vals[] = {
256 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
257 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
258 V: Attrs.VectorizeWidth))};
259
260 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
261 }
262
263 if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
264 bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
265 Metadata *Vals[] = {
266 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.scalable.enable"),
267 ConstantAsMetadata::get(
268 C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx), V: IsScalable))};
269 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
270 }
271
272 // Setting interleave.count
273 if (Attrs.InterleaveCount > 0) {
274 Metadata *Vals[] = {
275 MDString::get(Context&: Ctx, Str: "llvm.loop.interleave.count"),
276 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
277 V: Attrs.InterleaveCount))};
278 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
279 }
280
281 // vectorize.enable is set if:
282 // 1) loop hint vectorize.enable is set, or
283 // 2) it is implied when vectorize.predicate is set, or
284 // 3) it is implied when vectorize.width is set to a value > 1
285 // 4) it is implied when vectorize.scalable.enable is true
286 // 5) it is implied when vectorize.width is unset (0) and the user
287 // explicitly requested fixed-width vectorization, i.e.
288 // vectorize.scalable.enable is false.
289 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
290 (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
291 Attrs.VectorizeWidth > 1 ||
292 Attrs.VectorizeScalable == LoopAttributes::Enable ||
293 (Attrs.VectorizeScalable == LoopAttributes::Disable &&
294 Attrs.VectorizeWidth != 1)) {
295 bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
296 Args.push_back(
297 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
298 ConstantAsMetadata::get(C: ConstantInt::get(
299 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: AttrVal))}));
300 }
301
302 if (FollowupHasTransforms)
303 Args.push_back(
304 Elt: createFollowupMetadata(FollowupName: "llvm.loop.vectorize.followup_all", LoopProperties: Followup));
305
306 HasUserTransforms = true;
307 return Args;
308}
309
310SmallVector<Metadata *, 4>
311LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
312 ArrayRef<Metadata *> LoopProperties,
313 bool &HasUserTransforms) {
314 LLVMContext &Ctx = Header->getContext();
315
316 std::optional<bool> Enabled;
317 if (Attrs.DistributeEnable == LoopAttributes::Disable)
318 Enabled = false;
319 if (Attrs.DistributeEnable == LoopAttributes::Enable)
320 Enabled = true;
321
322 if (Enabled != true) {
323 SmallVector<Metadata *, 4> NewLoopProperties;
324 if (Enabled == false) {
325 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
326 NewLoopProperties.push_back(
327 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
328 ConstantAsMetadata::get(C: ConstantInt::get(
329 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
330 LoopProperties = NewLoopProperties;
331 }
332 return createLoopVectorizeMetadata(Attrs, LoopProperties,
333 HasUserTransforms);
334 }
335
336 bool FollowupHasTransforms = false;
337 SmallVector<Metadata *, 4> Followup =
338 createLoopVectorizeMetadata(Attrs, LoopProperties, HasUserTransforms&: FollowupHasTransforms);
339
340 SmallVector<Metadata *, 4> Args;
341 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
342
343 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
344 ConstantAsMetadata::get(C: ConstantInt::get(
345 Ty: llvm::Type::getInt1Ty(C&: Ctx),
346 V: (Attrs.DistributeEnable == LoopAttributes::Enable)))};
347 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
348
349 if (FollowupHasTransforms)
350 Args.push_back(
351 Elt: createFollowupMetadata(FollowupName: "llvm.loop.distribute.followup_all", LoopProperties: Followup));
352
353 HasUserTransforms = true;
354 return Args;
355}
356
357SmallVector<Metadata *, 4>
358LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
359 ArrayRef<Metadata *> LoopProperties,
360 bool &HasUserTransforms) {
361 LLVMContext &Ctx = Header->getContext();
362
363 std::optional<bool> Enabled;
364 if (Attrs.UnrollEnable == LoopAttributes::Disable)
365 Enabled = false;
366 else if (Attrs.UnrollEnable == LoopAttributes::Full)
367 Enabled = true;
368
369 if (Enabled != true) {
370 SmallVector<Metadata *, 4> NewLoopProperties;
371 if (Enabled == false) {
372 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
373 NewLoopProperties.push_back(
374 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
375 LoopProperties = NewLoopProperties;
376 }
377 return createLoopDistributeMetadata(Attrs, LoopProperties,
378 HasUserTransforms);
379 }
380
381 SmallVector<Metadata *, 4> Args;
382 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
383 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full")));
384
385 // No follow-up: there is no loop after full unrolling.
386 // TODO: Warn if there are transformations after full unrolling.
387
388 HasUserTransforms = true;
389 return Args;
390}
391
392SmallVector<Metadata *, 4> LoopInfo::createMetadata(
393 const LoopAttributes &Attrs,
394 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
395 bool &HasUserTransforms) {
396 SmallVector<Metadata *, 3> LoopProperties;
397
398 // If we have a valid start debug location for the loop, add it.
399 if (StartLoc) {
400 LoopProperties.push_back(Elt: StartLoc.getAsMDNode());
401
402 // If we also have a valid end debug location for the loop, add it.
403 if (EndLoc)
404 LoopProperties.push_back(Elt: EndLoc.getAsMDNode());
405 }
406
407 LLVMContext &Ctx = Header->getContext();
408 if (Attrs.MustProgress)
409 LoopProperties.push_back(
410 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.mustprogress")));
411
412 assert(!!AccGroup == Attrs.IsParallel &&
413 "There must be an access group iff the loop is parallel");
414 if (Attrs.IsParallel) {
415 LoopProperties.push_back(Elt: MDNode::get(
416 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccGroup}));
417 }
418
419 // Setting clang::code_align attribute.
420 if (Attrs.CodeAlign > 0) {
421 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.align"),
422 ConstantAsMetadata::get(C: ConstantInt::get(
423 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.CodeAlign))};
424 LoopProperties.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
425 }
426
427 llvm::append_range(C&: LoopProperties, R&: AdditionalLoopProperties);
428 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
429}
430
431LoopAttributes::LoopAttributes(bool IsParallel)
432 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
433 UnrollEnable(LoopAttributes::Unspecified),
434 UnrollAndJamEnable(LoopAttributes::Unspecified),
435 VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
436 VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
437 UnrollCount(0), UnrollAndJamCount(0),
438 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
439 PipelineInitiationInterval(0), CodeAlign(0), MustProgress(false) {}
440
441void LoopAttributes::clear() {
442 IsParallel = false;
443 VectorizeWidth = 0;
444 VectorizeScalable = LoopAttributes::Unspecified;
445 InterleaveCount = 0;
446 UnrollCount = 0;
447 UnrollAndJamCount = 0;
448 VectorizeEnable = LoopAttributes::Unspecified;
449 UnrollEnable = LoopAttributes::Unspecified;
450 UnrollAndJamEnable = LoopAttributes::Unspecified;
451 VectorizePredicateEnable = LoopAttributes::Unspecified;
452 DistributeEnable = LoopAttributes::Unspecified;
453 PipelineDisabled = false;
454 PipelineInitiationInterval = 0;
455 CodeAlign = 0;
456 MustProgress = false;
457}
458
459LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
460 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
461 LoopInfo *Parent)
462 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
463 Parent(Parent) {
464
465 if (Attrs.IsParallel) {
466 // Create an access group for this loop.
467 LLVMContext &Ctx = Header->getContext();
468 AccGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
469 }
470
471 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
472 Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
473 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
474 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
475 Attrs.PipelineInitiationInterval == 0 &&
476 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
477 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
478 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
479 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
480 Attrs.DistributeEnable == LoopAttributes::Unspecified &&
481 Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress)
482 return;
483
484 TempLoopID = MDNode::getTemporary(Context&: Header->getContext(), MDs: {});
485}
486
487void LoopInfo::finish() {
488 // We did not annotate the loop body instructions because there are no
489 // attributes for this loop.
490 if (!TempLoopID)
491 return;
492
493 MDNode *LoopID;
494 LoopAttributes CurLoopAttr = Attrs;
495 LLVMContext &Ctx = Header->getContext();
496
497 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
498 Parent->Attrs.UnrollAndJamCount != 0)) {
499 // Parent unroll-and-jams this loop.
500 // Split the transformations in those that happens before the unroll-and-jam
501 // and those after.
502
503 LoopAttributes BeforeJam, AfterJam;
504
505 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
506
507 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
508 BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
509 BeforeJam.InterleaveCount = Attrs.InterleaveCount;
510 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
511 BeforeJam.DistributeEnable = Attrs.DistributeEnable;
512 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
513
514 switch (Attrs.UnrollEnable) {
515 case LoopAttributes::Unspecified:
516 case LoopAttributes::Disable:
517 BeforeJam.UnrollEnable = Attrs.UnrollEnable;
518 AfterJam.UnrollEnable = Attrs.UnrollEnable;
519 break;
520 case LoopAttributes::Full:
521 BeforeJam.UnrollEnable = LoopAttributes::Full;
522 break;
523 case LoopAttributes::Enable:
524 AfterJam.UnrollEnable = LoopAttributes::Enable;
525 break;
526 }
527
528 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
529 AfterJam.UnrollCount = Attrs.UnrollCount;
530 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
531 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
532
533 // If this loop is subject of an unroll-and-jam by the parent loop, and has
534 // an unroll-and-jam annotation itself, we have to decide whether to first
535 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
536 // UnrollAndJam pass processes loops from inner to outer, so we apply the
537 // inner first.
538 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
539 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
540
541 // Set the inner followup metadata to process by the outer loop. Only
542 // consider the first inner loop.
543 if (!Parent->UnrollAndJamInnerFollowup) {
544 // Splitting the attributes into a BeforeJam and an AfterJam part will
545 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
546 // to be forwarded to the AfterJam part. We detect the situation here and
547 // add it manually.
548 SmallVector<Metadata *, 1> BeforeLoopProperties;
549 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
550 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
551 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
552 BeforeJam.VectorizeScalable == LoopAttributes::Enable)
553 BeforeLoopProperties.push_back(
554 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
555
556 bool InnerFollowupHasTransform = false;
557 SmallVector<Metadata *, 4> InnerFollowup = createMetadata(
558 Attrs: AfterJam, AdditionalLoopProperties: BeforeLoopProperties, HasUserTransforms&: InnerFollowupHasTransform);
559 if (InnerFollowupHasTransform)
560 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
561 }
562
563 CurLoopAttr = BeforeJam;
564 }
565
566 bool HasUserTransforms = false;
567 SmallVector<Metadata *, 4> Properties =
568 createMetadata(Attrs: CurLoopAttr, AdditionalLoopProperties: {}, HasUserTransforms);
569 SmallVector<Metadata *, 4> Args;
570 Args.push_back(Elt: nullptr);
571 Args.append(in_start: Properties.begin(), in_end: Properties.end());
572 LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
573 LoopID->replaceOperandWith(I: 0, New: LoopID);
574
575 TempLoopID->replaceAllUsesWith(MD: LoopID);
576}
577
578void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
579 const llvm::DebugLoc &EndLoc) {
580 Active.emplace_back(
581 Args: new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
582 Active.empty() ? nullptr : Active.back().get()));
583 // Clear the attributes so nested loops do not inherit them.
584 StagedAttrs.clear();
585}
586
587void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
588 const clang::CodeGenOptions &CGOpts,
589 ArrayRef<const clang::Attr *> Attrs,
590 const llvm::DebugLoc &StartLoc,
591 const llvm::DebugLoc &EndLoc, bool MustProgress) {
592 // Identify loop hint attributes from Attrs.
593 for (const auto *Attr : Attrs) {
594 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Val: Attr);
595 const OpenCLUnrollHintAttr *OpenCLHint =
596 dyn_cast<OpenCLUnrollHintAttr>(Val: Attr);
597 const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Val: Attr);
598 // Skip non loop hint attributes
599 if (!LH && !OpenCLHint && !HLSLLoopHint) {
600 continue;
601 }
602
603 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
604 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
605 unsigned ValueInt = 1;
606 // Translate opencl_unroll_hint attribute argument to
607 // equivalent LoopHintAttr enums.
608 // OpenCL v2.0 s6.11.5:
609 // 0 - enable unroll (no argument).
610 // 1 - disable unroll.
611 // other positive integer n - unroll by n.
612 if (OpenCLHint) {
613 ValueInt = OpenCLHint->getUnrollHint();
614 if (ValueInt == 0) {
615 State = LoopHintAttr::Enable;
616 } else if (ValueInt != 1) {
617 Option = LoopHintAttr::UnrollCount;
618 State = LoopHintAttr::Numeric;
619 }
620 } else if (HLSLLoopHint) {
621 ValueInt = HLSLLoopHint->getDirective();
622 if (HLSLLoopHint->getSemanticSpelling() ==
623 HLSLLoopHintAttr::Spelling::Microsoft_unroll) {
624 if (ValueInt == 0)
625 State = LoopHintAttr::Enable;
626 if (ValueInt > 0) {
627 Option = LoopHintAttr::UnrollCount;
628 State = LoopHintAttr::Numeric;
629 }
630 }
631 } else if (LH) {
632 auto *ValueExpr = LH->getValue();
633 if (ValueExpr) {
634 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
635 ValueInt = ValueAPS.getSExtValue();
636 }
637
638 Option = LH->getOption();
639 State = LH->getState();
640 }
641 switch (State) {
642 case LoopHintAttr::Disable:
643 switch (Option) {
644 case LoopHintAttr::Vectorize:
645 // Disable vectorization by specifying a width of 1.
646 setVectorizeWidth(1);
647 setVectorizeScalable(LoopAttributes::Unspecified);
648 break;
649 case LoopHintAttr::Interleave:
650 // Disable interleaving by speciyfing a count of 1.
651 setInterleaveCount(1);
652 break;
653 case LoopHintAttr::Unroll:
654 setUnrollState(LoopAttributes::Disable);
655 break;
656 case LoopHintAttr::UnrollAndJam:
657 setUnrollAndJamState(LoopAttributes::Disable);
658 break;
659 case LoopHintAttr::VectorizePredicate:
660 setVectorizePredicateState(LoopAttributes::Disable);
661 break;
662 case LoopHintAttr::Distribute:
663 setDistributeState(false);
664 break;
665 case LoopHintAttr::PipelineDisabled:
666 setPipelineDisabled(true);
667 break;
668 case LoopHintAttr::UnrollCount:
669 case LoopHintAttr::UnrollAndJamCount:
670 case LoopHintAttr::VectorizeWidth:
671 case LoopHintAttr::InterleaveCount:
672 case LoopHintAttr::PipelineInitiationInterval:
673 llvm_unreachable("Options cannot be disabled.");
674 break;
675 }
676 break;
677 case LoopHintAttr::Enable:
678 switch (Option) {
679 case LoopHintAttr::Vectorize:
680 case LoopHintAttr::Interleave:
681 setVectorizeEnable(true);
682 break;
683 case LoopHintAttr::Unroll:
684 setUnrollState(LoopAttributes::Enable);
685 break;
686 case LoopHintAttr::UnrollAndJam:
687 setUnrollAndJamState(LoopAttributes::Enable);
688 break;
689 case LoopHintAttr::VectorizePredicate:
690 setVectorizePredicateState(LoopAttributes::Enable);
691 break;
692 case LoopHintAttr::Distribute:
693 setDistributeState(true);
694 break;
695 case LoopHintAttr::UnrollCount:
696 case LoopHintAttr::UnrollAndJamCount:
697 case LoopHintAttr::VectorizeWidth:
698 case LoopHintAttr::InterleaveCount:
699 case LoopHintAttr::PipelineDisabled:
700 case LoopHintAttr::PipelineInitiationInterval:
701 llvm_unreachable("Options cannot enabled.");
702 break;
703 }
704 break;
705 case LoopHintAttr::AssumeSafety:
706 switch (Option) {
707 case LoopHintAttr::Vectorize:
708 case LoopHintAttr::Interleave:
709 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
710 setParallel(true);
711 setVectorizeEnable(true);
712 break;
713 case LoopHintAttr::Unroll:
714 case LoopHintAttr::UnrollAndJam:
715 case LoopHintAttr::VectorizePredicate:
716 case LoopHintAttr::UnrollCount:
717 case LoopHintAttr::UnrollAndJamCount:
718 case LoopHintAttr::VectorizeWidth:
719 case LoopHintAttr::InterleaveCount:
720 case LoopHintAttr::Distribute:
721 case LoopHintAttr::PipelineDisabled:
722 case LoopHintAttr::PipelineInitiationInterval:
723 llvm_unreachable("Options cannot be used to assume mem safety.");
724 break;
725 }
726 break;
727 case LoopHintAttr::Full:
728 switch (Option) {
729 case LoopHintAttr::Unroll:
730 setUnrollState(LoopAttributes::Full);
731 break;
732 case LoopHintAttr::UnrollAndJam:
733 setUnrollAndJamState(LoopAttributes::Full);
734 break;
735 case LoopHintAttr::Vectorize:
736 case LoopHintAttr::Interleave:
737 case LoopHintAttr::UnrollCount:
738 case LoopHintAttr::UnrollAndJamCount:
739 case LoopHintAttr::VectorizeWidth:
740 case LoopHintAttr::InterleaveCount:
741 case LoopHintAttr::Distribute:
742 case LoopHintAttr::PipelineDisabled:
743 case LoopHintAttr::PipelineInitiationInterval:
744 case LoopHintAttr::VectorizePredicate:
745 llvm_unreachable("Options cannot be used with 'full' hint.");
746 break;
747 }
748 break;
749 case LoopHintAttr::FixedWidth:
750 case LoopHintAttr::ScalableWidth:
751 switch (Option) {
752 case LoopHintAttr::VectorizeWidth:
753 setVectorizeScalable(State == LoopHintAttr::ScalableWidth
754 ? LoopAttributes::Enable
755 : LoopAttributes::Disable);
756 if (LH->getValue())
757 setVectorizeWidth(ValueInt);
758 break;
759 default:
760 llvm_unreachable("Options cannot be used with 'scalable' hint.");
761 break;
762 }
763 break;
764 case LoopHintAttr::Numeric:
765 switch (Option) {
766 case LoopHintAttr::InterleaveCount:
767 setInterleaveCount(ValueInt);
768 break;
769 case LoopHintAttr::UnrollCount:
770 setUnrollCount(ValueInt);
771 break;
772 case LoopHintAttr::UnrollAndJamCount:
773 setUnrollAndJamCount(ValueInt);
774 break;
775 case LoopHintAttr::PipelineInitiationInterval:
776 setPipelineInitiationInterval(ValueInt);
777 break;
778 case LoopHintAttr::Unroll:
779 case LoopHintAttr::UnrollAndJam:
780 case LoopHintAttr::VectorizePredicate:
781 case LoopHintAttr::Vectorize:
782 case LoopHintAttr::VectorizeWidth:
783 case LoopHintAttr::Interleave:
784 case LoopHintAttr::Distribute:
785 case LoopHintAttr::PipelineDisabled:
786 llvm_unreachable("Options cannot be assigned a value.");
787 break;
788 }
789 break;
790 }
791 }
792
793 // Identify loop attribute 'code_align' from Attrs.
794 // For attribute code_align:
795 // n - 'llvm.loop.align i32 n' metadata will be emitted.
796 if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(container: Attrs)) {
797 const auto *CE = cast<ConstantExpr>(Val: CodeAlign->getAlignment());
798 llvm::APSInt ArgVal = CE->getResultAsAPSInt();
799 setCodeAlign(ArgVal.getSExtValue());
800 }
801
802 setMustProgress(MustProgress);
803
804 if (CGOpts.OptimizationLevel > 0)
805 // Disable unrolling for the loop, if unrolling is disabled (via
806 // -fno-unroll-loops) and no pragmas override the decision.
807 if (!CGOpts.UnrollLoops &&
808 (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
809 StagedAttrs.UnrollCount == 0))
810 setUnrollState(LoopAttributes::Disable);
811
812 /// Stage the attributes.
813 push(Header, StartLoc, EndLoc);
814}
815
816void LoopInfoStack::pop() {
817 assert(!Active.empty() && "No active loops to pop");
818 Active.back()->finish();
819 Active.pop_back();
820}
821
822void LoopInfoStack::InsertHelper(Instruction *I) const {
823 if (I->mayReadOrWriteMemory()) {
824 SmallVector<Metadata *, 4> AccessGroups;
825 for (const auto &AL : Active) {
826 // Here we assume that every loop that has an access group is parallel.
827 if (MDNode *Group = AL->getAccessGroup())
828 AccessGroups.push_back(Elt: Group);
829 }
830 MDNode *UnionMD = nullptr;
831 if (AccessGroups.size() == 1)
832 UnionMD = cast<MDNode>(Val: AccessGroups[0]);
833 else if (AccessGroups.size() >= 2)
834 UnionMD = MDNode::get(Context&: I->getContext(), MDs: AccessGroups);
835 I->setMetadata(Kind: "llvm.access.group", Node: UnionMD);
836 }
837
838 if (!hasInfo())
839 return;
840
841 const LoopInfo &L = getInfo();
842 if (!L.getLoopID())
843 return;
844
845 if (I->isTerminator()) {
846 for (BasicBlock *Succ : successors(I))
847 if (Succ == L.getHeader()) {
848 I->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: L.getLoopID());
849 break;
850 }
851 return;
852 }
853}
854