1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "clang/AST/Expr.h"
13#include "clang/Basic/CodeGenOptions.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
20#include <optional>
21using namespace clang::CodeGen;
22using namespace llvm;
23
24MDNode *
25LoopInfo::createFollowupMetadata(const char *FollowupName,
26 ArrayRef<llvm::Metadata *> LoopProperties) {
27 LLVMContext &Ctx = Header->getContext();
28
29 SmallVector<Metadata *, 4> Args;
30 Args.push_back(Elt: MDString::get(Context&: Ctx, Str: FollowupName));
31 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
32 return MDNode::get(Context&: Ctx, MDs: Args);
33}
34
35SmallVector<Metadata *, 4>
36LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37 ArrayRef<Metadata *> LoopProperties,
38 bool &HasUserTransforms) {
39 LLVMContext &Ctx = Header->getContext();
40
41 std::optional<bool> Enabled;
42 if (Attrs.PipelineDisabled)
43 Enabled = false;
44 else if (Attrs.PipelineInitiationInterval != 0)
45 Enabled = true;
46
47 SmallVector<Metadata *, 4> Args;
48 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
49
50 if (Enabled != true) {
51 if (Enabled == false) {
52 Args.push_back(
53 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.disable"),
54 ConstantAsMetadata::get(C: ConstantInt::get(
55 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 1))}));
56 }
57 return Args;
58 }
59
60 if (Attrs.PipelineInitiationInterval > 0) {
61 Metadata *Vals[] = {
62 MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.initiationinterval"),
63 ConstantAsMetadata::get(C: ConstantInt::get(
64 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.PipelineInitiationInterval))};
65 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
66 }
67
68 // No follow-up: This is the last transformation.
69
70 HasUserTransforms = true;
71 return Args;
72}
73
74SmallVector<Metadata *, 4>
75LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
76 ArrayRef<Metadata *> LoopProperties,
77 bool &HasUserTransforms) {
78 LLVMContext &Ctx = Header->getContext();
79
80 std::optional<bool> Enabled;
81 if (Attrs.UnrollEnable == LoopAttributes::Disable)
82 Enabled = false;
83 else if (Attrs.UnrollEnable == LoopAttributes::Full)
84 Enabled = std::nullopt;
85 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
86 Attrs.UnrollCount != 0)
87 Enabled = true;
88
89 if (Enabled != true) {
90 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
91 // if unrolling is disabled.
92 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
93 }
94
95 SmallVector<Metadata *, 4> FollowupLoopProperties;
96
97 // Apply all loop properties to the unrolled loop.
98 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
99
100 // Don't unroll an already unrolled loop.
101 FollowupLoopProperties.push_back(
102 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
103
104 bool FollowupHasTransforms = false;
105 SmallVector<Metadata *, 4> Followup = createPipeliningMetadata(
106 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
107
108 SmallVector<Metadata *, 4> Args;
109 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
110
111 // Setting unroll.count
112 if (Attrs.UnrollCount > 0) {
113 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"),
114 ConstantAsMetadata::get(C: ConstantInt::get(
115 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.UnrollCount))};
116 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
117 }
118
119 // Setting unroll.full or unroll.disable
120 if (Attrs.UnrollEnable == LoopAttributes::Enable) {
121 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")};
122 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
123 }
124
125 if (FollowupHasTransforms)
126 Args.push_back(
127 Elt: createFollowupMetadata(FollowupName: "llvm.loop.unroll.followup_all", LoopProperties: Followup));
128
129 HasUserTransforms = true;
130 return Args;
131}
132
133SmallVector<Metadata *, 4>
134LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
135 ArrayRef<Metadata *> LoopProperties,
136 bool &HasUserTransforms) {
137 LLVMContext &Ctx = Header->getContext();
138
139 std::optional<bool> Enabled;
140 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
141 Enabled = false;
142 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
143 Attrs.UnrollAndJamCount != 0)
144 Enabled = true;
145
146 if (Enabled != true) {
147 SmallVector<Metadata *, 4> NewLoopProperties;
148 if (Enabled == false) {
149 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
150 NewLoopProperties.push_back(Elt: MDNode::get(
151 Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
152 LoopProperties = NewLoopProperties;
153 }
154 return createPartialUnrollMetadata(Attrs, LoopProperties,
155 HasUserTransforms);
156 }
157
158 SmallVector<Metadata *, 4> FollowupLoopProperties;
159 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
160 FollowupLoopProperties.push_back(
161 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
162
163 bool FollowupHasTransforms = false;
164 SmallVector<Metadata *, 4> Followup = createPartialUnrollMetadata(
165 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
166
167 SmallVector<Metadata *, 4> Args;
168 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
169
170 // Setting unroll_and_jam.count
171 if (Attrs.UnrollAndJamCount > 0) {
172 Metadata *Vals[] = {
173 MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.count"),
174 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
175 V: Attrs.UnrollAndJamCount))};
176 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
177 }
178
179 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
180 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.enable")};
181 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
182 }
183
184 if (FollowupHasTransforms)
185 Args.push_back(Elt: createFollowupMetadata(
186 FollowupName: "llvm.loop.unroll_and_jam.followup_outer", LoopProperties: Followup));
187
188 if (UnrollAndJamInnerFollowup.has_value())
189 Args.push_back(Elt: createFollowupMetadata(
190 FollowupName: "llvm.loop.unroll_and_jam.followup_inner", LoopProperties: *UnrollAndJamInnerFollowup));
191
192 HasUserTransforms = true;
193 return Args;
194}
195
196SmallVector<Metadata *, 4>
197LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
198 ArrayRef<Metadata *> LoopProperties,
199 bool &HasUserTransforms) {
200 LLVMContext &Ctx = Header->getContext();
201
202 std::optional<bool> Enabled;
203 if (Attrs.VectorizeEnable == LoopAttributes::Disable)
204 Enabled = false;
205 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
206 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
207 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
208 Attrs.VectorizeScalable != LoopAttributes::Unspecified)
209 Enabled = true;
210
211 if (Enabled != true) {
212 SmallVector<Metadata *, 4> NewLoopProperties;
213 if (Enabled == false) {
214 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
215 NewLoopProperties.push_back(
216 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
217 ConstantAsMetadata::get(C: ConstantInt::get(
218 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
219 LoopProperties = NewLoopProperties;
220 }
221 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
222 }
223
224 SmallVector<Metadata *, 4> Args;
225 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
226
227 // Setting vectorize.predicate when it has been specified and vectorization
228 // has not been disabled.
229 bool IsVectorPredicateEnabled = false;
230 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
231 IsVectorPredicateEnabled =
232 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
233
234 Metadata *Vals[] = {
235 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.predicate.enable"),
236 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx),
237 V: IsVectorPredicateEnabled))};
238 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
239 }
240
241 // Setting vectorize.width
242 if (Attrs.VectorizeWidth > 0) {
243 Metadata *Vals[] = {
244 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
245 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
246 V: Attrs.VectorizeWidth))};
247
248 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
249 }
250
251 if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
252 bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
253 Metadata *Vals[] = {
254 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.scalable.enable"),
255 ConstantAsMetadata::get(
256 C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx), V: IsScalable))};
257 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
258 }
259
260 // Setting interleave.count
261 if (Attrs.InterleaveCount > 0) {
262 Metadata *Vals[] = {
263 MDString::get(Context&: Ctx, Str: "llvm.loop.interleave.count"),
264 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
265 V: Attrs.InterleaveCount))};
266 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
267 }
268
269 // vectorize.enable is set if:
270 // 1) loop hint vectorize.enable is set, or
271 // 2) it is implied when vectorize.predicate is set, or
272 // 3) it is implied when vectorize.width is set to a value > 1
273 // 4) it is implied when vectorize.scalable.enable is true
274 // 5) it is implied when vectorize.width is unset (0) and the user
275 // explicitly requested fixed-width vectorization, i.e.
276 // vectorize.scalable.enable is false.
277 bool VectorizeEnabled = false;
278 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
279 (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
280 Attrs.VectorizeWidth > 1 ||
281 Attrs.VectorizeScalable == LoopAttributes::Enable ||
282 (Attrs.VectorizeScalable == LoopAttributes::Disable &&
283 Attrs.VectorizeWidth != 1)) {
284 VectorizeEnabled = Attrs.VectorizeEnable != LoopAttributes::Disable;
285 Args.push_back(
286 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
287 ConstantAsMetadata::get(C: ConstantInt::get(
288 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: VectorizeEnabled))}));
289 }
290
291 // Apply all loop properties to the vectorized loop.
292 SmallVector<Metadata *, 4> FollowupLoopProperties;
293
294 // If vectorization is not explicitly enabled, the follow-up metadata will be
295 // directly appended to the list currently being created. In that case, adding
296 // LoopProperties to FollowupLoopProperties would result in duplication.
297 if (VectorizeEnabled)
298 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
299
300 // Don't vectorize an already vectorized loop.
301 FollowupLoopProperties.push_back(
302 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
303
304 bool FollowupHasTransforms = false;
305 SmallVector<Metadata *, 4> Followup = createUnrollAndJamMetadata(
306 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
307
308 if (FollowupHasTransforms) {
309 // If vectorization is explicitly enabled, we create a follow-up metadata,
310 // otherwise directly add the contents of it to Args.
311 if (VectorizeEnabled)
312 Args.push_back(
313 Elt: createFollowupMetadata(FollowupName: "llvm.loop.vectorize.followup_all", LoopProperties: Followup));
314 else
315 Args.append(in_start: Followup.begin(), in_end: Followup.end());
316 }
317
318 HasUserTransforms = true;
319 return Args;
320}
321
322SmallVector<Metadata *, 4>
323LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
324 ArrayRef<Metadata *> LoopProperties,
325 bool &HasUserTransforms) {
326 LLVMContext &Ctx = Header->getContext();
327
328 std::optional<bool> Enabled;
329 if (Attrs.DistributeEnable == LoopAttributes::Disable)
330 Enabled = false;
331 if (Attrs.DistributeEnable == LoopAttributes::Enable)
332 Enabled = true;
333
334 if (Enabled != true) {
335 SmallVector<Metadata *, 4> NewLoopProperties;
336 if (Enabled == false) {
337 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
338 NewLoopProperties.push_back(
339 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
340 ConstantAsMetadata::get(C: ConstantInt::get(
341 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
342 LoopProperties = NewLoopProperties;
343 }
344 return createLoopVectorizeMetadata(Attrs, LoopProperties,
345 HasUserTransforms);
346 }
347
348 bool FollowupHasTransforms = false;
349 SmallVector<Metadata *, 4> Followup =
350 createLoopVectorizeMetadata(Attrs, LoopProperties, HasUserTransforms&: FollowupHasTransforms);
351
352 SmallVector<Metadata *, 4> Args;
353 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
354
355 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
356 ConstantAsMetadata::get(C: ConstantInt::get(
357 Ty: llvm::Type::getInt1Ty(C&: Ctx),
358 V: (Attrs.DistributeEnable == LoopAttributes::Enable)))};
359 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
360
361 if (FollowupHasTransforms)
362 Args.push_back(
363 Elt: createFollowupMetadata(FollowupName: "llvm.loop.distribute.followup_all", LoopProperties: Followup));
364
365 HasUserTransforms = true;
366 return Args;
367}
368
369SmallVector<Metadata *, 4>
370LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
371 ArrayRef<Metadata *> LoopProperties,
372 bool &HasUserTransforms) {
373 LLVMContext &Ctx = Header->getContext();
374
375 std::optional<bool> Enabled;
376 if (Attrs.UnrollEnable == LoopAttributes::Disable)
377 Enabled = false;
378 else if (Attrs.UnrollEnable == LoopAttributes::Full)
379 Enabled = true;
380
381 if (Enabled != true) {
382 SmallVector<Metadata *, 4> NewLoopProperties;
383 if (Enabled == false) {
384 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
385 NewLoopProperties.push_back(
386 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
387 LoopProperties = NewLoopProperties;
388 }
389 return createLoopDistributeMetadata(Attrs, LoopProperties,
390 HasUserTransforms);
391 }
392
393 SmallVector<Metadata *, 4> Args;
394 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
395 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full")));
396
397 // No follow-up: there is no loop after full unrolling.
398 // TODO: Warn if there are transformations after full unrolling.
399
400 HasUserTransforms = true;
401 return Args;
402}
403
404SmallVector<Metadata *, 4> LoopInfo::createMetadata(
405 const LoopAttributes &Attrs,
406 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
407 bool &HasUserTransforms) {
408 SmallVector<Metadata *, 3> LoopProperties;
409
410 // If we have a valid start debug location for the loop, add it.
411 if (StartLoc) {
412 LoopProperties.push_back(Elt: StartLoc.getAsMDNode());
413
414 // If we also have a valid end debug location for the loop, add it.
415 if (EndLoc)
416 LoopProperties.push_back(Elt: EndLoc.getAsMDNode());
417 }
418
419 LLVMContext &Ctx = Header->getContext();
420 if (Attrs.MustProgress)
421 LoopProperties.push_back(
422 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.mustprogress")));
423
424 if (Attrs.LICMDisabled)
425 LoopProperties.push_back(
426 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.licm.disable")));
427
428 assert(!!AccGroup == Attrs.IsParallel &&
429 "There must be an access group iff the loop is parallel");
430 if (Attrs.IsParallel) {
431 LoopProperties.push_back(Elt: MDNode::get(
432 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccGroup}));
433 }
434
435 // Setting clang::code_align attribute.
436 if (Attrs.CodeAlign > 0) {
437 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.align"),
438 ConstantAsMetadata::get(C: ConstantInt::get(
439 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.CodeAlign))};
440 LoopProperties.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
441 }
442
443 llvm::append_range(C&: LoopProperties, R&: AdditionalLoopProperties);
444 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
445}
446
447LoopAttributes::LoopAttributes(bool IsParallel)
448 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
449 UnrollEnable(LoopAttributes::Unspecified),
450 UnrollAndJamEnable(LoopAttributes::Unspecified),
451 VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
452 VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
453 UnrollCount(0), UnrollAndJamCount(0),
454 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
455 LICMDisabled(false), PipelineInitiationInterval(0), CodeAlign(0),
456 MustProgress(false) {}
457
458void LoopAttributes::clear() {
459 IsParallel = false;
460 VectorizeWidth = 0;
461 VectorizeScalable = LoopAttributes::Unspecified;
462 InterleaveCount = 0;
463 UnrollCount = 0;
464 UnrollAndJamCount = 0;
465 VectorizeEnable = LoopAttributes::Unspecified;
466 UnrollEnable = LoopAttributes::Unspecified;
467 UnrollAndJamEnable = LoopAttributes::Unspecified;
468 VectorizePredicateEnable = LoopAttributes::Unspecified;
469 DistributeEnable = LoopAttributes::Unspecified;
470 PipelineDisabled = false;
471 LICMDisabled = false;
472 PipelineInitiationInterval = 0;
473 CodeAlign = 0;
474 MustProgress = false;
475}
476
477LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
478 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
479 LoopInfo *Parent)
480 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
481 Parent(Parent) {
482
483 if (Attrs.IsParallel) {
484 // Create an access group for this loop.
485 LLVMContext &Ctx = Header->getContext();
486 AccGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
487 }
488
489 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
490 Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
491 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
492 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
493 !Attrs.LICMDisabled && Attrs.PipelineInitiationInterval == 0 &&
494 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
495 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
496 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
497 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
498 Attrs.DistributeEnable == LoopAttributes::Unspecified &&
499 Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress)
500 return;
501
502 TempLoopID = MDNode::getTemporary(Context&: Header->getContext(), MDs: {});
503}
504
505void LoopInfo::finish() {
506 // We did not annotate the loop body instructions because there are no
507 // attributes for this loop.
508 if (!TempLoopID)
509 return;
510
511 MDNode *LoopID;
512 LoopAttributes CurLoopAttr = Attrs;
513 LLVMContext &Ctx = Header->getContext();
514
515 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
516 Parent->Attrs.UnrollAndJamCount != 0)) {
517 // Parent unroll-and-jams this loop.
518 // Split the transformations in those that happens before the unroll-and-jam
519 // and those after.
520
521 LoopAttributes BeforeJam, AfterJam;
522
523 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
524
525 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
526 BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
527 BeforeJam.InterleaveCount = Attrs.InterleaveCount;
528 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
529 BeforeJam.DistributeEnable = Attrs.DistributeEnable;
530 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
531 BeforeJam.LICMDisabled = Attrs.LICMDisabled;
532
533 switch (Attrs.UnrollEnable) {
534 case LoopAttributes::Unspecified:
535 case LoopAttributes::Disable:
536 BeforeJam.UnrollEnable = Attrs.UnrollEnable;
537 AfterJam.UnrollEnable = Attrs.UnrollEnable;
538 break;
539 case LoopAttributes::Full:
540 BeforeJam.UnrollEnable = LoopAttributes::Full;
541 break;
542 case LoopAttributes::Enable:
543 AfterJam.UnrollEnable = LoopAttributes::Enable;
544 break;
545 }
546
547 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
548 AfterJam.UnrollCount = Attrs.UnrollCount;
549 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
550 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
551
552 // If this loop is subject of an unroll-and-jam by the parent loop, and has
553 // an unroll-and-jam annotation itself, we have to decide whether to first
554 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
555 // UnrollAndJam pass processes loops from inner to outer, so we apply the
556 // inner first.
557 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
558 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
559
560 // Set the inner followup metadata to process by the outer loop. Only
561 // consider the first inner loop.
562 if (!Parent->UnrollAndJamInnerFollowup) {
563 // Splitting the attributes into a BeforeJam and an AfterJam part will
564 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
565 // to be forwarded to the AfterJam part. We detect the situation here and
566 // add it manually.
567 SmallVector<Metadata *, 1> BeforeLoopProperties;
568 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
569 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
570 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
571 BeforeJam.VectorizeScalable == LoopAttributes::Enable)
572 BeforeLoopProperties.push_back(
573 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
574
575 bool InnerFollowupHasTransform = false;
576 SmallVector<Metadata *, 4> InnerFollowup = createMetadata(
577 Attrs: AfterJam, AdditionalLoopProperties: BeforeLoopProperties, HasUserTransforms&: InnerFollowupHasTransform);
578 if (InnerFollowupHasTransform)
579 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
580 }
581
582 CurLoopAttr = BeforeJam;
583 }
584
585 bool HasUserTransforms = false;
586 SmallVector<Metadata *, 4> Properties =
587 createMetadata(Attrs: CurLoopAttr, AdditionalLoopProperties: {}, HasUserTransforms);
588 SmallVector<Metadata *, 4> Args;
589 Args.push_back(Elt: nullptr);
590 Args.append(in_start: Properties.begin(), in_end: Properties.end());
591 LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
592 LoopID->replaceOperandWith(I: 0, New: LoopID);
593
594 TempLoopID->replaceAllUsesWith(MD: LoopID);
595}
596
597void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
598 const llvm::DebugLoc &EndLoc) {
599 Active.emplace_back(
600 Args: new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
601 Active.empty() ? nullptr : Active.back().get()));
602 // Clear the attributes so nested loops do not inherit them.
603 StagedAttrs.clear();
604}
605
606void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
607 const clang::CodeGenOptions &CGOpts,
608 ArrayRef<const clang::Attr *> Attrs,
609 const llvm::DebugLoc &StartLoc,
610 const llvm::DebugLoc &EndLoc, bool MustProgress) {
611 // Identify loop hint attributes from Attrs.
612 for (const auto *Attr : Attrs) {
613 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Val: Attr);
614 const OpenCLUnrollHintAttr *OpenCLHint =
615 dyn_cast<OpenCLUnrollHintAttr>(Val: Attr);
616 const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Val: Attr);
617 // Skip non loop hint attributes
618 if (!LH && !OpenCLHint && !HLSLLoopHint) {
619 continue;
620 }
621
622 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
623 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
624 unsigned ValueInt = 1;
625 // Translate opencl_unroll_hint attribute argument to
626 // equivalent LoopHintAttr enums.
627 // OpenCL v2.0 s6.11.5:
628 // 0 - enable unroll (no argument).
629 // 1 - disable unroll.
630 // other positive integer n - unroll by n.
631 if (OpenCLHint) {
632 ValueInt = OpenCLHint->getUnrollHint();
633 if (ValueInt == 0) {
634 State = LoopHintAttr::Enable;
635 } else if (ValueInt != 1) {
636 Option = LoopHintAttr::UnrollCount;
637 State = LoopHintAttr::Numeric;
638 }
639 } else if (HLSLLoopHint) {
640 ValueInt = HLSLLoopHint->getDirective();
641 if (HLSLLoopHint->getSemanticSpelling() ==
642 HLSLLoopHintAttr::Spelling::Microsoft_unroll) {
643 if (ValueInt == 0)
644 State = LoopHintAttr::Enable;
645 if (ValueInt > 0) {
646 Option = LoopHintAttr::UnrollCount;
647 State = LoopHintAttr::Numeric;
648 }
649 }
650 } else if (LH) {
651 auto *ValueExpr = LH->getValue();
652 if (ValueExpr) {
653 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
654 ValueInt = ValueAPS.getSExtValue();
655 }
656
657 Option = LH->getOption();
658 State = LH->getState();
659 }
660 switch (State) {
661 case LoopHintAttr::Disable:
662 switch (Option) {
663 case LoopHintAttr::Vectorize:
664 // Disable vectorization by specifying a width of 1.
665 setVectorizeWidth(1);
666 setVectorizeScalable(LoopAttributes::Unspecified);
667 break;
668 case LoopHintAttr::Interleave:
669 // Disable interleaving by speciyfing a count of 1.
670 setInterleaveCount(1);
671 break;
672 case LoopHintAttr::Unroll:
673 setUnrollState(LoopAttributes::Disable);
674 break;
675 case LoopHintAttr::UnrollAndJam:
676 setUnrollAndJamState(LoopAttributes::Disable);
677 break;
678 case LoopHintAttr::VectorizePredicate:
679 setVectorizePredicateState(LoopAttributes::Disable);
680 break;
681 case LoopHintAttr::Distribute:
682 setDistributeState(false);
683 break;
684 case LoopHintAttr::PipelineDisabled:
685 setPipelineDisabled(true);
686 break;
687 case LoopHintAttr::LICMDisabled:
688 setLICMDisabled(true);
689 break;
690 case LoopHintAttr::UnrollCount:
691 case LoopHintAttr::UnrollAndJamCount:
692 case LoopHintAttr::VectorizeWidth:
693 case LoopHintAttr::InterleaveCount:
694 case LoopHintAttr::PipelineInitiationInterval:
695 llvm_unreachable("Options cannot be disabled.");
696 break;
697 }
698 break;
699 case LoopHintAttr::Enable:
700 switch (Option) {
701 case LoopHintAttr::Vectorize:
702 case LoopHintAttr::Interleave:
703 setVectorizeEnable(true);
704 break;
705 case LoopHintAttr::Unroll:
706 setUnrollState(LoopAttributes::Enable);
707 break;
708 case LoopHintAttr::UnrollAndJam:
709 setUnrollAndJamState(LoopAttributes::Enable);
710 break;
711 case LoopHintAttr::VectorizePredicate:
712 setVectorizePredicateState(LoopAttributes::Enable);
713 break;
714 case LoopHintAttr::Distribute:
715 setDistributeState(true);
716 break;
717 case LoopHintAttr::UnrollCount:
718 case LoopHintAttr::UnrollAndJamCount:
719 case LoopHintAttr::VectorizeWidth:
720 case LoopHintAttr::InterleaveCount:
721 case LoopHintAttr::PipelineDisabled:
722 case LoopHintAttr::PipelineInitiationInterval:
723 case LoopHintAttr::LICMDisabled:
724 llvm_unreachable("Options cannot enabled.");
725 break;
726 }
727 break;
728 case LoopHintAttr::AssumeSafety:
729 switch (Option) {
730 case LoopHintAttr::Vectorize:
731 case LoopHintAttr::Interleave:
732 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
733 setParallel(true);
734 setVectorizeEnable(true);
735 break;
736 case LoopHintAttr::Unroll:
737 case LoopHintAttr::UnrollAndJam:
738 case LoopHintAttr::VectorizePredicate:
739 case LoopHintAttr::UnrollCount:
740 case LoopHintAttr::UnrollAndJamCount:
741 case LoopHintAttr::VectorizeWidth:
742 case LoopHintAttr::InterleaveCount:
743 case LoopHintAttr::Distribute:
744 case LoopHintAttr::PipelineDisabled:
745 case LoopHintAttr::PipelineInitiationInterval:
746 case LoopHintAttr::LICMDisabled:
747 llvm_unreachable("Options cannot be used to assume mem safety.");
748 break;
749 }
750 break;
751 case LoopHintAttr::Full:
752 switch (Option) {
753 case LoopHintAttr::Unroll:
754 setUnrollState(LoopAttributes::Full);
755 break;
756 case LoopHintAttr::UnrollAndJam:
757 setUnrollAndJamState(LoopAttributes::Full);
758 break;
759 case LoopHintAttr::Vectorize:
760 case LoopHintAttr::Interleave:
761 case LoopHintAttr::UnrollCount:
762 case LoopHintAttr::UnrollAndJamCount:
763 case LoopHintAttr::VectorizeWidth:
764 case LoopHintAttr::InterleaveCount:
765 case LoopHintAttr::Distribute:
766 case LoopHintAttr::PipelineDisabled:
767 case LoopHintAttr::PipelineInitiationInterval:
768 case LoopHintAttr::VectorizePredicate:
769 case LoopHintAttr::LICMDisabled:
770 llvm_unreachable("Options cannot be used with 'full' hint.");
771 break;
772 }
773 break;
774 case LoopHintAttr::FixedWidth:
775 case LoopHintAttr::ScalableWidth:
776 switch (Option) {
777 case LoopHintAttr::VectorizeWidth:
778 setVectorizeScalable(State == LoopHintAttr::ScalableWidth
779 ? LoopAttributes::Enable
780 : LoopAttributes::Disable);
781 if (LH->getValue())
782 setVectorizeWidth(ValueInt);
783 break;
784 default:
785 llvm_unreachable("Options cannot be used with 'scalable' hint.");
786 break;
787 }
788 break;
789 case LoopHintAttr::Numeric:
790 switch (Option) {
791 case LoopHintAttr::InterleaveCount:
792 setInterleaveCount(ValueInt);
793 break;
794 case LoopHintAttr::UnrollCount:
795 setUnrollCount(ValueInt);
796 break;
797 case LoopHintAttr::UnrollAndJamCount:
798 setUnrollAndJamCount(ValueInt);
799 break;
800 case LoopHintAttr::PipelineInitiationInterval:
801 setPipelineInitiationInterval(ValueInt);
802 break;
803 case LoopHintAttr::Unroll:
804 case LoopHintAttr::UnrollAndJam:
805 case LoopHintAttr::VectorizePredicate:
806 case LoopHintAttr::Vectorize:
807 case LoopHintAttr::VectorizeWidth:
808 case LoopHintAttr::Interleave:
809 case LoopHintAttr::Distribute:
810 case LoopHintAttr::PipelineDisabled:
811 case LoopHintAttr::LICMDisabled:
812 llvm_unreachable("Options cannot be assigned a value.");
813 break;
814 }
815 break;
816 }
817 }
818
819 // Identify loop attribute 'code_align' from Attrs.
820 // For attribute code_align:
821 // n - 'llvm.loop.align i32 n' metadata will be emitted.
822 if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(container: Attrs)) {
823 const auto *CE = cast<ConstantExpr>(Val: CodeAlign->getAlignment());
824 llvm::APSInt ArgVal = CE->getResultAsAPSInt();
825 setCodeAlign(ArgVal.getSExtValue());
826 }
827
828 setMustProgress(MustProgress);
829
830 if (CGOpts.OptimizationLevel > 0)
831 // Disable unrolling for the loop, if unrolling is disabled (via
832 // -fno-unroll-loops) and no pragmas override the decision.
833 if (!CGOpts.UnrollLoops &&
834 (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
835 StagedAttrs.UnrollCount == 0))
836 setUnrollState(LoopAttributes::Disable);
837
838 /// Stage the attributes.
839 push(Header, StartLoc, EndLoc);
840}
841
842void LoopInfoStack::pop() {
843 assert(!Active.empty() && "No active loops to pop");
844 Active.back()->finish();
845 Active.pop_back();
846}
847
848void LoopInfoStack::InsertHelper(Instruction *I) const {
849 if (I->mayReadOrWriteMemory()) {
850 SmallVector<Metadata *, 4> AccessGroups;
851 for (const auto &AL : Active) {
852 // Here we assume that every loop that has an access group is parallel.
853 if (MDNode *Group = AL->getAccessGroup())
854 AccessGroups.push_back(Elt: Group);
855 }
856 MDNode *UnionMD = nullptr;
857 if (AccessGroups.size() == 1)
858 UnionMD = cast<MDNode>(Val: AccessGroups[0]);
859 else if (AccessGroups.size() >= 2)
860 UnionMD = MDNode::get(Context&: I->getContext(), MDs: AccessGroups);
861 I->setMetadata(Kind: "llvm.access.group", Node: UnionMD);
862 }
863
864 if (!hasInfo())
865 return;
866
867 const LoopInfo &L = getInfo();
868 if (!L.getLoopID())
869 return;
870
871 if (I->isTerminator()) {
872 for (BasicBlock *Succ : successors(I))
873 if (Succ == L.getHeader()) {
874 I->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: L.getLoopID());
875 break;
876 }
877 return;
878 }
879}
880