1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "clang/AST/Expr.h"
13#include "clang/Basic/CodeGenOptions.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
20#include <optional>
21using namespace clang::CodeGen;
22using namespace llvm;
23
24MDNode *clang::CodeGen::LoopInfo::createFollowupMetadata(
25 const char *FollowupName, ArrayRef<llvm::Metadata *> LoopProperties) {
26 LLVMContext &Ctx = Header->getContext();
27
28 SmallVector<Metadata *, 4> Args;
29 Args.push_back(Elt: MDString::get(Context&: Ctx, Str: FollowupName));
30 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
31 return MDNode::get(Context&: Ctx, MDs: Args);
32}
33
34SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createPipeliningMetadata(
35 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
36 bool &HasUserTransforms) {
37 LLVMContext &Ctx = Header->getContext();
38
39 std::optional<bool> Enabled;
40 if (Attrs.PipelineDisabled)
41 Enabled = false;
42 else if (Attrs.PipelineInitiationInterval != 0)
43 Enabled = true;
44
45 SmallVector<Metadata *, 4> Args;
46 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
47
48 if (Enabled != true) {
49 if (Enabled == false) {
50 Args.push_back(
51 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.disable"),
52 ConstantAsMetadata::get(C: ConstantInt::get(
53 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 1))}));
54 }
55 return Args;
56 }
57
58 if (Attrs.PipelineInitiationInterval > 0) {
59 Metadata *Vals[] = {
60 MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.initiationinterval"),
61 ConstantAsMetadata::get(C: ConstantInt::get(
62 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.PipelineInitiationInterval))};
63 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
64 }
65
66 // No follow-up: This is the last transformation.
67
68 HasUserTransforms = true;
69 return Args;
70}
71
72SmallVector<Metadata *, 4>
73clang::CodeGen::LoopInfo::createPartialUnrollMetadata(
74 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
75 bool &HasUserTransforms) {
76 LLVMContext &Ctx = Header->getContext();
77
78 std::optional<bool> Enabled;
79 if (Attrs.UnrollEnable == LoopAttributes::Disable)
80 Enabled = false;
81 else if (Attrs.UnrollEnable == LoopAttributes::Full)
82 Enabled = std::nullopt;
83 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
84 Attrs.UnrollCount != 0)
85 Enabled = true;
86
87 if (Enabled != true) {
88 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
89 // if unrolling is disabled.
90 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
91 }
92
93 SmallVector<Metadata *, 4> FollowupLoopProperties;
94
95 // Apply all loop properties to the unrolled loop.
96 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
97
98 // Don't unroll an already unrolled loop.
99 FollowupLoopProperties.push_back(
100 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
101
102 bool FollowupHasTransforms = false;
103 SmallVector<Metadata *, 4> Followup = createPipeliningMetadata(
104 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
105
106 SmallVector<Metadata *, 4> Args;
107 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
108
109 // Setting unroll.count
110 if (Attrs.UnrollCount > 0) {
111 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"),
112 ConstantAsMetadata::get(C: ConstantInt::get(
113 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.UnrollCount))};
114 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
115 }
116
117 // Setting unroll.full or unroll.disable
118 if (Attrs.UnrollEnable == LoopAttributes::Enable) {
119 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")};
120 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
121 }
122
123 if (FollowupHasTransforms)
124 Args.push_back(
125 Elt: createFollowupMetadata(FollowupName: "llvm.loop.unroll.followup_all", LoopProperties: Followup));
126
127 HasUserTransforms = true;
128 return Args;
129}
130
131SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createUnrollAndJamMetadata(
132 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
133 bool &HasUserTransforms) {
134 LLVMContext &Ctx = Header->getContext();
135
136 std::optional<bool> Enabled;
137 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
138 Enabled = false;
139 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
140 Attrs.UnrollAndJamCount != 0)
141 Enabled = true;
142
143 if (Enabled != true) {
144 SmallVector<Metadata *, 4> NewLoopProperties;
145 if (Enabled == false) {
146 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
147 NewLoopProperties.push_back(Elt: MDNode::get(
148 Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
149 LoopProperties = NewLoopProperties;
150 }
151 return createPartialUnrollMetadata(Attrs, LoopProperties,
152 HasUserTransforms);
153 }
154
155 SmallVector<Metadata *, 4> FollowupLoopProperties;
156 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
157 FollowupLoopProperties.push_back(
158 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
159
160 bool FollowupHasTransforms = false;
161 SmallVector<Metadata *, 4> Followup = createPartialUnrollMetadata(
162 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
163
164 SmallVector<Metadata *, 4> Args;
165 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
166
167 // Setting unroll_and_jam.count
168 if (Attrs.UnrollAndJamCount > 0) {
169 Metadata *Vals[] = {
170 MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.count"),
171 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
172 V: Attrs.UnrollAndJamCount))};
173 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
174 }
175
176 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
177 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.enable")};
178 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
179 }
180
181 if (FollowupHasTransforms)
182 Args.push_back(Elt: createFollowupMetadata(
183 FollowupName: "llvm.loop.unroll_and_jam.followup_outer", LoopProperties: Followup));
184
185 if (UnrollAndJamInnerFollowup.has_value())
186 Args.push_back(Elt: createFollowupMetadata(
187 FollowupName: "llvm.loop.unroll_and_jam.followup_inner", LoopProperties: *UnrollAndJamInnerFollowup));
188
189 HasUserTransforms = true;
190 return Args;
191}
192
193SmallVector<Metadata *, 4>
194clang::CodeGen::LoopInfo::createLoopVectorizeMetadata(
195 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
196 bool &HasUserTransforms) {
197 LLVMContext &Ctx = Header->getContext();
198
199 std::optional<bool> Enabled;
200 if (Attrs.VectorizeEnable == LoopAttributes::Disable)
201 Enabled = false;
202 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
203 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
204 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
205 Attrs.VectorizeScalable != LoopAttributes::Unspecified)
206 Enabled = true;
207
208 if (Enabled != true) {
209 SmallVector<Metadata *, 4> NewLoopProperties;
210 if (Enabled == false) {
211 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
212 NewLoopProperties.push_back(Elt: MDNode::get(
213 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.disable")}));
214 LoopProperties = NewLoopProperties;
215 }
216 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
217 }
218
219 SmallVector<Metadata *, 4> Args;
220 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
221
222 // Setting vectorize.predicate when it has been specified and vectorization
223 // has not been disabled.
224 bool IsVectorPredicateEnabled = false;
225 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
226 IsVectorPredicateEnabled =
227 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
228
229 Args.push_back(Elt: MDNode::get(
230 Context&: Ctx,
231 MDs: {MDString::get(Context&: Ctx, Str: IsVectorPredicateEnabled
232 ? "llvm.loop.vectorize.predicate.enable"
233 : "llvm.loop.vectorize.predicate.disable")}));
234 }
235
236 // Setting vectorize.width
237 if (Attrs.VectorizeWidth > 0) {
238 Metadata *Vals[] = {
239 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
240 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
241 V: Attrs.VectorizeWidth))};
242
243 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
244 }
245
246 if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
247 bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
248 Args.push_back(Elt: MDNode::get(
249 Context&: Ctx, MDs: {MDString::get(
250 Context&: Ctx, Str: IsScalable ? "llvm.loop.vectorize.scalable.enable"
251 : "llvm.loop.vectorize.scalable.disable")}));
252 }
253
254 // Setting interleave.count
255 if (Attrs.InterleaveCount > 0) {
256 Metadata *Vals[] = {
257 MDString::get(Context&: Ctx, Str: "llvm.loop.interleave.count"),
258 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
259 V: Attrs.InterleaveCount))};
260 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
261 }
262
263 // vectorize.enable is set if:
264 // 1) loop hint vectorize.enable is set, or
265 // 2) it is implied when vectorize.predicate is set, or
266 // 3) it is implied when vectorize.width is set to a value > 1
267 // 4) it is implied when vectorize.scalable.enable is true
268 // 5) it is implied when vectorize.width is unset (0) and the user
269 // explicitly requested fixed-width vectorization, i.e.
270 // vectorize.scalable.enable is false.
271 bool VectorizeEnabled = false;
272 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
273 (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
274 Attrs.VectorizeWidth > 1 ||
275 Attrs.VectorizeScalable == LoopAttributes::Enable ||
276 (Attrs.VectorizeScalable == LoopAttributes::Disable &&
277 Attrs.VectorizeWidth != 1)) {
278 VectorizeEnabled = Attrs.VectorizeEnable != LoopAttributes::Disable;
279 Args.push_back(Elt: MDNode::get(
280 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: VectorizeEnabled
281 ? "llvm.loop.vectorize.enable"
282 : "llvm.loop.vectorize.disable")}));
283 }
284
285 // Apply all loop properties to the vectorized loop.
286 SmallVector<Metadata *, 4> FollowupLoopProperties;
287
288 // If vectorization is not explicitly enabled, the follow-up metadata will be
289 // directly appended to the list currently being created. In that case, adding
290 // LoopProperties to FollowupLoopProperties would result in duplication.
291 if (VectorizeEnabled)
292 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
293
294 // Don't vectorize an already vectorized loop.
295 FollowupLoopProperties.push_back(
296 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
297
298 bool FollowupHasTransforms = false;
299 SmallVector<Metadata *, 4> Followup = createUnrollAndJamMetadata(
300 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
301
302 if (FollowupHasTransforms) {
303 // If vectorization is explicitly enabled, we create a follow-up metadata,
304 // otherwise directly add the contents of it to Args.
305 if (VectorizeEnabled)
306 Args.push_back(
307 Elt: createFollowupMetadata(FollowupName: "llvm.loop.vectorize.followup_all", LoopProperties: Followup));
308 else
309 Args.append(in_start: Followup.begin(), in_end: Followup.end());
310 }
311
312 HasUserTransforms = true;
313 return Args;
314}
315
316SmallVector<Metadata *, 4>
317clang::CodeGen::LoopInfo::createLoopDistributeMetadata(
318 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
319 bool &HasUserTransforms) {
320 LLVMContext &Ctx = Header->getContext();
321
322 std::optional<bool> Enabled;
323 if (Attrs.DistributeEnable == LoopAttributes::Disable)
324 Enabled = false;
325 if (Attrs.DistributeEnable == LoopAttributes::Enable)
326 Enabled = true;
327
328 if (Enabled != true) {
329 SmallVector<Metadata *, 4> NewLoopProperties;
330 if (Enabled == false) {
331 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
332 NewLoopProperties.push_back(Elt: MDNode::get(
333 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.disable")}));
334 LoopProperties = NewLoopProperties;
335 }
336 return createLoopVectorizeMetadata(Attrs, LoopProperties,
337 HasUserTransforms);
338 }
339
340 bool FollowupHasTransforms = false;
341 SmallVector<Metadata *, 4> Followup =
342 createLoopVectorizeMetadata(Attrs, LoopProperties, HasUserTransforms&: FollowupHasTransforms);
343
344 SmallVector<Metadata *, 4> Args;
345 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
346
347 Args.push_back(
348 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable")}));
349
350 if (FollowupHasTransforms)
351 Args.push_back(
352 Elt: createFollowupMetadata(FollowupName: "llvm.loop.distribute.followup_all", LoopProperties: Followup));
353
354 HasUserTransforms = true;
355 return Args;
356}
357
358SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createFullUnrollMetadata(
359 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
360 bool &HasUserTransforms) {
361 LLVMContext &Ctx = Header->getContext();
362
363 std::optional<bool> Enabled;
364 if (Attrs.UnrollEnable == LoopAttributes::Disable)
365 Enabled = false;
366 else if (Attrs.UnrollEnable == LoopAttributes::Full)
367 Enabled = true;
368
369 if (Enabled != true) {
370 SmallVector<Metadata *, 4> NewLoopProperties;
371 if (Enabled == false) {
372 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
373 NewLoopProperties.push_back(
374 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
375 LoopProperties = NewLoopProperties;
376 }
377 return createLoopDistributeMetadata(Attrs, LoopProperties,
378 HasUserTransforms);
379 }
380
381 SmallVector<Metadata *, 4> Args;
382 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
383 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full")));
384
385 // No follow-up: there is no loop after full unrolling.
386 // TODO: Warn if there are transformations after full unrolling.
387
388 HasUserTransforms = true;
389 return Args;
390}
391
392SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createMetadata(
393 const LoopAttributes &Attrs,
394 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
395 bool &HasUserTransforms) {
396 SmallVector<Metadata *, 3> LoopProperties;
397
398 // If we have a valid start debug location for the loop, add it.
399 if (StartLoc) {
400 LoopProperties.push_back(Elt: StartLoc.getAsMDNode());
401
402 // If we also have a valid end debug location for the loop, add it.
403 if (EndLoc)
404 LoopProperties.push_back(Elt: EndLoc.getAsMDNode());
405 }
406
407 LLVMContext &Ctx = Header->getContext();
408 if (Attrs.MustProgress)
409 LoopProperties.push_back(
410 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.mustprogress")));
411
412 if (Attrs.LICMDisabled)
413 LoopProperties.push_back(
414 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.licm.disable")));
415
416 assert(!!AccGroup == Attrs.IsParallel &&
417 "There must be an access group iff the loop is parallel");
418 if (Attrs.IsParallel) {
419 LoopProperties.push_back(Elt: MDNode::get(
420 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccGroup}));
421 }
422
423 // Setting clang::code_align attribute.
424 if (Attrs.CodeAlign > 0) {
425 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.align"),
426 ConstantAsMetadata::get(C: ConstantInt::get(
427 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.CodeAlign))};
428 LoopProperties.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
429 }
430
431 llvm::append_range(C&: LoopProperties, R&: AdditionalLoopProperties);
432 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
433}
434
435clang::CodeGen::LoopInfo::LoopInfo(BasicBlock *Header,
436 const LoopAttributes &Attrs,
437 const llvm::DebugLoc &StartLoc,
438 const llvm::DebugLoc &EndLoc,
439 LoopInfo *Parent)
440 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
441 Parent(Parent) {
442
443 if (Attrs.IsParallel) {
444 // Create an access group for this loop.
445 LLVMContext &Ctx = Header->getContext();
446 AccGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
447 }
448
449 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
450 Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
451 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
452 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
453 !Attrs.LICMDisabled && Attrs.PipelineInitiationInterval == 0 &&
454 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
455 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
456 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
457 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
458 Attrs.DistributeEnable == LoopAttributes::Unspecified &&
459 Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress)
460 return;
461
462 TempLoopID = MDNode::getTemporary(Context&: Header->getContext(), MDs: {});
463}
464
465void clang::CodeGen::LoopInfo::finish() {
466 // We did not annotate the loop body instructions because there are no
467 // attributes for this loop.
468 if (!TempLoopID)
469 return;
470
471 MDNode *LoopID;
472 LoopAttributes CurLoopAttr = Attrs;
473 LLVMContext &Ctx = Header->getContext();
474
475 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
476 Parent->Attrs.UnrollAndJamCount != 0)) {
477 // Parent unroll-and-jams this loop.
478 // Split the transformations in those that happens before the unroll-and-jam
479 // and those after.
480
481 LoopAttributes BeforeJam, AfterJam;
482
483 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
484
485 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
486 BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
487 BeforeJam.InterleaveCount = Attrs.InterleaveCount;
488 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
489 BeforeJam.DistributeEnable = Attrs.DistributeEnable;
490 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
491 BeforeJam.LICMDisabled = Attrs.LICMDisabled;
492
493 switch (Attrs.UnrollEnable) {
494 case LoopAttributes::Unspecified:
495 case LoopAttributes::Disable:
496 BeforeJam.UnrollEnable = Attrs.UnrollEnable;
497 AfterJam.UnrollEnable = Attrs.UnrollEnable;
498 break;
499 case LoopAttributes::Full:
500 BeforeJam.UnrollEnable = LoopAttributes::Full;
501 break;
502 case LoopAttributes::Enable:
503 AfterJam.UnrollEnable = LoopAttributes::Enable;
504 break;
505 }
506
507 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
508 AfterJam.UnrollCount = Attrs.UnrollCount;
509 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
510 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
511
512 // If this loop is subject of an unroll-and-jam by the parent loop, and has
513 // an unroll-and-jam annotation itself, we have to decide whether to first
514 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
515 // UnrollAndJam pass processes loops from inner to outer, so we apply the
516 // inner first.
517 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
518 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
519
520 // Set the inner followup metadata to process by the outer loop. Only
521 // consider the first inner loop.
522 if (!Parent->UnrollAndJamInnerFollowup) {
523 // Splitting the attributes into a BeforeJam and an AfterJam part will
524 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
525 // to be forwarded to the AfterJam part. We detect the situation here and
526 // add it manually.
527 SmallVector<Metadata *, 1> BeforeLoopProperties;
528 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
529 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
530 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
531 BeforeJam.VectorizeScalable == LoopAttributes::Enable)
532 BeforeLoopProperties.push_back(
533 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
534
535 bool InnerFollowupHasTransform = false;
536 SmallVector<Metadata *, 4> InnerFollowup = createMetadata(
537 Attrs: AfterJam, AdditionalLoopProperties: BeforeLoopProperties, HasUserTransforms&: InnerFollowupHasTransform);
538 if (InnerFollowupHasTransform)
539 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
540 }
541
542 CurLoopAttr = BeforeJam;
543 }
544
545 bool HasUserTransforms = false;
546 SmallVector<Metadata *, 4> Properties =
547 createMetadata(Attrs: CurLoopAttr, AdditionalLoopProperties: {}, HasUserTransforms);
548 SmallVector<Metadata *, 4> Args;
549 Args.push_back(Elt: nullptr);
550 Args.append(in_start: Properties.begin(), in_end: Properties.end());
551 LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
552 LoopID->replaceOperandWith(I: 0, New: LoopID);
553
554 TempLoopID->replaceAllUsesWith(MD: LoopID);
555}
556
557void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
558 const llvm::DebugLoc &EndLoc) {
559 Active.emplace_back(
560 Args: new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
561 Active.empty() ? nullptr : Active.back().get()));
562 // Clear the attributes so nested loops do not inherit them.
563 StagedAttrs.clear();
564}
565
566void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
567 const clang::CodeGenOptions &CGOpts,
568 ArrayRef<const clang::Attr *> Attrs,
569 const llvm::DebugLoc &StartLoc,
570 const llvm::DebugLoc &EndLoc, bool MustProgress) {
571 /// Stage the attributes.
572 clang::CodeGenUtils::updateLoopAttributes(Attrs&: StagedAttrs, Ctx, CGOpts, StmtAttrs: Attrs,
573 MustProgress);
574 push(Header, StartLoc, EndLoc);
575}
576
577void LoopInfoStack::pop() {
578 assert(!Active.empty() && "No active loops to pop");
579 Active.back()->finish();
580 Active.pop_back();
581}
582
583void LoopInfoStack::InsertHelper(Instruction *I) const {
584 if (I->mayReadOrWriteMemory()) {
585 SmallVector<Metadata *, 4> AccessGroups;
586 for (const auto &AL : Active) {
587 // Here we assume that every loop that has an access group is parallel.
588 if (MDNode *Group = AL->getAccessGroup())
589 AccessGroups.push_back(Elt: Group);
590 }
591 MDNode *UnionMD = nullptr;
592 if (AccessGroups.size() == 1)
593 UnionMD = cast<MDNode>(Val: AccessGroups[0]);
594 else if (AccessGroups.size() >= 2)
595 UnionMD = MDNode::get(Context&: I->getContext(), MDs: AccessGroups);
596 I->setMetadata(Kind: "llvm.access.group", Node: UnionMD);
597 }
598
599 if (!hasInfo())
600 return;
601
602 const LoopInfo &L = getInfo();
603 if (!L.getLoopID())
604 return;
605
606 if (I->isTerminator()) {
607 for (BasicBlock *Succ : successors(I))
608 if (Succ == L.getHeader()) {
609 I->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: L.getLoopID());
610 break;
611 }
612 return;
613 }
614}
615