1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "clang/AST/Expr.h"
13#include "clang/Basic/CodeGenOptions.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
20#include <optional>
21using namespace clang::CodeGen;
22using namespace llvm;
23
24MDNode *clang::CodeGen::LoopInfo::createFollowupMetadata(
25 const char *FollowupName, ArrayRef<llvm::Metadata *> LoopProperties) {
26 LLVMContext &Ctx = Header->getContext();
27
28 SmallVector<Metadata *, 4> Args;
29 Args.push_back(Elt: MDString::get(Context&: Ctx, Str: FollowupName));
30 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
31 return MDNode::get(Context&: Ctx, MDs: Args);
32}
33
34SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createPipeliningMetadata(
35 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
36 bool &HasUserTransforms) {
37 LLVMContext &Ctx = Header->getContext();
38
39 std::optional<bool> Enabled;
40 if (Attrs.PipelineDisabled)
41 Enabled = false;
42 else if (Attrs.PipelineInitiationInterval != 0)
43 Enabled = true;
44
45 SmallVector<Metadata *, 4> Args;
46 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
47
48 if (Enabled != true) {
49 if (Enabled == false) {
50 Args.push_back(
51 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.disable"),
52 ConstantAsMetadata::get(C: ConstantInt::get(
53 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 1))}));
54 }
55 return Args;
56 }
57
58 if (Attrs.PipelineInitiationInterval > 0) {
59 Metadata *Vals[] = {
60 MDString::get(Context&: Ctx, Str: "llvm.loop.pipeline.initiationinterval"),
61 ConstantAsMetadata::get(C: ConstantInt::get(
62 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.PipelineInitiationInterval))};
63 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
64 }
65
66 // No follow-up: This is the last transformation.
67
68 HasUserTransforms = true;
69 return Args;
70}
71
72SmallVector<Metadata *, 4>
73clang::CodeGen::LoopInfo::createPartialUnrollMetadata(
74 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
75 bool &HasUserTransforms) {
76 LLVMContext &Ctx = Header->getContext();
77
78 std::optional<bool> Enabled;
79 if (Attrs.UnrollEnable == LoopAttributes::Disable)
80 Enabled = false;
81 else if (Attrs.UnrollEnable == LoopAttributes::Full)
82 Enabled = std::nullopt;
83 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
84 Attrs.UnrollCount != 0)
85 Enabled = true;
86
87 if (Enabled != true) {
88 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
89 // if unrolling is disabled.
90 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
91 }
92
93 SmallVector<Metadata *, 4> FollowupLoopProperties;
94
95 // Apply all loop properties to the unrolled loop.
96 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
97
98 // Don't unroll an already unrolled loop.
99 FollowupLoopProperties.push_back(
100 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
101
102 bool FollowupHasTransforms = false;
103 SmallVector<Metadata *, 4> Followup = createPipeliningMetadata(
104 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
105
106 SmallVector<Metadata *, 4> Args;
107 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
108
109 // Setting unroll.count
110 if (Attrs.UnrollCount > 0) {
111 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"),
112 ConstantAsMetadata::get(C: ConstantInt::get(
113 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.UnrollCount))};
114 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
115 }
116
117 // Setting unroll.full or unroll.disable
118 if (Attrs.UnrollEnable == LoopAttributes::Enable) {
119 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")};
120 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
121 }
122
123 if (FollowupHasTransforms)
124 Args.push_back(
125 Elt: createFollowupMetadata(FollowupName: "llvm.loop.unroll.followup_all", LoopProperties: Followup));
126
127 HasUserTransforms = true;
128 return Args;
129}
130
131SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createUnrollAndJamMetadata(
132 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
133 bool &HasUserTransforms) {
134 LLVMContext &Ctx = Header->getContext();
135
136 std::optional<bool> Enabled;
137 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
138 Enabled = false;
139 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
140 Attrs.UnrollAndJamCount != 0)
141 Enabled = true;
142
143 if (Enabled != true) {
144 SmallVector<Metadata *, 4> NewLoopProperties;
145 if (Enabled == false) {
146 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
147 NewLoopProperties.push_back(Elt: MDNode::get(
148 Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
149 LoopProperties = NewLoopProperties;
150 }
151 return createPartialUnrollMetadata(Attrs, LoopProperties,
152 HasUserTransforms);
153 }
154
155 SmallVector<Metadata *, 4> FollowupLoopProperties;
156 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
157 FollowupLoopProperties.push_back(
158 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.disable")));
159
160 bool FollowupHasTransforms = false;
161 SmallVector<Metadata *, 4> Followup = createPartialUnrollMetadata(
162 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
163
164 SmallVector<Metadata *, 4> Args;
165 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
166
167 // Setting unroll_and_jam.count
168 if (Attrs.UnrollAndJamCount > 0) {
169 Metadata *Vals[] = {
170 MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.count"),
171 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
172 V: Attrs.UnrollAndJamCount))};
173 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
174 }
175
176 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
177 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll_and_jam.enable")};
178 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
179 }
180
181 if (FollowupHasTransforms)
182 Args.push_back(Elt: createFollowupMetadata(
183 FollowupName: "llvm.loop.unroll_and_jam.followup_outer", LoopProperties: Followup));
184
185 if (UnrollAndJamInnerFollowup.has_value())
186 Args.push_back(Elt: createFollowupMetadata(
187 FollowupName: "llvm.loop.unroll_and_jam.followup_inner", LoopProperties: *UnrollAndJamInnerFollowup));
188
189 HasUserTransforms = true;
190 return Args;
191}
192
193SmallVector<Metadata *, 4>
194clang::CodeGen::LoopInfo::createLoopVectorizeMetadata(
195 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
196 bool &HasUserTransforms) {
197 LLVMContext &Ctx = Header->getContext();
198
199 std::optional<bool> Enabled;
200 if (Attrs.VectorizeEnable == LoopAttributes::Disable)
201 Enabled = false;
202 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
203 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
204 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
205 Attrs.VectorizeScalable != LoopAttributes::Unspecified)
206 Enabled = true;
207
208 if (Enabled != true) {
209 SmallVector<Metadata *, 4> NewLoopProperties;
210 if (Enabled == false) {
211 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
212 NewLoopProperties.push_back(
213 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
214 ConstantAsMetadata::get(C: ConstantInt::get(
215 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
216 LoopProperties = NewLoopProperties;
217 }
218 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
219 }
220
221 SmallVector<Metadata *, 4> Args;
222 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
223
224 // Setting vectorize.predicate when it has been specified and vectorization
225 // has not been disabled.
226 bool IsVectorPredicateEnabled = false;
227 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
228 IsVectorPredicateEnabled =
229 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
230
231 Metadata *Vals[] = {
232 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.predicate.enable"),
233 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx),
234 V: IsVectorPredicateEnabled))};
235 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
236 }
237
238 // Setting vectorize.width
239 if (Attrs.VectorizeWidth > 0) {
240 Metadata *Vals[] = {
241 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
242 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
243 V: Attrs.VectorizeWidth))};
244
245 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
246 }
247
248 if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
249 bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
250 Metadata *Vals[] = {
251 MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.scalable.enable"),
252 ConstantAsMetadata::get(
253 C: ConstantInt::get(Ty: llvm::Type::getInt1Ty(C&: Ctx), V: IsScalable))};
254 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
255 }
256
257 // Setting interleave.count
258 if (Attrs.InterleaveCount > 0) {
259 Metadata *Vals[] = {
260 MDString::get(Context&: Ctx, Str: "llvm.loop.interleave.count"),
261 ConstantAsMetadata::get(C: ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: Ctx),
262 V: Attrs.InterleaveCount))};
263 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
264 }
265
266 // vectorize.enable is set if:
267 // 1) loop hint vectorize.enable is set, or
268 // 2) it is implied when vectorize.predicate is set, or
269 // 3) it is implied when vectorize.width is set to a value > 1
270 // 4) it is implied when vectorize.scalable.enable is true
271 // 5) it is implied when vectorize.width is unset (0) and the user
272 // explicitly requested fixed-width vectorization, i.e.
273 // vectorize.scalable.enable is false.
274 bool VectorizeEnabled = false;
275 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
276 (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
277 Attrs.VectorizeWidth > 1 ||
278 Attrs.VectorizeScalable == LoopAttributes::Enable ||
279 (Attrs.VectorizeScalable == LoopAttributes::Disable &&
280 Attrs.VectorizeWidth != 1)) {
281 VectorizeEnabled = Attrs.VectorizeEnable != LoopAttributes::Disable;
282 Args.push_back(
283 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
284 ConstantAsMetadata::get(C: ConstantInt::get(
285 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: VectorizeEnabled))}));
286 }
287
288 // Apply all loop properties to the vectorized loop.
289 SmallVector<Metadata *, 4> FollowupLoopProperties;
290
291 // If vectorization is not explicitly enabled, the follow-up metadata will be
292 // directly appended to the list currently being created. In that case, adding
293 // LoopProperties to FollowupLoopProperties would result in duplication.
294 if (VectorizeEnabled)
295 FollowupLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
296
297 // Don't vectorize an already vectorized loop.
298 FollowupLoopProperties.push_back(
299 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
300
301 bool FollowupHasTransforms = false;
302 SmallVector<Metadata *, 4> Followup = createUnrollAndJamMetadata(
303 Attrs, LoopProperties: FollowupLoopProperties, HasUserTransforms&: FollowupHasTransforms);
304
305 if (FollowupHasTransforms) {
306 // If vectorization is explicitly enabled, we create a follow-up metadata,
307 // otherwise directly add the contents of it to Args.
308 if (VectorizeEnabled)
309 Args.push_back(
310 Elt: createFollowupMetadata(FollowupName: "llvm.loop.vectorize.followup_all", LoopProperties: Followup));
311 else
312 Args.append(in_start: Followup.begin(), in_end: Followup.end());
313 }
314
315 HasUserTransforms = true;
316 return Args;
317}
318
319SmallVector<Metadata *, 4>
320clang::CodeGen::LoopInfo::createLoopDistributeMetadata(
321 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
322 bool &HasUserTransforms) {
323 LLVMContext &Ctx = Header->getContext();
324
325 std::optional<bool> Enabled;
326 if (Attrs.DistributeEnable == LoopAttributes::Disable)
327 Enabled = false;
328 if (Attrs.DistributeEnable == LoopAttributes::Enable)
329 Enabled = true;
330
331 if (Enabled != true) {
332 SmallVector<Metadata *, 4> NewLoopProperties;
333 if (Enabled == false) {
334 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
335 NewLoopProperties.push_back(
336 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
337 ConstantAsMetadata::get(C: ConstantInt::get(
338 Ty: llvm::Type::getInt1Ty(C&: Ctx), V: 0))}));
339 LoopProperties = NewLoopProperties;
340 }
341 return createLoopVectorizeMetadata(Attrs, LoopProperties,
342 HasUserTransforms);
343 }
344
345 bool FollowupHasTransforms = false;
346 SmallVector<Metadata *, 4> Followup =
347 createLoopVectorizeMetadata(Attrs, LoopProperties, HasUserTransforms&: FollowupHasTransforms);
348
349 SmallVector<Metadata *, 4> Args;
350 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
351
352 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.distribute.enable"),
353 ConstantAsMetadata::get(C: ConstantInt::get(
354 Ty: llvm::Type::getInt1Ty(C&: Ctx),
355 V: (Attrs.DistributeEnable == LoopAttributes::Enable)))};
356 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
357
358 if (FollowupHasTransforms)
359 Args.push_back(
360 Elt: createFollowupMetadata(FollowupName: "llvm.loop.distribute.followup_all", LoopProperties: Followup));
361
362 HasUserTransforms = true;
363 return Args;
364}
365
366SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createFullUnrollMetadata(
367 const LoopAttributes &Attrs, ArrayRef<Metadata *> LoopProperties,
368 bool &HasUserTransforms) {
369 LLVMContext &Ctx = Header->getContext();
370
371 std::optional<bool> Enabled;
372 if (Attrs.UnrollEnable == LoopAttributes::Disable)
373 Enabled = false;
374 else if (Attrs.UnrollEnable == LoopAttributes::Full)
375 Enabled = true;
376
377 if (Enabled != true) {
378 SmallVector<Metadata *, 4> NewLoopProperties;
379 if (Enabled == false) {
380 NewLoopProperties.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
381 NewLoopProperties.push_back(
382 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.disable")));
383 LoopProperties = NewLoopProperties;
384 }
385 return createLoopDistributeMetadata(Attrs, LoopProperties,
386 HasUserTransforms);
387 }
388
389 SmallVector<Metadata *, 4> Args;
390 Args.append(in_start: LoopProperties.begin(), in_end: LoopProperties.end());
391 Args.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full")));
392
393 // No follow-up: there is no loop after full unrolling.
394 // TODO: Warn if there are transformations after full unrolling.
395
396 HasUserTransforms = true;
397 return Args;
398}
399
400SmallVector<Metadata *, 4> clang::CodeGen::LoopInfo::createMetadata(
401 const LoopAttributes &Attrs,
402 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
403 bool &HasUserTransforms) {
404 SmallVector<Metadata *, 3> LoopProperties;
405
406 // If we have a valid start debug location for the loop, add it.
407 if (StartLoc) {
408 LoopProperties.push_back(Elt: StartLoc.getAsMDNode());
409
410 // If we also have a valid end debug location for the loop, add it.
411 if (EndLoc)
412 LoopProperties.push_back(Elt: EndLoc.getAsMDNode());
413 }
414
415 LLVMContext &Ctx = Header->getContext();
416 if (Attrs.MustProgress)
417 LoopProperties.push_back(
418 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.mustprogress")));
419
420 if (Attrs.LICMDisabled)
421 LoopProperties.push_back(
422 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.licm.disable")));
423
424 assert(!!AccGroup == Attrs.IsParallel &&
425 "There must be an access group iff the loop is parallel");
426 if (Attrs.IsParallel) {
427 LoopProperties.push_back(Elt: MDNode::get(
428 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccGroup}));
429 }
430
431 // Setting clang::code_align attribute.
432 if (Attrs.CodeAlign > 0) {
433 Metadata *Vals[] = {MDString::get(Context&: Ctx, Str: "llvm.loop.align"),
434 ConstantAsMetadata::get(C: ConstantInt::get(
435 Ty: llvm::Type::getInt32Ty(C&: Ctx), V: Attrs.CodeAlign))};
436 LoopProperties.push_back(Elt: MDNode::get(Context&: Ctx, MDs: Vals));
437 }
438
439 llvm::append_range(C&: LoopProperties, R&: AdditionalLoopProperties);
440 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
441}
442
443LoopAttributes::LoopAttributes(bool IsParallel)
444 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
445 UnrollEnable(LoopAttributes::Unspecified),
446 UnrollAndJamEnable(LoopAttributes::Unspecified),
447 VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
448 VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
449 UnrollCount(0), UnrollAndJamCount(0),
450 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
451 LICMDisabled(false), PipelineInitiationInterval(0), CodeAlign(0),
452 MustProgress(false) {}
453
454void LoopAttributes::clear() {
455 IsParallel = false;
456 VectorizeWidth = 0;
457 VectorizeScalable = LoopAttributes::Unspecified;
458 InterleaveCount = 0;
459 UnrollCount = 0;
460 UnrollAndJamCount = 0;
461 VectorizeEnable = LoopAttributes::Unspecified;
462 UnrollEnable = LoopAttributes::Unspecified;
463 UnrollAndJamEnable = LoopAttributes::Unspecified;
464 VectorizePredicateEnable = LoopAttributes::Unspecified;
465 DistributeEnable = LoopAttributes::Unspecified;
466 PipelineDisabled = false;
467 LICMDisabled = false;
468 PipelineInitiationInterval = 0;
469 CodeAlign = 0;
470 MustProgress = false;
471}
472
473clang::CodeGen::LoopInfo::LoopInfo(BasicBlock *Header,
474 const LoopAttributes &Attrs,
475 const llvm::DebugLoc &StartLoc,
476 const llvm::DebugLoc &EndLoc,
477 LoopInfo *Parent)
478 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
479 Parent(Parent) {
480
481 if (Attrs.IsParallel) {
482 // Create an access group for this loop.
483 LLVMContext &Ctx = Header->getContext();
484 AccGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
485 }
486
487 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
488 Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
489 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
490 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
491 !Attrs.LICMDisabled && Attrs.PipelineInitiationInterval == 0 &&
492 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
493 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
494 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
495 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
496 Attrs.DistributeEnable == LoopAttributes::Unspecified &&
497 Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress)
498 return;
499
500 TempLoopID = MDNode::getTemporary(Context&: Header->getContext(), MDs: {});
501}
502
503void clang::CodeGen::LoopInfo::finish() {
504 // We did not annotate the loop body instructions because there are no
505 // attributes for this loop.
506 if (!TempLoopID)
507 return;
508
509 MDNode *LoopID;
510 LoopAttributes CurLoopAttr = Attrs;
511 LLVMContext &Ctx = Header->getContext();
512
513 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
514 Parent->Attrs.UnrollAndJamCount != 0)) {
515 // Parent unroll-and-jams this loop.
516 // Split the transformations in those that happens before the unroll-and-jam
517 // and those after.
518
519 LoopAttributes BeforeJam, AfterJam;
520
521 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
522
523 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
524 BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
525 BeforeJam.InterleaveCount = Attrs.InterleaveCount;
526 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
527 BeforeJam.DistributeEnable = Attrs.DistributeEnable;
528 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
529 BeforeJam.LICMDisabled = Attrs.LICMDisabled;
530
531 switch (Attrs.UnrollEnable) {
532 case LoopAttributes::Unspecified:
533 case LoopAttributes::Disable:
534 BeforeJam.UnrollEnable = Attrs.UnrollEnable;
535 AfterJam.UnrollEnable = Attrs.UnrollEnable;
536 break;
537 case LoopAttributes::Full:
538 BeforeJam.UnrollEnable = LoopAttributes::Full;
539 break;
540 case LoopAttributes::Enable:
541 AfterJam.UnrollEnable = LoopAttributes::Enable;
542 break;
543 }
544
545 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
546 AfterJam.UnrollCount = Attrs.UnrollCount;
547 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
548 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
549
550 // If this loop is subject of an unroll-and-jam by the parent loop, and has
551 // an unroll-and-jam annotation itself, we have to decide whether to first
552 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
553 // UnrollAndJam pass processes loops from inner to outer, so we apply the
554 // inner first.
555 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
556 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
557
558 // Set the inner followup metadata to process by the outer loop. Only
559 // consider the first inner loop.
560 if (!Parent->UnrollAndJamInnerFollowup) {
561 // Splitting the attributes into a BeforeJam and an AfterJam part will
562 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
563 // to be forwarded to the AfterJam part. We detect the situation here and
564 // add it manually.
565 SmallVector<Metadata *, 1> BeforeLoopProperties;
566 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
567 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
568 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
569 BeforeJam.VectorizeScalable == LoopAttributes::Enable)
570 BeforeLoopProperties.push_back(
571 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.isvectorized")));
572
573 bool InnerFollowupHasTransform = false;
574 SmallVector<Metadata *, 4> InnerFollowup = createMetadata(
575 Attrs: AfterJam, AdditionalLoopProperties: BeforeLoopProperties, HasUserTransforms&: InnerFollowupHasTransform);
576 if (InnerFollowupHasTransform)
577 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
578 }
579
580 CurLoopAttr = BeforeJam;
581 }
582
583 bool HasUserTransforms = false;
584 SmallVector<Metadata *, 4> Properties =
585 createMetadata(Attrs: CurLoopAttr, AdditionalLoopProperties: {}, HasUserTransforms);
586 SmallVector<Metadata *, 4> Args;
587 Args.push_back(Elt: nullptr);
588 Args.append(in_start: Properties.begin(), in_end: Properties.end());
589 LoopID = MDNode::getDistinct(Context&: Ctx, MDs: Args);
590 LoopID->replaceOperandWith(I: 0, New: LoopID);
591
592 TempLoopID->replaceAllUsesWith(MD: LoopID);
593}
594
595void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
596 const llvm::DebugLoc &EndLoc) {
597 Active.emplace_back(
598 Args: new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
599 Active.empty() ? nullptr : Active.back().get()));
600 // Clear the attributes so nested loops do not inherit them.
601 StagedAttrs.clear();
602}
603
604void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
605 const clang::CodeGenOptions &CGOpts,
606 ArrayRef<const clang::Attr *> Attrs,
607 const llvm::DebugLoc &StartLoc,
608 const llvm::DebugLoc &EndLoc, bool MustProgress) {
609 // Identify loop hint attributes from Attrs.
610 for (const auto *Attr : Attrs) {
611 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Val: Attr);
612 const OpenCLUnrollHintAttr *OpenCLHint =
613 dyn_cast<OpenCLUnrollHintAttr>(Val: Attr);
614 const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Val: Attr);
615 // Skip non loop hint attributes
616 if (!LH && !OpenCLHint && !HLSLLoopHint) {
617 continue;
618 }
619
620 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
621 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
622 unsigned ValueInt = 1;
623 // Translate opencl_unroll_hint attribute argument to
624 // equivalent LoopHintAttr enums.
625 // OpenCL v2.0 s6.11.5:
626 // 0 - enable unroll (no argument).
627 // 1 - disable unroll.
628 // other positive integer n - unroll by n.
629 if (OpenCLHint) {
630 ValueInt = OpenCLHint->getUnrollHint();
631 if (ValueInt == 0) {
632 State = LoopHintAttr::Enable;
633 } else if (ValueInt != 1) {
634 Option = LoopHintAttr::UnrollCount;
635 State = LoopHintAttr::Numeric;
636 }
637 } else if (HLSLLoopHint) {
638 ValueInt = HLSLLoopHint->getDirective();
639 if (HLSLLoopHint->getSemanticSpelling() ==
640 HLSLLoopHintAttr::Spelling::Microsoft_unroll) {
641 if (ValueInt == 0)
642 State = LoopHintAttr::Enable;
643 if (ValueInt > 0) {
644 Option = LoopHintAttr::UnrollCount;
645 State = LoopHintAttr::Numeric;
646 }
647 }
648 } else if (LH) {
649 auto *ValueExpr = LH->getValue();
650 if (ValueExpr) {
651 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
652 ValueInt = ValueAPS.getSExtValue();
653 }
654
655 Option = LH->getOption();
656 State = LH->getState();
657 }
658 switch (State) {
659 case LoopHintAttr::Disable:
660 switch (Option) {
661 case LoopHintAttr::Vectorize:
662 // Disable vectorization by specifying a width of 1.
663 setVectorizeWidth(1);
664 setVectorizeScalable(LoopAttributes::Unspecified);
665 break;
666 case LoopHintAttr::Interleave:
667 // Disable interleaving by speciyfing a count of 1.
668 setInterleaveCount(1);
669 break;
670 case LoopHintAttr::Unroll:
671 setUnrollState(LoopAttributes::Disable);
672 break;
673 case LoopHintAttr::UnrollAndJam:
674 setUnrollAndJamState(LoopAttributes::Disable);
675 break;
676 case LoopHintAttr::VectorizePredicate:
677 setVectorizePredicateState(LoopAttributes::Disable);
678 break;
679 case LoopHintAttr::Distribute:
680 setDistributeState(false);
681 break;
682 case LoopHintAttr::PipelineDisabled:
683 setPipelineDisabled(true);
684 break;
685 case LoopHintAttr::LICMDisabled:
686 setLICMDisabled(true);
687 break;
688 case LoopHintAttr::UnrollCount:
689 case LoopHintAttr::UnrollAndJamCount:
690 case LoopHintAttr::VectorizeWidth:
691 case LoopHintAttr::InterleaveCount:
692 case LoopHintAttr::PipelineInitiationInterval:
693 llvm_unreachable("Options cannot be disabled.");
694 break;
695 }
696 break;
697 case LoopHintAttr::Enable:
698 switch (Option) {
699 case LoopHintAttr::Vectorize:
700 case LoopHintAttr::Interleave:
701 setVectorizeEnable(true);
702 break;
703 case LoopHintAttr::Unroll:
704 setUnrollState(LoopAttributes::Enable);
705 break;
706 case LoopHintAttr::UnrollAndJam:
707 setUnrollAndJamState(LoopAttributes::Enable);
708 break;
709 case LoopHintAttr::VectorizePredicate:
710 setVectorizePredicateState(LoopAttributes::Enable);
711 break;
712 case LoopHintAttr::Distribute:
713 setDistributeState(true);
714 break;
715 case LoopHintAttr::UnrollCount:
716 case LoopHintAttr::UnrollAndJamCount:
717 case LoopHintAttr::VectorizeWidth:
718 case LoopHintAttr::InterleaveCount:
719 case LoopHintAttr::PipelineDisabled:
720 case LoopHintAttr::PipelineInitiationInterval:
721 case LoopHintAttr::LICMDisabled:
722 llvm_unreachable("Options cannot enabled.");
723 break;
724 }
725 break;
726 case LoopHintAttr::AssumeSafety:
727 switch (Option) {
728 case LoopHintAttr::Vectorize:
729 case LoopHintAttr::Interleave:
730 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
731 setParallel(true);
732 setVectorizeEnable(true);
733 break;
734 case LoopHintAttr::Unroll:
735 case LoopHintAttr::UnrollAndJam:
736 case LoopHintAttr::VectorizePredicate:
737 case LoopHintAttr::UnrollCount:
738 case LoopHintAttr::UnrollAndJamCount:
739 case LoopHintAttr::VectorizeWidth:
740 case LoopHintAttr::InterleaveCount:
741 case LoopHintAttr::Distribute:
742 case LoopHintAttr::PipelineDisabled:
743 case LoopHintAttr::PipelineInitiationInterval:
744 case LoopHintAttr::LICMDisabled:
745 llvm_unreachable("Options cannot be used to assume mem safety.");
746 break;
747 }
748 break;
749 case LoopHintAttr::Full:
750 switch (Option) {
751 case LoopHintAttr::Unroll:
752 setUnrollState(LoopAttributes::Full);
753 break;
754 case LoopHintAttr::UnrollAndJam:
755 setUnrollAndJamState(LoopAttributes::Full);
756 break;
757 case LoopHintAttr::Vectorize:
758 case LoopHintAttr::Interleave:
759 case LoopHintAttr::UnrollCount:
760 case LoopHintAttr::UnrollAndJamCount:
761 case LoopHintAttr::VectorizeWidth:
762 case LoopHintAttr::InterleaveCount:
763 case LoopHintAttr::Distribute:
764 case LoopHintAttr::PipelineDisabled:
765 case LoopHintAttr::PipelineInitiationInterval:
766 case LoopHintAttr::VectorizePredicate:
767 case LoopHintAttr::LICMDisabled:
768 llvm_unreachable("Options cannot be used with 'full' hint.");
769 break;
770 }
771 break;
772 case LoopHintAttr::FixedWidth:
773 case LoopHintAttr::ScalableWidth:
774 switch (Option) {
775 case LoopHintAttr::VectorizeWidth:
776 setVectorizeScalable(State == LoopHintAttr::ScalableWidth
777 ? LoopAttributes::Enable
778 : LoopAttributes::Disable);
779 if (LH->getValue())
780 setVectorizeWidth(ValueInt);
781 break;
782 default:
783 llvm_unreachable("Options cannot be used with 'scalable' hint.");
784 break;
785 }
786 break;
787 case LoopHintAttr::Numeric:
788 switch (Option) {
789 case LoopHintAttr::InterleaveCount:
790 setInterleaveCount(ValueInt);
791 break;
792 case LoopHintAttr::UnrollCount:
793 setUnrollCount(ValueInt);
794 break;
795 case LoopHintAttr::UnrollAndJamCount:
796 setUnrollAndJamCount(ValueInt);
797 break;
798 case LoopHintAttr::PipelineInitiationInterval:
799 setPipelineInitiationInterval(ValueInt);
800 break;
801 case LoopHintAttr::Unroll:
802 case LoopHintAttr::UnrollAndJam:
803 case LoopHintAttr::VectorizePredicate:
804 case LoopHintAttr::Vectorize:
805 case LoopHintAttr::VectorizeWidth:
806 case LoopHintAttr::Interleave:
807 case LoopHintAttr::Distribute:
808 case LoopHintAttr::PipelineDisabled:
809 case LoopHintAttr::LICMDisabled:
810 llvm_unreachable("Options cannot be assigned a value.");
811 break;
812 }
813 break;
814 }
815 }
816
817 // Identify loop attribute 'code_align' from Attrs.
818 // For attribute code_align:
819 // n - 'llvm.loop.align i32 n' metadata will be emitted.
820 if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(container: Attrs)) {
821 const auto *CE = cast<ConstantExpr>(Val: CodeAlign->getAlignment());
822 llvm::APSInt ArgVal = CE->getResultAsAPSInt();
823 setCodeAlign(ArgVal.getSExtValue());
824 }
825
826 setMustProgress(MustProgress);
827
828 if (CGOpts.OptimizationLevel > 0)
829 // Disable unrolling for the loop, if unrolling is disabled (via
830 // -fno-unroll-loops) and no pragmas override the decision.
831 if (!CGOpts.UnrollLoops &&
832 (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
833 StagedAttrs.UnrollCount == 0))
834 setUnrollState(LoopAttributes::Disable);
835
836 /// Stage the attributes.
837 push(Header, StartLoc, EndLoc);
838}
839
840void LoopInfoStack::pop() {
841 assert(!Active.empty() && "No active loops to pop");
842 Active.back()->finish();
843 Active.pop_back();
844}
845
846void LoopInfoStack::InsertHelper(Instruction *I) const {
847 if (I->mayReadOrWriteMemory()) {
848 SmallVector<Metadata *, 4> AccessGroups;
849 for (const auto &AL : Active) {
850 // Here we assume that every loop that has an access group is parallel.
851 if (MDNode *Group = AL->getAccessGroup())
852 AccessGroups.push_back(Elt: Group);
853 }
854 MDNode *UnionMD = nullptr;
855 if (AccessGroups.size() == 1)
856 UnionMD = cast<MDNode>(Val: AccessGroups[0]);
857 else if (AccessGroups.size() >= 2)
858 UnionMD = MDNode::get(Context&: I->getContext(), MDs: AccessGroups);
859 I->setMetadata(Kind: "llvm.access.group", Node: UnionMD);
860 }
861
862 if (!hasInfo())
863 return;
864
865 const LoopInfo &L = getInfo();
866 if (!L.getLoopID())
867 return;
868
869 if (I->isTerminator()) {
870 for (BasicBlock *Succ : successors(I))
871 if (Succ == L.getHeader()) {
872 I->setMetadata(KindID: llvm::LLVMContext::MD_loop, Node: L.getLoopID());
873 break;
874 }
875 return;
876 }
877}
878