1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGDebugInfo.h"
15#include "CGOpenMPRuntime.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
18#include "CodeGenPGO.h"
19#include "TargetInfo.h"
20#include "clang/AST/ASTContext.h"
21#include "clang/AST/Attr.h"
22#include "clang/AST/DeclOpenMP.h"
23#include "clang/AST/OpenMPClause.h"
24#include "clang/AST/Stmt.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/PrettyStackTrace.h"
30#include "clang/Basic/SourceManager.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/BinaryFormat/Dwarf.h"
33#include "llvm/Frontend/OpenMP/OMPConstants.h"
34#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugInfoMetadata.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
41#include <optional>
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
47
48static const VarDecl *getBaseDecl(const Expr *Ref);
49static OpenMPDirectiveKind
50getEffectiveDirectiveKind(const OMPExecutableDirective &S);
51
52/// Whether a combined `distribute parallel for` may use the fused
53/// distr_static_chunk + static_chunkone schedule (enum 93): one
54/// for_static_init, no surrounding distribute_static_init.
55static bool canEmitGPUFusedDistSchedule(const CodeGenModule &CGM,
56 const OMPLoopDirective &S,
57 OpenMPDirectiveKind DKind) {
58 // 'teams loop' is always emitted as 'distribute', and 'target teams loop'
59 // only becomes 'distribute parallel for' if canBeParallelFor() holds.
60 // Without the inner worksharing loop, there is nothing that would schedule
61 // the iteration space if the outer distribute loop is omitted.
62 if (DKind == OMPD_teams_loop)
63 return false;
64 if (const auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(Val: &S);
65 TTLD && !TTLD->canBeParallelFor())
66 return false;
67 // Reduction-only for now. Non-reduction cases might follow in the future, but
68 // need more analysis for maximum profit.
69 return CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
70 isOpenMPLoopBoundSharingDirective(Kind: DKind) &&
71 S.hasClausesOfKind<OMPReductionClause>() &&
72 !S.getSingleClause<OMPDistScheduleClause>() &&
73 !S.getSingleClause<OMPScheduleClause>() &&
74 !S.getSingleClause<OMPOrderedClause>();
75}
76
77namespace {
78/// Lexical scope for OpenMP executable constructs, that handles correct codegen
79/// for captured expressions.
80class OMPLexicalScope : public CodeGenFunction::LexicalScope {
81 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
82 for (const auto *C : S.clauses()) {
83 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
84 if (const auto *PreInit =
85 cast_or_null<DeclStmt>(Val: CPI->getPreInitStmt())) {
86 for (const auto *I : PreInit->decls()) {
87 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
88 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
89 } else {
90 CodeGenFunction::AutoVarEmission Emission =
91 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
92 CGF.EmitAutoVarCleanups(emission: Emission);
93 }
94 }
95 }
96 }
97 }
98 }
99 CodeGenFunction::OMPPrivateScope InlinedShareds;
100
101 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
102 return CGF.LambdaCaptureFields.lookup(Val: VD) ||
103 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
104 (isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl) &&
105 cast<BlockDecl>(Val: CGF.CurCodeDecl)->capturesVariable(var: VD));
106 }
107
108public:
109 OMPLexicalScope(
110 CodeGenFunction &CGF, const OMPExecutableDirective &S,
111 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
112 const bool EmitPreInitStmt = true)
113 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
114 InlinedShareds(CGF) {
115 if (EmitPreInitStmt)
116 emitPreInitStmt(CGF, S);
117 if (!CapturedRegion)
118 return;
119 assert(S.hasAssociatedStmt() &&
120 "Expected associated statement for inlined directive.");
121 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: *CapturedRegion);
122 for (const auto &C : CS->captures()) {
123 if (C.capturesVariable() || C.capturesVariableByCopy()) {
124 auto *VD = C.getCapturedVar();
125 assert(VD == VD->getCanonicalDecl() &&
126 "Canonical decl must be captured.");
127 DeclRefExpr DRE(
128 CGF.getContext(), const_cast<VarDecl *>(VD),
129 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
130 InlinedShareds.isGlobalVarCaptured(VD)),
131 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
132 InlinedShareds.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
133 }
134 }
135 (void)InlinedShareds.Privatize();
136 }
137};
138
139/// Lexical scope for OpenMP parallel construct, that handles correct codegen
140/// for captured expressions.
141class OMPParallelScope final : public OMPLexicalScope {
142 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
143 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
144 return !(isOpenMPTargetExecutionDirective(DKind: EKind) ||
145 isOpenMPLoopBoundSharingDirective(Kind: EKind)) &&
146 isOpenMPParallelDirective(DKind: EKind);
147 }
148
149public:
150 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
151 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
152 EmitPreInitStmt(S)) {}
153};
154
155/// Lexical scope for OpenMP teams construct, that handles correct codegen
156/// for captured expressions.
157class OMPTeamsScope final : public OMPLexicalScope {
158 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
159 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
160 return !isOpenMPTargetExecutionDirective(DKind: EKind) &&
161 isOpenMPTeamsDirective(DKind: EKind);
162 }
163
164public:
165 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
166 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
167 EmitPreInitStmt(S)) {}
168};
169
170/// Private scope for OpenMP loop-based directives, that supports capturing
171/// of used expression from loop statement.
172class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
173 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
174 const Stmt *PreInits;
175 CodeGenFunction::OMPMapVars PreCondVars;
176 SmallVector<const Decl *, 4> DoacrossOnlyDecls;
177 if (auto *LD = dyn_cast<OMPLoopDirective>(Val: &S)) {
178 // Emit init, __range, __begin and __end variables for C++ range loops.
179 (void)OMPLoopBasedDirective::doForAllLoops(
180 CurStmt: LD->getInnermostCapturedStmt()->getCapturedStmt(),
181 /*TryImperfectlyNestedLoops=*/true, NumLoops: LD->getLoopsNumber(),
182 Callback: [&CGF](unsigned Cnt, const Stmt *CurStmt) {
183 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(Val: CurStmt)) {
184 if (const Stmt *Init = CXXFor->getInit())
185 CGF.EmitStmt(S: Init);
186 CGF.EmitStmt(S: CXXFor->getRangeStmt());
187 CGF.EmitStmt(S: CXXFor->getBeginStmt());
188 CGF.EmitStmt(S: CXXFor->getEndStmt());
189 }
190 return false;
191 });
192 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
193 for (const auto *E : LD->counters()) {
194 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
195 EmittedAsPrivate.insert(V: VD->getCanonicalDecl());
196 (void)PreCondVars.setVarAddr(
197 CGF, LocalVD: VD, TempAddr: CGF.CreateMemTemp(T: VD->getType().getNonReferenceType()));
198 }
199 // Mark private vars as undefs.
200 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
201 for (const Expr *IRef : C->varlist()) {
202 const auto *OrigDecl = cast<DeclRefExpr>(Val: IRef)->getDecl();
203 const auto *OrigVD = dyn_cast<VarDecl>(Val: OrigDecl);
204 if (!OrigVD)
205 continue;
206 if (EmittedAsPrivate.insert(V: OrigVD->getCanonicalDecl()).second) {
207 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
208 (void)PreCondVars.setVarAddr(
209 CGF, LocalVD: OrigVD,
210 TempAddr: Address(llvm::UndefValue::get(T: CGF.ConvertTypeForMem(
211 T: CGF.getContext().getPointerType(T: OrigVDTy))),
212 CGF.ConvertTypeForMem(T: OrigVDTy),
213 CGF.getContext().getDeclAlign(D: OrigVD)));
214 }
215 }
216 }
217 (void)PreCondVars.apply(CGF);
218 // The pre-inits also use the bounds of the loops that an 'ordered(n)'
219 // clause counts beyond the collapsed ones. Such a loop is emitted as a
220 // regular loop in the body and declares its own helpers, so these copies
221 // only live until the pre-inits.
222 if (const auto *C = LD->getSingleClause<OMPOrderedClause>())
223 (void)OMPLoopBasedDirective::doForAllLoops(
224 CurStmt: LD->getInnermostCapturedStmt()->getCapturedStmt(),
225 /*TryImperfectlyNestedLoops=*/true,
226 NumLoops: C->getLoopNumIterations().size(),
227 Callback: [&CGF, LD, &DoacrossOnlyDecls](unsigned Cnt, const Stmt *CurStmt) {
228 const auto *CXXFor = dyn_cast<CXXForRangeStmt>(Val: CurStmt);
229 if (!CXXFor || Cnt < LD->getLoopsNumber())
230 return false;
231 for (const Stmt *Helper : std::initializer_list<const Stmt *>{
232 CXXFor->getInit(), CXXFor->getRangeStmt(),
233 CXXFor->getBeginStmt(), CXXFor->getEndStmt()}) {
234 if (!Helper)
235 continue;
236 CGF.EmitStmt(S: Helper);
237 if (const auto *DS = dyn_cast<DeclStmt>(Val: Helper))
238 llvm::append_range(C&: DoacrossOnlyDecls, R: DS->decls());
239 }
240 return false;
241 });
242 PreInits = LD->getPreInits();
243 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(Val: &S)) {
244 PreInits = Tile->getPreInits();
245 } else if (const auto *Stripe = dyn_cast<OMPStripeDirective>(Val: &S)) {
246 PreInits = Stripe->getPreInits();
247 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(Val: &S)) {
248 PreInits = Unroll->getPreInits();
249 } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(Val: &S)) {
250 PreInits = Reverse->getPreInits();
251 } else if (const auto *Split = dyn_cast<OMPSplitDirective>(Val: &S)) {
252 PreInits = Split->getPreInits();
253 } else if (const auto *Interchange =
254 dyn_cast<OMPInterchangeDirective>(Val: &S)) {
255 PreInits = Interchange->getPreInits();
256 } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(Val: &S)) {
257 PreInits = Flatten->getPreInits();
258 } else {
259 llvm_unreachable("Unknown loop-based directive kind.");
260 }
261 doEmitPreinits(PreInits);
262 for (const Decl *D : DoacrossOnlyDecls)
263 CGF.eraseLocalDecl(D);
264 PreCondVars.restore(CGF);
265 }
266
267 void
268 emitPreInitStmt(CodeGenFunction &CGF,
269 const OMPCanonicalLoopSequenceTransformationDirective &S) {
270 const Stmt *PreInits;
271 if (const auto *Fuse = dyn_cast<OMPFuseDirective>(Val: &S)) {
272 PreInits = Fuse->getPreInits();
273 } else {
274 llvm_unreachable(
275 "Unknown canonical loop sequence transform directive kind.");
276 }
277 doEmitPreinits(PreInits);
278 }
279
280 void doEmitPreinits(const Stmt *PreInits) {
281 if (PreInits) {
282 // CompoundStmts and DeclStmts are used as lists of PreInit statements and
283 // declarations. Since declarations must be visible in the the following
284 // that they initialize, unpack the CompoundStmt they are nested in.
285 SmallVector<const Stmt *> PreInitStmts;
286 if (auto *PreInitCompound = dyn_cast<CompoundStmt>(Val: PreInits))
287 llvm::append_range(C&: PreInitStmts, R: PreInitCompound->body());
288 else
289 PreInitStmts.push_back(Elt: PreInits);
290
291 for (const Stmt *S : PreInitStmts) {
292 // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted
293 // here.
294 if (auto *PreInitDecl = dyn_cast<DeclStmt>(Val: S)) {
295 for (Decl *I : PreInitDecl->decls())
296 CGF.EmitVarDecl(D: cast<VarDecl>(Val&: *I));
297 continue;
298 }
299 CGF.EmitStmt(S);
300 }
301 }
302 }
303
304public:
305 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
306 : CodeGenFunction::RunCleanupsScope(CGF) {
307 emitPreInitStmt(CGF, S);
308 }
309 OMPLoopScope(CodeGenFunction &CGF,
310 const OMPCanonicalLoopSequenceTransformationDirective &S)
311 : CodeGenFunction::RunCleanupsScope(CGF) {
312 emitPreInitStmt(CGF, S);
313 }
314};
315
316class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
317 CodeGenFunction::OMPPrivateScope InlinedShareds;
318
319 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
320 return CGF.LambdaCaptureFields.lookup(Val: VD) ||
321 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
322 (isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl) &&
323 cast<BlockDecl>(Val: CGF.CurCodeDecl)->capturesVariable(var: VD));
324 }
325
326public:
327 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
328 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
329 InlinedShareds(CGF) {
330 for (const auto *C : S.clauses()) {
331 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
332 if (const auto *PreInit =
333 cast_or_null<DeclStmt>(Val: CPI->getPreInitStmt())) {
334 for (const auto *I : PreInit->decls()) {
335 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
336 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
337 } else {
338 CodeGenFunction::AutoVarEmission Emission =
339 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
340 CGF.EmitAutoVarCleanups(emission: Emission);
341 }
342 }
343 }
344 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(Val: C)) {
345 for (const Expr *E : UDP->varlist()) {
346 const Decl *D = cast<DeclRefExpr>(Val: E)->getDecl();
347 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
348 CGF.EmitVarDecl(D: *OED);
349 }
350 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(Val: C)) {
351 for (const Expr *E : UDP->varlist()) {
352 const Decl *D = getBaseDecl(Ref: E);
353 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
354 CGF.EmitVarDecl(D: *OED);
355 }
356 }
357 }
358 if (!isOpenMPSimdDirective(DKind: S.getDirectiveKind()))
359 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: InlinedShareds);
360 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(Val: &S)) {
361 if (const Expr *E = TG->getReductionRef())
362 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl()));
363 }
364 // Temp copy arrays for inscan reductions should not be emitted as they are
365 // not used in simd only mode.
366 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
367 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
368 if (C->getModifier() != OMPC_REDUCTION_inscan)
369 continue;
370 for (const Expr *E : C->copy_array_temps())
371 CopyArrayTemps.insert(V: cast<DeclRefExpr>(Val: E)->getDecl());
372 }
373 const auto *CS = cast_or_null<CapturedStmt>(Val: S.getAssociatedStmt());
374 while (CS) {
375 for (auto &C : CS->captures()) {
376 if (C.capturesVariable() || C.capturesVariableByCopy()) {
377 auto *VD = C.getCapturedVar();
378 if (CopyArrayTemps.contains(V: VD))
379 continue;
380 assert(VD == VD->getCanonicalDecl() &&
381 "Canonical decl must be captured.");
382 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
383 isCapturedVar(CGF, VD) ||
384 (CGF.CapturedStmtInfo &&
385 InlinedShareds.isGlobalVarCaptured(VD)),
386 VD->getType().getNonReferenceType(), VK_LValue,
387 C.getLocation());
388 InlinedShareds.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
389 }
390 }
391 CS = dyn_cast<CapturedStmt>(Val: CS->getCapturedStmt());
392 }
393 (void)InlinedShareds.Privatize();
394 }
395};
396
397} // namespace
398
399// Map `loop` with a bind clause, `teams loop`, `target loop`, and
400// `target teams loop` to the construct CodeGen emits.
401static OpenMPDirectiveKind
402getEffectiveDirectiveKind(const OMPExecutableDirective &S) {
403 OpenMPDirectiveKind Kind = S.getDirectiveKind();
404
405 if (Kind == OMPD_teams_loop)
406 return OMPD_teams_distribute;
407 if (Kind == OMPD_target_loop)
408 return OMPD_target_simd;
409 if (Kind == OMPD_target_teams_loop)
410 return cast<OMPTargetTeamsGenericLoopDirective>(Val: S).canBeParallelFor()
411 ? OMPD_target_teams_distribute_parallel_for
412 : OMPD_target_teams_distribute;
413
414 if (Kind != OMPD_loop)
415 return Kind;
416
417 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;
418 if (const auto *C = S.getSingleClause<OMPBindClause>())
419 BindKind = C->getBindKind();
420
421 switch (BindKind) {
422 case OMPC_BIND_parallel:
423 return OMPD_for;
424 case OMPC_BIND_teams:
425 return OMPD_distribute;
426 case OMPC_BIND_thread:
427 return OMPD_simd;
428 default:
429 return OMPD_loop;
430 }
431}
432
433static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
434 const OMPExecutableDirective &S,
435 const RegionCodeGenTy &CodeGen);
436
437Address CodeGenFunction::EmitOMPBindingOriginalAddr(const BindingDecl *BD,
438 SourceLocation Loc) {
439 if (CapturedStmtInfo &&
440 CapturedStmtInfo->getKind() == CapturedRegionKind::CR_OpenMP) {
441 if (const auto *DD = dyn_cast<VarDecl>(Val: BD->getDecomposedDecl())) {
442 if (CapturedStmtInfo->lookup(VD: DD))
443 return EmitOMPCapturedBindingLValue(BD).getAddress();
444 }
445 }
446 DeclRefExpr DRE(getContext(), const_cast<BindingDecl *>(BD),
447 /*RefersToEnclosingVariableOrCapture=*/false, BD->getType(),
448 VK_LValue, Loc);
449 return EmitLValue(E: &DRE).getAddress();
450}
451
452LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
453 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(Val: E)) {
454 if (const auto *OrigVD = dyn_cast<VarDecl>(Val: OrigDRE->getDecl())) {
455 OrigVD = OrigVD->getCanonicalDecl();
456 bool IsCaptured =
457 LambdaCaptureFields.lookup(Val: OrigVD) ||
458 (CapturedStmtInfo && CapturedStmtInfo->lookup(VD: OrigVD)) ||
459 (isa_and_nonnull<BlockDecl>(Val: CurCodeDecl));
460 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
461 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
462 return EmitLValue(E: &DRE);
463 }
464 if (const auto *OrigBD = dyn_cast<BindingDecl>(Val: OrigDRE->getDecl())) {
465 OrigBD = cast<BindingDecl>(Val: OrigBD->getCanonicalDecl());
466 const auto *DD = cast<VarDecl>(Val: OrigBD->getDecomposedDecl());
467 bool IsCaptured = LambdaCaptureFields.lookup(Val: OrigBD) ||
468 (CapturedStmtInfo && CapturedStmtInfo->lookup(VD: DD)) ||
469 isa_and_nonnull<BlockDecl>(Val: CurCodeDecl);
470 DeclRefExpr DRE(getContext(), const_cast<BindingDecl *>(OrigBD),
471 IsCaptured, OrigDRE->getType(), VK_LValue,
472 OrigDRE->getExprLoc());
473 return EmitLValue(E: &DRE);
474 }
475 }
476 return EmitLValue(E);
477}
478
479llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
480 ASTContext &C = getContext();
481 llvm::Value *Size = nullptr;
482 auto SizeInChars = C.getTypeSizeInChars(T: Ty);
483 if (SizeInChars.isZero()) {
484 // getTypeSizeInChars() returns 0 for a VLA.
485 while (const VariableArrayType *VAT = C.getAsVariableArrayType(T: Ty)) {
486 VlaSizePair VlaSize = getVLASize(vla: VAT);
487 Ty = VlaSize.Type;
488 Size =
489 Size ? Builder.CreateNUWMul(LHS: Size, RHS: VlaSize.NumElts) : VlaSize.NumElts;
490 }
491 SizeInChars = C.getTypeSizeInChars(T: Ty);
492 if (SizeInChars.isZero())
493 return llvm::ConstantInt::get(Ty: SizeTy, /*V=*/0);
494 return Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: SizeInChars));
495 }
496 return CGM.getSize(numChars: SizeInChars);
497}
498
499void CodeGenFunction::GenerateOpenMPCapturedVars(
500 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
501 const RecordDecl *RD = S.getCapturedRecordDecl();
502 auto CurField = RD->field_begin();
503 auto CurCap = S.captures().begin();
504 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
505 E = S.capture_init_end();
506 I != E; ++I, ++CurField, ++CurCap) {
507 if (CurField->hasCapturedVLAType()) {
508 const VariableArrayType *VAT = CurField->getCapturedVLAType();
509 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
510 CapturedVars.push_back(Elt: Val);
511 } else if (CurCap->capturesThis()) {
512 CapturedVars.push_back(Elt: CXXThisValue);
513 } else if (CurCap->capturesVariableByCopy()) {
514 llvm::Value *CV = EmitLoadOfScalar(lvalue: EmitLValue(E: *I), Loc: CurCap->getLocation());
515
516 // If the field is not a pointer, we need to save the actual value
517 // and load it as a void pointer.
518 if (!CurField->getType()->isAnyPointerType()) {
519 ASTContext &Ctx = getContext();
520 Address DstAddr = CreateMemTempWithoutCast(
521 T: Ctx.getUIntPtrType(),
522 Name: Twine(CurCap->getCapturedVar()->getName(), ".casted"));
523 LValue DstLV = MakeAddrLValue(Addr: DstAddr, T: Ctx.getUIntPtrType());
524
525 llvm::Value *SrcAddrVal = EmitScalarConversion(
526 Src: DstAddr.emitRawPointer(CGF&: *this),
527 SrcTy: Ctx.getPointerType(T: Ctx.getUIntPtrType()),
528 DstTy: Ctx.getPointerType(T: CurField->getType()), Loc: CurCap->getLocation());
529 LValue SrcLV =
530 MakeNaturalAlignAddrLValue(V: SrcAddrVal, T: CurField->getType());
531
532 // Store the value using the source type pointer.
533 EmitStoreThroughLValue(Src: RValue::get(V: CV), Dst: SrcLV);
534
535 // Load the value using the destination type pointer.
536 CV = EmitLoadOfScalar(lvalue: DstLV, Loc: CurCap->getLocation());
537 }
538 CapturedVars.push_back(Elt: CV);
539 } else {
540 assert(CurCap->capturesVariable() && "Expected capture by reference.");
541 llvm::Value *Addr = EmitLValue(E: *I).getAddress().emitRawPointer(CGF&: *this);
542 // Sema strips the address space from the type of the captured field.
543 llvm::Type *ArgTy = ConvertType(T: CurField->getType());
544 if (Addr->getType() != ArgTy)
545 Addr = performAddrSpaceCast(Src: Addr, DestTy: ArgTy);
546 CapturedVars.push_back(Elt: Addr);
547 }
548 }
549}
550
551static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
552 QualType DstType, StringRef Name,
553 LValue AddrLV) {
554 ASTContext &Ctx = CGF.getContext();
555
556 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
557 Src: AddrLV.getAddress().emitRawPointer(CGF), SrcTy: Ctx.getUIntPtrType(),
558 DstTy: Ctx.getPointerType(T: DstType), Loc);
559 // FIXME: should the pointee type (DstType) be passed?
560 Address TmpAddr =
561 CGF.MakeNaturalAlignAddrLValue(V: CastedPtr, T: DstType).getAddress();
562 return TmpAddr;
563}
564
565static QualType getCanonicalParamType(ASTContext &C, QualType T) {
566 if (T->isLValueReferenceType())
567 return C.getLValueReferenceType(
568 T: getCanonicalParamType(C, T: T.getNonReferenceType()),
569 /*SpelledAsLValue=*/false);
570 if (T->isPointerType())
571 return C.getPointerType(T: getCanonicalParamType(C, T: T->getPointeeType()));
572 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
573 if (const auto *VLA = dyn_cast<VariableArrayType>(Val: A))
574 return getCanonicalParamType(C, T: VLA->getElementType());
575 if (!A->isVariablyModifiedType())
576 return C.getCanonicalType(T);
577 }
578 return C.getCanonicalParamType(T);
579}
580
581namespace {
582/// Contains required data for proper outlined function codegen.
583struct FunctionOptions {
584 /// Captured statement for which the function is generated.
585 const CapturedStmt *S = nullptr;
586 /// true if cast to/from UIntPtr is required for variables captured by
587 /// value.
588 const bool UIntPtrCastRequired = true;
589 /// true if only casted arguments must be registered as local args or VLA
590 /// sizes.
591 const bool RegisterCastedArgsOnly = false;
592 /// Name of the generated function.
593 const StringRef FunctionName;
594 /// Location of the non-debug version of the outlined function.
595 SourceLocation Loc;
596 const bool IsDeviceKernel = false;
597 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
598 bool RegisterCastedArgsOnly, StringRef FunctionName,
599 SourceLocation Loc, bool IsDeviceKernel)
600 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
601 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
602 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
603};
604} // namespace
605
606static llvm::Function *emitOutlinedFunctionPrologue(
607 CodeGenFunction &CGF, FunctionArgList &Args,
608 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
609 &LocalAddrs,
610 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
611 &VLASizes,
612 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
613 const CapturedDecl *CD = FO.S->getCapturedDecl();
614 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
615 assert(CD->hasBody() && "missing CapturedDecl body");
616
617 CXXThisValue = nullptr;
618 // Build the argument list.
619 CodeGenModule &CGM = CGF.CGM;
620 ASTContext &Ctx = CGM.getContext();
621 FunctionArgList TargetArgs;
622 Args.append(in_start: CD->param_begin(),
623 in_end: std::next(x: CD->param_begin(), n: CD->getContextParamPosition()));
624 TargetArgs.append(
625 in_start: CD->param_begin(),
626 in_end: std::next(x: CD->param_begin(), n: CD->getContextParamPosition()));
627 auto I = FO.S->captures().begin();
628 FunctionDecl *DebugFunctionDecl = nullptr;
629 if (!FO.UIntPtrCastRequired) {
630 FunctionProtoType::ExtProtoInfo EPI;
631 QualType FunctionTy = Ctx.getFunctionType(ResultTy: Ctx.VoidTy, Args: {}, EPI);
632 DebugFunctionDecl = FunctionDecl::Create(
633 C&: Ctx, DC: Ctx.getTranslationUnitDecl(), StartLoc: FO.S->getBeginLoc(),
634 NLoc: SourceLocation(), N: DeclarationName(), T: FunctionTy,
635 TInfo: Ctx.getTrivialTypeSourceInfo(T: FunctionTy), SC: SC_Static,
636 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
637 /*hasWrittenPrototype=*/false);
638 }
639 for (const FieldDecl *FD : RD->fields()) {
640 QualType ArgType = FD->getType();
641 IdentifierInfo *II = nullptr;
642 VarDecl *CapVar = nullptr;
643
644 // If this is a capture by copy and the type is not a pointer, the outlined
645 // function argument type should be uintptr and the value properly casted to
646 // uintptr. This is necessary given that the runtime library is only able to
647 // deal with pointers. We can pass in the same way the VLA type sizes to the
648 // outlined function.
649 if (FO.UIntPtrCastRequired &&
650 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
651 I->capturesVariableArrayType()))
652 ArgType = Ctx.getUIntPtrType();
653
654 if (I->capturesVariable() || I->capturesVariableByCopy()) {
655 CapVar = I->getCapturedVar();
656 II = CapVar->getIdentifier();
657 } else if (I->capturesThis()) {
658 II = &Ctx.Idents.get(Name: "this");
659 } else {
660 assert(I->capturesVariableArrayType());
661 II = &Ctx.Idents.get(Name: "vla");
662 }
663 if (ArgType->isVariablyModifiedType())
664 ArgType = getCanonicalParamType(C&: Ctx, T: ArgType);
665 VarDecl *Arg;
666 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
667 Arg = ImplicitParamDecl::Create(C&: Ctx, /*DC=*/nullptr, IdLoc: FD->getLocation(),
668 Id: II, T: ArgType,
669 ParamKind: ImplicitParamKind::ThreadPrivateVar);
670 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
671 Arg = ParmVarDecl::Create(
672 C&: Ctx, DC: DebugFunctionDecl,
673 StartLoc: CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
674 IdLoc: CapVar ? CapVar->getLocation() : FD->getLocation(), Id: II, T: ArgType,
675 /*TInfo=*/nullptr, S: SC_None, /*DefArg=*/nullptr);
676 } else {
677 Arg = ImplicitParamDecl::Create(C&: Ctx, /*DC=*/nullptr, IdLoc: FD->getLocation(),
678 Id: II, T: ArgType, ParamKind: ImplicitParamKind::Other);
679 }
680 Args.emplace_back(Args&: Arg);
681 // Do not cast arguments if we emit function with non-original types.
682 TargetArgs.emplace_back(
683 Args: FO.UIntPtrCastRequired
684 ? Arg
685 : CGM.getOpenMPRuntime().translateParameter(FD, NativeParam: Arg));
686 ++I;
687 }
688 Args.append(in_start: std::next(x: CD->param_begin(), n: CD->getContextParamPosition() + 1),
689 in_end: CD->param_end());
690 TargetArgs.append(
691 in_start: std::next(x: CD->param_begin(), n: CD->getContextParamPosition() + 1),
692 in_end: CD->param_end());
693
694 // Create the function declaration.
695 const CGFunctionInfo &FuncInfo =
696 FO.IsDeviceKernel
697 ? CGM.getTypes().arrangeDeviceKernelCallerDeclaration(resultType: Ctx.VoidTy,
698 args: TargetArgs)
699 : CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy,
700 args: TargetArgs);
701 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
702
703 auto *F =
704 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
705 N: FO.FunctionName, M: &CGM.getModule());
706 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
707
708 // Adjust the calling convention for SPIR-V targets to avoid mismatches
709 // between callee and caller.
710 if (CGM.getTriple().isSPIRV() && !FO.IsDeviceKernel)
711 F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
712
713 if (CD->isNothrow())
714 F->setDoesNotThrow();
715
716 // Always inline the outlined function if optimizations are enabled.
717 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
718 F->removeFnAttr(Kind: llvm::Attribute::NoInline);
719 F->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
720 }
721 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
722 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
723
724 // Generate the function.
725 CGF.StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args: TargetArgs,
726 Loc: FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
727 StartLoc: FO.UIntPtrCastRequired ? FO.Loc
728 : CD->getBody()->getBeginLoc());
729 unsigned Cnt = CD->getContextParamPosition();
730 I = FO.S->captures().begin();
731 for (const FieldDecl *FD : RD->fields()) {
732 // Do not map arguments if we emit function with non-original types.
733 Address LocalAddr(Address::invalid());
734 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
735 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, NativeParam: Args[Cnt],
736 TargetParam: TargetArgs[Cnt]);
737 } else {
738 LocalAddr = CGF.GetAddrOfLocalVar(VD: Args[Cnt]);
739 }
740 // If we are capturing a pointer by copy we don't need to do anything, just
741 // use the value that we get from the arguments.
742 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
743 const VarDecl *CurVD = I->getCapturedVar();
744 if (!FO.RegisterCastedArgsOnly)
745 LocalAddrs.insert(KV: {Args[Cnt], {CurVD, LocalAddr}});
746 ++Cnt;
747 ++I;
748 continue;
749 }
750
751 LValue ArgLVal = CGF.MakeAddrLValue(Addr: LocalAddr, T: Args[Cnt]->getType(),
752 Source: AlignmentSource::Decl);
753 if (FD->hasCapturedVLAType()) {
754 if (FO.UIntPtrCastRequired) {
755 ArgLVal = CGF.MakeAddrLValue(
756 Addr: castValueFromUintptr(CGF, Loc: I->getLocation(), DstType: FD->getType(),
757 Name: Args[Cnt]->getName(), AddrLV: ArgLVal),
758 T: FD->getType(), Source: AlignmentSource::Decl);
759 }
760 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: I->getLocation());
761 const VariableArrayType *VAT = FD->getCapturedVLAType();
762 VLASizes.try_emplace(Key: Args[Cnt], Args: VAT->getSizeExpr(), Args&: ExprArg);
763 } else if (I->capturesVariable()) {
764 const VarDecl *Var = I->getCapturedVar();
765 QualType VarTy = Var->getType();
766 Address ArgAddr = ArgLVal.getAddress();
767 if (ArgLVal.getType()->isLValueReferenceType()) {
768 ArgAddr = CGF.EmitLoadOfReference(RefLVal: ArgLVal);
769 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
770 assert(ArgLVal.getType()->isPointerType());
771 ArgAddr = CGF.EmitLoadOfPointer(
772 Ptr: ArgAddr, PtrTy: ArgLVal.getType()->castAs<PointerType>());
773 }
774 if (!FO.RegisterCastedArgsOnly) {
775 LocalAddrs.insert(
776 KV: {Args[Cnt], {Var, ArgAddr.withAlignment(NewAlignment: Ctx.getDeclAlign(D: Var))}});
777 }
778 } else if (I->capturesVariableByCopy()) {
779 assert(!FD->getType()->isAnyPointerType() &&
780 "Not expecting a captured pointer.");
781 const VarDecl *Var = I->getCapturedVar();
782 LocalAddrs.insert(KV: {Args[Cnt],
783 {Var, FO.UIntPtrCastRequired
784 ? castValueFromUintptr(
785 CGF, Loc: I->getLocation(), DstType: FD->getType(),
786 Name: Args[Cnt]->getName(), AddrLV: ArgLVal)
787 : ArgLVal.getAddress()}});
788 } else {
789 // If 'this' is captured, load it into CXXThisValue.
790 assert(I->capturesThis());
791 CXXThisValue = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: I->getLocation());
792 LocalAddrs.insert(KV: {Args[Cnt], {nullptr, ArgLVal.getAddress()}});
793 }
794 ++Cnt;
795 ++I;
796 }
797
798 return F;
799}
800
801static llvm::Function *emitOutlinedFunctionPrologueAggregate(
802 CodeGenFunction &CGF, FunctionArgList &Args,
803 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
804 &LocalAddrs,
805 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
806 &VLASizes,
807 llvm::Value *&CXXThisValue, llvm::Value *&ContextV, const CapturedStmt &CS,
808 SourceLocation Loc, StringRef FunctionName) {
809 const CapturedDecl *CD = CS.getCapturedDecl();
810 const RecordDecl *RD = CS.getCapturedRecordDecl();
811
812 CXXThisValue = nullptr;
813 CodeGenModule &CGM = CGF.CGM;
814 ASTContext &Ctx = CGM.getContext();
815 Args.push_back(Elt: CD->getContextParam());
816
817 const CGFunctionInfo &FuncInfo =
818 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
819 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
820
821 auto *F =
822 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
823 N: FunctionName, M: &CGM.getModule());
824 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
825 if (CD->isNothrow())
826 F->setDoesNotThrow();
827
828 CGF.StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc, StartLoc: Loc);
829 Address ContextAddr = CGF.GetAddrOfLocalVar(VD: CD->getContextParam());
830 ContextV = CGF.Builder.CreateLoad(Addr: ContextAddr);
831
832 // The runtime passes arguments as an array of pointers.
833 llvm::Type *PtrTy = CGF.Builder.getPtrTy();
834 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
835 CharUnits SlotAlign = CharUnits::fromQuantity(Quantity: PtrAlign.value());
836
837 for (auto [FD, C, FieldIdx] :
838 llvm::zip(t: RD->fields(), u: CS.captures(),
839 args: llvm::seq<unsigned>(Size: RD->getNumFields()))) {
840 llvm::Value *SlotPtr =
841 CGF.Builder.CreateConstInBoundsGEP1_32(Ty: PtrTy, Ptr: ContextV, Idx0: FieldIdx);
842 llvm::Value *Slot = CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Ptr: SlotPtr, Align: PtrAlign);
843
844 // Generate the appropriate load from the per-argument storage. This
845 // includes all of the user arguments as well as the implicit kernel
846 // argument pointer.
847 if (C.capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
848 const VarDecl *CurVD = C.getCapturedVar();
849 Slot->setName(CurVD->getName());
850 Address SlotAddr(Slot, PtrTy, SlotAlign);
851 LocalAddrs.insert(KV: {FD, {CurVD, SlotAddr}});
852 } else if (FD->hasCapturedVLAType()) {
853 // VLA size is stored as intptr_t directly in the slot.
854 Address SlotAddr(Slot, CGF.ConvertTypeForMem(T: FD->getType()), SlotAlign);
855 LValue ArgLVal =
856 CGF.MakeAddrLValue(Addr: SlotAddr, T: FD->getType(), Source: AlignmentSource::Decl);
857 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: C.getLocation());
858 const VariableArrayType *VAT = FD->getCapturedVLAType();
859 VLASizes.try_emplace(Key: FD, Args: VAT->getSizeExpr(), Args&: ExprArg);
860 } else if (C.capturesVariable()) {
861 const VarDecl *Var = C.getCapturedVar();
862 QualType VarTy = Var->getType();
863
864 if (VarTy->isVariablyModifiedType() && VarTy->isPointerType()) {
865 Slot->setName(Var->getName() + ".addr");
866 Address SlotAddr(Slot, PtrTy, SlotAlign);
867 LocalAddrs.insert(KV: {FD, {Var, SlotAddr}});
868 } else {
869 llvm::Value *VarAddr = CGF.Builder.CreateAlignedLoad(
870 Ty: PtrTy, Ptr: Slot, Align: PtrAlign, Name: Var->getName());
871 LocalAddrs.insert(KV: {FD,
872 {Var, Address(VarAddr, CGF.ConvertTypeForMem(T: VarTy),
873 Ctx.getDeclAlign(D: Var))}});
874 }
875 } else if (C.capturesVariableByCopy()) {
876 assert(!FD->getType()->isAnyPointerType() &&
877 "Not expecting a captured pointer.");
878 const VarDecl *Var = C.getCapturedVar();
879 QualType FieldTy = FD->getType();
880
881 // Scalar values are promoted and stored directly in the slot.
882 Address SlotAddr(Slot, CGF.ConvertTypeForMem(T: FieldTy), SlotAlign);
883 Address CopyAddr =
884 CGF.CreateMemTemp(T: FieldTy, Align: Ctx.getDeclAlign(D: FD), Name: Var->getName());
885 LValue SrcLVal =
886 CGF.MakeAddrLValue(Addr: SlotAddr, T: FieldTy, Source: AlignmentSource::Decl);
887 LValue CopyLVal =
888 CGF.MakeAddrLValue(Addr: CopyAddr, T: FieldTy, Source: AlignmentSource::Decl);
889
890 RValue ArgRVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc: C.getLocation());
891 CGF.EmitStoreThroughLValue(Src: ArgRVal, Dst: CopyLVal);
892
893 LocalAddrs.insert(KV: {FD, {Var, CopyAddr}});
894 } else {
895 assert(C.capturesThis() && "Default case expected to be CXX 'this'");
896 CXXThisValue =
897 CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Ptr: Slot, Align: PtrAlign, Name: "this");
898 Address SlotAddr(Slot, PtrTy, SlotAlign);
899 LocalAddrs.insert(KV: {FD, {nullptr, SlotAddr}});
900 }
901 }
902
903 return F;
904}
905
906llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunction(
907 const CapturedStmt &S, const OMPExecutableDirective &D) {
908 SourceLocation Loc = D.getBeginLoc();
909 assert(
910 CapturedStmtInfo &&
911 "CapturedStmtInfo should be set when generating the captured function");
912 const CapturedDecl *CD = S.getCapturedDecl();
913 // Build the argument list.
914 bool NeedWrapperFunction =
915 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
916 FunctionArgList Args, WrapperArgs;
917 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
918 WrapperLocalAddrs;
919 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
920 WrapperVLASizes;
921 SmallString<256> Buffer;
922 llvm::raw_svector_ostream Out(Buffer);
923 Out << CapturedStmtInfo->getHelperName();
924 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
925 bool IsDeviceKernel = CGM.getOpenMPRuntime().isGPU() &&
926 isOpenMPTargetExecutionDirective(DKind: EKind) &&
927 D.getCapturedStmt(RegionKind: OMPD_target) == &S;
928 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
929 llvm::Function *WrapperF = nullptr;
930 if (NeedWrapperFunction) {
931 // Emit the final kernel early to allow attributes to be added by the
932 // OpenMPI-IR-Builder.
933 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
934 /*RegisterCastedArgsOnly=*/true,
935 CapturedStmtInfo->getHelperName(), Loc,
936 IsDeviceKernel);
937 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
938 WrapperF =
939 emitOutlinedFunctionPrologue(CGF&: WrapperCGF, Args, LocalAddrs, VLASizes,
940 CXXThisValue&: WrapperCGF.CXXThisValue, FO: WrapperFO);
941 Out << "_debug__";
942 }
943 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
944 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
945 llvm::Function *F = emitOutlinedFunctionPrologue(
946 CGF&: *this, Args&: WrapperArgs, LocalAddrs&: WrapperLocalAddrs, VLASizes&: WrapperVLASizes, CXXThisValue, FO);
947 CodeGenFunction::OMPPrivateScope LocalScope(*this);
948 for (const auto &LocalAddrPair : WrapperLocalAddrs) {
949 if (LocalAddrPair.second.first) {
950 LocalScope.addPrivate(LocalVD: LocalAddrPair.second.first,
951 Addr: LocalAddrPair.second.second);
952 }
953 }
954 (void)LocalScope.Privatize();
955 for (const auto &VLASizePair : WrapperVLASizes)
956 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
957 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
958 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
959 LocalScope.ForceCleanup();
960 FinishFunction(EndLoc: CD->getBodyRBrace());
961 if (!NeedWrapperFunction)
962 return F;
963
964 // Reverse the order.
965 WrapperF->removeFromParent();
966 F->getParent()->getFunctionList().insertAfter(where: F->getIterator(), New: WrapperF);
967
968 llvm::SmallVector<llvm::Value *, 4> CallArgs;
969 auto *PI = F->arg_begin();
970 for (const auto *Arg : Args) {
971 llvm::Value *CallArg;
972 auto I = LocalAddrs.find(Key: Arg);
973 if (I != LocalAddrs.end()) {
974 LValue LV = WrapperCGF.MakeAddrLValue(
975 Addr: I->second.second,
976 T: I->second.first ? I->second.first->getType() : Arg->getType(),
977 Source: AlignmentSource::Decl);
978 if (LV.getType()->isAnyComplexType())
979 LV.setAddress(LV.getAddress().withElementType(ElemTy: PI->getType()));
980 CallArg = WrapperCGF.EmitLoadOfScalar(lvalue: LV, Loc: S.getBeginLoc());
981 } else {
982 auto EI = VLASizes.find(Val: Arg);
983 if (EI != VLASizes.end()) {
984 CallArg = EI->second.second;
985 } else {
986 LValue LV =
987 WrapperCGF.MakeAddrLValue(Addr: WrapperCGF.GetAddrOfLocalVar(VD: Arg),
988 T: Arg->getType(), Source: AlignmentSource::Decl);
989 CallArg = WrapperCGF.EmitLoadOfScalar(lvalue: LV, Loc: S.getBeginLoc());
990 }
991 }
992 CallArgs.emplace_back(Args: WrapperCGF.EmitFromMemory(Value: CallArg, Ty: Arg->getType()));
993 ++PI;
994 }
995 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF&: WrapperCGF, Loc, OutlinedFn: F, Args: CallArgs);
996 WrapperCGF.FinishFunction();
997 return WrapperF;
998}
999
1000llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunctionAggregate(
1001 const CapturedStmt &S, const OMPExecutableDirective &D) {
1002 SourceLocation Loc = D.getBeginLoc();
1003 assert(
1004 CapturedStmtInfo &&
1005 "CapturedStmtInfo should be set when generating the captured function");
1006 const CapturedDecl *CD = S.getCapturedDecl();
1007 const RecordDecl *RD = S.getCapturedRecordDecl();
1008 StringRef FunctionName = CapturedStmtInfo->getHelperName();
1009 bool NeedWrapperFunction =
1010 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
1011
1012 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
1013 llvm::Function *WrapperF = nullptr;
1014 llvm::Value *WrapperContextV = nullptr;
1015 if (NeedWrapperFunction) {
1016 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
1017 FunctionArgList WrapperArgs;
1018 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
1019 WrapperLocalAddrs;
1020 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
1021 WrapperVLASizes;
1022 WrapperF = emitOutlinedFunctionPrologueAggregate(
1023 CGF&: WrapperCGF, Args&: WrapperArgs, LocalAddrs&: WrapperLocalAddrs, VLASizes&: WrapperVLASizes,
1024 CXXThisValue&: WrapperCGF.CXXThisValue, ContextV&: WrapperContextV, CS: S, Loc, FunctionName);
1025 }
1026
1027 FunctionArgList Args;
1028 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
1029 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
1030 llvm::Function *F;
1031
1032 if (NeedWrapperFunction) {
1033 SmallString<256> Buffer;
1034 llvm::raw_svector_ostream Out(Buffer);
1035 Out << FunctionName << "_debug__";
1036
1037 FunctionOptions FO(&S, /*UIntPtrCastRequired=*/false,
1038 /*RegisterCastedArgsOnly=*/false, Out.str(), Loc,
1039 /*IsDeviceKernel=*/false);
1040 F = emitOutlinedFunctionPrologue(CGF&: *this, Args, LocalAddrs, VLASizes,
1041 CXXThisValue, FO);
1042 } else {
1043 llvm::Value *ContextV = nullptr;
1044 F = emitOutlinedFunctionPrologueAggregate(CGF&: *this, Args, LocalAddrs, VLASizes,
1045 CXXThisValue, ContextV, CS: S, Loc,
1046 FunctionName);
1047
1048 const RecordDecl *RD = S.getCapturedRecordDecl();
1049 unsigned FieldIdx = RD->getNumFields();
1050 for (unsigned I = 0; I < CD->getNumParams(); ++I) {
1051 const ImplicitParamDecl *Param = CD->getParam(i: I);
1052 if (Param == CD->getContextParam())
1053 continue;
1054 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
1055 llvm::Value *SlotPtr = Builder.CreateConstInBoundsGEP1_32(
1056 Ty: Builder.getPtrTy(), Ptr: ContextV, Idx0: FieldIdx,
1057 Name: Twine(Param->getName()) + ".addr");
1058 llvm::Value *ParamAddr =
1059 Builder.CreateAlignedLoad(Ty: Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1060 llvm::Value *ParamVal = Builder.CreateAlignedLoad(
1061 Ty: Builder.getPtrTy(), Ptr: ParamAddr, Align: PtrAlign, Name: Param->getName());
1062 Address ParamLocalAddr =
1063 CreateMemTemp(T: Param->getType(), Name: Param->getName());
1064 Builder.CreateStore(Val: ParamVal, Addr: ParamLocalAddr);
1065 LocalAddrs.insert(KV: {Param, {Param, ParamLocalAddr}});
1066 ++FieldIdx;
1067 }
1068 }
1069
1070 CodeGenFunction::OMPPrivateScope LocalScope(*this);
1071 for (const auto &LocalAddrPair : LocalAddrs) {
1072 if (LocalAddrPair.second.first)
1073 LocalScope.addPrivate(LocalVD: LocalAddrPair.second.first,
1074 Addr: LocalAddrPair.second.second);
1075 }
1076 (void)LocalScope.Privatize();
1077 for (const auto &VLASizePair : VLASizes)
1078 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
1079 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
1080 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
1081 (void)LocalScope.ForceCleanup();
1082 FinishFunction(EndLoc: CD->getBodyRBrace());
1083
1084 if (!NeedWrapperFunction)
1085 return F;
1086
1087 // Reverse the order.
1088 WrapperF->removeFromParent();
1089 F->getParent()->getFunctionList().insertAfter(where: F->getIterator(), New: WrapperF);
1090
1091 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
1092 llvm::SmallVector<llvm::Value *, 16> CallArgs;
1093 assert(CD->getContextParamPosition() == 0 &&
1094 "Expected context param at position 0 for target regions");
1095 assert(RD->getNumFields() + 1 == F->arg_size() && "Argument count mismatch");
1096
1097 for (auto [FD, InnerParam, SlotIdx] : llvm::zip(
1098 t: RD->fields(), u: F->args(), args: llvm::seq<unsigned>(Size: RD->getNumFields()))) {
1099 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1100 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: WrapperContextV, Idx0: SlotIdx);
1101 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1102 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1103 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1104 Ty: InnerParam.getType(), Ptr: Slot, Align: PtrAlign, Name: InnerParam.getName());
1105 CallArgs.push_back(Elt: Val);
1106 }
1107
1108 // Handle the load from the implicit dyn_ptr at the end of the __context.
1109 unsigned SlotIdx = RD->getNumFields();
1110 auto InnerParam = F->arg_begin() + SlotIdx;
1111 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1112 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: WrapperContextV, Idx0: SlotIdx);
1113 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1114 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1115 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1116 Ty: InnerParam->getType(), Ptr: Slot, Align: PtrAlign, Name: InnerParam->getName());
1117 CallArgs.push_back(Elt: Val);
1118
1119 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF&: WrapperCGF, Loc, OutlinedFn: F, Args: CallArgs);
1120 WrapperCGF.FinishFunction();
1121 return WrapperF;
1122}
1123
1124//===----------------------------------------------------------------------===//
1125// OpenMP Directive Emission
1126//===----------------------------------------------------------------------===//
1127void CodeGenFunction::EmitOMPAggregateAssign(
1128 Address DestAddr, Address SrcAddr, QualType OriginalType,
1129 const llvm::function_ref<void(Address, Address)> CopyGen) {
1130 // Perform element-by-element initialization.
1131 QualType ElementTy;
1132
1133 // Drill down to the base element type on both arrays.
1134 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
1135 llvm::Value *NumElements = emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
1136 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
1137
1138 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(CGF&: *this);
1139 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF&: *this);
1140 // Cast from pointer to array type to pointer to single element.
1141 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(Ty: DestAddr.getElementType(),
1142 Ptr: DestBegin, IdxList: NumElements);
1143
1144 // The basic structure here is a while-do loop.
1145 llvm::BasicBlock *BodyBB = createBasicBlock(name: "omp.arraycpy.body");
1146 llvm::BasicBlock *DoneBB = createBasicBlock(name: "omp.arraycpy.done");
1147 llvm::Value *IsEmpty =
1148 Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arraycpy.isempty");
1149 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
1150
1151 // Enter the loop body, making that address the current address.
1152 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
1153 EmitBlock(BB: BodyBB);
1154
1155 CharUnits ElementSize = getContext().getTypeSizeInChars(T: ElementTy);
1156
1157 llvm::PHINode *SrcElementPHI =
1158 Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
1159 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
1160 Address SrcElementCurrent =
1161 Address(SrcElementPHI, SrcAddr.getElementType(),
1162 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
1163
1164 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
1165 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
1166 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
1167 Address DestElementCurrent =
1168 Address(DestElementPHI, DestAddr.getElementType(),
1169 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
1170
1171 // Emit copy.
1172 CopyGen(DestElementCurrent, SrcElementCurrent);
1173
1174 // Shift the address forward by one element.
1175 llvm::Value *DestElementNext =
1176 Builder.CreateConstGEP1_32(Ty: DestAddr.getElementType(), Ptr: DestElementPHI,
1177 /*Idx0=*/1, Name: "omp.arraycpy.dest.element");
1178 llvm::Value *SrcElementNext =
1179 Builder.CreateConstGEP1_32(Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI,
1180 /*Idx0=*/1, Name: "omp.arraycpy.src.element");
1181 // Check whether we've reached the end.
1182 llvm::Value *Done =
1183 Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
1184 Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
1185 DestElementPHI->addIncoming(V: DestElementNext, BB: Builder.GetInsertBlock());
1186 SrcElementPHI->addIncoming(V: SrcElementNext, BB: Builder.GetInsertBlock());
1187
1188 // Done.
1189 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
1190}
1191
1192void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
1193 Address SrcAddr, const VarDecl *DestVD,
1194 const VarDecl *SrcVD, const Expr *Copy) {
1195 if (OriginalType->isArrayType()) {
1196 const auto *BO = dyn_cast<BinaryOperator>(Val: Copy);
1197 if (BO && BO->getOpcode() == BO_Assign) {
1198 // Perform simple memcpy for simple copying.
1199 LValue Dest = MakeAddrLValue(Addr: DestAddr, T: OriginalType);
1200 LValue Src = MakeAddrLValue(Addr: SrcAddr, T: OriginalType);
1201 EmitAggregateAssign(Dest, Src, EltTy: OriginalType);
1202 } else {
1203 // For arrays with complex element types perform element by element
1204 // copying.
1205 EmitOMPAggregateAssign(
1206 DestAddr, SrcAddr, OriginalType,
1207 CopyGen: [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
1208 // Working with the single array element, so have to remap
1209 // destination and source variables to corresponding array
1210 // elements.
1211 CodeGenFunction::OMPPrivateScope Remap(*this);
1212 Remap.addPrivate(LocalVD: DestVD, Addr: DestElement);
1213 Remap.addPrivate(LocalVD: SrcVD, Addr: SrcElement);
1214 (void)Remap.Privatize();
1215 EmitIgnoredExpr(E: Copy);
1216 });
1217 }
1218 } else {
1219 // Remap pseudo source variable to private copy.
1220 CodeGenFunction::OMPPrivateScope Remap(*this);
1221 Remap.addPrivate(LocalVD: SrcVD, Addr: SrcAddr);
1222 Remap.addPrivate(LocalVD: DestVD, Addr: DestAddr);
1223 (void)Remap.Privatize();
1224 // Emit copying of the whole variable.
1225 EmitIgnoredExpr(E: Copy);
1226 }
1227}
1228
1229bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
1230 OMPPrivateScope &PrivateScope) {
1231 if (!HaveInsertPoint())
1232 return false;
1233 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1234 bool DeviceConstTarget = getLangOpts().OpenMPIsTargetDevice &&
1235 isOpenMPTargetExecutionDirective(DKind: EKind);
1236 bool FirstprivateIsLastprivate = false;
1237 llvm::SmallDenseMap<const Decl *, OpenMPLastprivateModifier> Lastprivates;
1238 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1239 for (const auto *D : C->varlist()) {
1240 const auto *VD = cast<DeclRefExpr>(Val: D)->getDecl();
1241 Lastprivates.try_emplace(Key: VD->getCanonicalDecl(), Args: C->getKind());
1242 }
1243 }
1244 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
1245 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
1246 getOpenMPCaptureRegions(CaptureRegions, DKind: EKind);
1247 // Force emission of the firstprivate copy if the directive does not emit
1248 // outlined function, like omp for, omp simd, omp distribute etc.
1249 bool MustEmitFirstprivateCopy =
1250 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
1251 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
1252 const auto *IRef = C->varlist_begin();
1253 const auto *InitsRef = C->inits().begin();
1254 for (const Expr *IInit : C->private_copies()) {
1255 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1256 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1257
1258 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
1259 // Check if this binding is also lastprivate.
1260 bool ThisFirstprivateIsLastprivate =
1261 Lastprivates.count(Val: BD->getCanonicalDecl()) > 0;
1262 const auto *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
1263 // If the decomposition is captured by copy, the captured field is
1264 // already a private copy; map the binding to its member directly.
1265 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate) {
1266 if (const FieldDecl *FD = CapturedStmtInfo->lookup(VD: DD)) {
1267 if (!FD->getType()->isReferenceType()) {
1268 bool IsRegistered = PrivateScope.addPrivate(
1269 LocalVD: BD, Addr: EmitOMPCapturedBindingLValue(BD).getAddress());
1270 assert(IsRegistered &&
1271 "firstprivate var already registered as firstprivate");
1272 (void)IsRegistered;
1273 ++IRef;
1274 ++InitsRef;
1275 continue;
1276 }
1277 }
1278 }
1279 const auto *VDInit =
1280 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *InitsRef)->getDecl());
1281 Address OriginalAddr =
1282 EmitOMPBindingOriginalAddr(BD, Loc: (*IRef)->getExprLoc());
1283
1284 QualType Type = VD->getType();
1285 bool IsRegistered;
1286 if (Type->isArrayType()) {
1287 // For array bindings, use array copy logic
1288 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
1289 const Expr *Init = VD->getInit();
1290 LValue OriginalLVal = MakeAddrLValue(Addr: OriginalAddr, T: Type);
1291 if (!Init || !isa<CXXConstructExpr>(Val: Init) ||
1292 isTrivialInitializer(Init)) {
1293 // Perform simple memcpy.
1294 LValue Dest = MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: Type);
1295 EmitAggregateAssign(Dest, Src: OriginalLVal, EltTy: Type);
1296 } else {
1297 EmitOMPAggregateAssign(
1298 DestAddr: Emission.getAllocatedAddress(), SrcAddr: OriginalAddr, OriginalType: Type,
1299 CopyGen: [this, VDInit, Init](Address DestElement, Address SrcElement) {
1300 // Clean up any temporaries needed by the initialization.
1301 RunCleanupsScope InitScope(*this);
1302 // Emit initialization for single element.
1303 setAddrOfLocalVar(VD: VDInit, Addr: SrcElement);
1304 EmitAnyExprToMem(E: Init, Location: DestElement,
1305 Quals: Init->getType().getQualifiers(),
1306 /*IsInitializer*/ false);
1307 LocalDeclMap.erase(Val: VDInit);
1308 });
1309 }
1310 EmitAutoVarCleanups(emission: Emission);
1311 IsRegistered =
1312 PrivateScope.addPrivate(LocalVD: BD, Addr: Emission.getAllocatedAddress());
1313 } else {
1314 // VD now has the binding's type (e.g., int), not the struct type.
1315 // Emit VD initialized from the binding's field address.
1316 setAddrOfLocalVar(VD: VDInit, Addr: OriginalAddr);
1317 EmitDecl(D: *VD);
1318 LocalDeclMap.erase(Val: VDInit);
1319 Address VDAddr = GetAddrOfLocalVar(VD);
1320 // VD is the private copy of the binding, map BD to VDAddr directly
1321 IsRegistered = PrivateScope.addPrivate(LocalVD: BD, Addr: VDAddr);
1322 }
1323
1324 assert(IsRegistered &&
1325 "firstprivate var already registered as firstprivate");
1326 (void)IsRegistered;
1327 FirstprivateIsLastprivate =
1328 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1329 ++IRef;
1330 ++InitsRef;
1331 continue;
1332 }
1333
1334 // Original VarDecl logic.
1335 const VarDecl *OrigVD = dyn_cast<VarDecl>(Val: OrigDecl);
1336 assert(OrigVD && "Expected VarDecl for non-BindingDecl firstprivate");
1337 bool ThisFirstprivateIsLastprivate =
1338 Lastprivates.count(Val: OrigVD->getCanonicalDecl()) > 0;
1339 const FieldDecl *FD = CapturedStmtInfo->lookup(VD: OrigVD);
1340 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
1341 !FD->getType()->isReferenceType() &&
1342 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1343 EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl());
1344 ++IRef;
1345 ++InitsRef;
1346 continue;
1347 }
1348 // Do not emit copy for firstprivate constant variables in target regions,
1349 // captured by reference.
1350 if (DeviceConstTarget && OrigVD->getType().isConstant(Ctx: getContext()) &&
1351 FD && FD->getType()->isReferenceType() &&
1352 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1353 EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl());
1354 ++IRef;
1355 ++InitsRef;
1356 continue;
1357 }
1358 FirstprivateIsLastprivate =
1359 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1360 if (EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl()).second) {
1361 const auto *VDInit =
1362 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *InitsRef)->getDecl());
1363 bool IsRegistered;
1364 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1365 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
1366 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1367 LValue OriginalLVal;
1368 if (!FD) {
1369 // Check if the firstprivate variable is just a constant value.
1370 ConstantEmission CE = tryEmitAsConstant(RefExpr: &DRE);
1371 if (CE && !CE.isReference()) {
1372 // Constant value, no need to create a copy.
1373 ++IRef;
1374 ++InitsRef;
1375 continue;
1376 }
1377 if (CE && CE.isReference()) {
1378 OriginalLVal = CE.getReferenceLValue(CGF&: *this, RefExpr: &DRE);
1379 } else {
1380 assert(!CE && "Expected non-constant firstprivate.");
1381 OriginalLVal = EmitLValue(E: &DRE);
1382 }
1383 } else {
1384 OriginalLVal = EmitLValue(E: &DRE);
1385 }
1386 QualType Type = VD->getType();
1387 if (Type->isArrayType()) {
1388 // Emit VarDecl with copy init for arrays.
1389 // Get the address of the original variable captured in current
1390 // captured region.
1391 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
1392 const Expr *Init = VD->getInit();
1393 if (!isa<CXXConstructExpr>(Val: Init) || isTrivialInitializer(Init)) {
1394 // Perform simple memcpy.
1395 LValue Dest = MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: Type);
1396 EmitAggregateAssign(Dest, Src: OriginalLVal, EltTy: Type);
1397 } else {
1398 EmitOMPAggregateAssign(
1399 DestAddr: Emission.getAllocatedAddress(), SrcAddr: OriginalLVal.getAddress(), OriginalType: Type,
1400 CopyGen: [this, VDInit, Init](Address DestElement, Address SrcElement) {
1401 // Clean up any temporaries needed by the
1402 // initialization.
1403 RunCleanupsScope InitScope(*this);
1404 // Emit initialization for single element.
1405 setAddrOfLocalVar(VD: VDInit, Addr: SrcElement);
1406 EmitAnyExprToMem(E: Init, Location: DestElement,
1407 Quals: Init->getType().getQualifiers(),
1408 /*IsInitializer*/ false);
1409 LocalDeclMap.erase(Val: VDInit);
1410 });
1411 }
1412 EmitAutoVarCleanups(emission: Emission);
1413 IsRegistered =
1414 PrivateScope.addPrivate(LocalVD: OrigVD, Addr: Emission.getAllocatedAddress());
1415 } else {
1416 Address OriginalAddr = OriginalLVal.getAddress();
1417 // Emit private VarDecl with copy init.
1418 // Remap temp VDInit variable to the address of the original
1419 // variable (for proper handling of captured global variables).
1420 setAddrOfLocalVar(VD: VDInit, Addr: OriginalAddr);
1421 EmitDecl(D: *VD);
1422 LocalDeclMap.erase(Val: VDInit);
1423 Address VDAddr = GetAddrOfLocalVar(VD);
1424 if (ThisFirstprivateIsLastprivate &&
1425 Lastprivates[OrigVD->getCanonicalDecl()] ==
1426 OMPC_LASTPRIVATE_conditional) {
1427 // Create/init special variable for lastprivate conditionals.
1428 llvm::Value *V =
1429 EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: VDAddr, T: (*IRef)->getType(),
1430 Source: AlignmentSource::Decl),
1431 Loc: (*IRef)->getExprLoc());
1432 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1433 CGF&: *this, VD: OrigVD);
1434 EmitStoreOfScalar(value: V, lvalue: MakeAddrLValue(Addr: VDAddr, T: (*IRef)->getType(),
1435 Source: AlignmentSource::Decl));
1436 LocalDeclMap.erase(Val: VD);
1437 setAddrOfLocalVar(VD, Addr: VDAddr);
1438 }
1439 IsRegistered = PrivateScope.addPrivate(LocalVD: OrigVD, Addr: VDAddr);
1440 }
1441 assert(IsRegistered &&
1442 "firstprivate var already registered as private");
1443 // Silence the warning about unused variable.
1444 (void)IsRegistered;
1445 }
1446 ++IRef;
1447 ++InitsRef;
1448 }
1449 }
1450 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1451}
1452
1453void CodeGenFunction::EmitOMPPrivateClause(
1454 const OMPExecutableDirective &D,
1455 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1456 if (!HaveInsertPoint())
1457 return;
1458 llvm::SmallDenseSet<const ValueDecl *> EmittedAsPrivate;
1459 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
1460 auto IRef = C->varlist_begin();
1461 for (const Expr *IInit : C->private_copies()) {
1462 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1463 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1464 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
1465 .second) {
1466 EmitDecl(D: *VD);
1467 bool IsRegistered =
1468 PrivateScope.addPrivate(LocalVD: OrigDecl, Addr: GetAddrOfLocalVar(VD));
1469 assert(IsRegistered && "private var already registered as private");
1470 (void)IsRegistered;
1471 }
1472 ++IRef;
1473 }
1474 }
1475}
1476
1477bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
1478 if (!HaveInsertPoint())
1479 return false;
1480 // threadprivate_var1 = master_threadprivate_var1;
1481 // operator=(threadprivate_var2, master_threadprivate_var2);
1482 // ...
1483 // __kmpc_barrier(&loc, global_tid);
1484 llvm::DenseSet<const VarDecl *> CopiedVars;
1485 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
1486 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
1487 auto IRef = C->varlist_begin();
1488 auto ISrcRef = C->source_exprs().begin();
1489 auto IDestRef = C->destination_exprs().begin();
1490 for (const Expr *AssignOp : C->assignment_ops()) {
1491 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
1492 QualType Type = VD->getType();
1493 if (CopiedVars.insert(V: VD->getCanonicalDecl()).second) {
1494 // Get the address of the master variable. If we are emitting code with
1495 // TLS support, the address is passed from the master as field in the
1496 // captured declaration.
1497 Address MasterAddr = Address::invalid();
1498 if (getLangOpts().OpenMPUseTLS &&
1499 getContext().getTargetInfo().isTLSSupported()) {
1500 assert(CapturedStmtInfo->lookup(VD) &&
1501 "Copyin threadprivates should have been captured!");
1502 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
1503 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1504 MasterAddr = EmitLValue(E: &DRE).getAddress();
1505 LocalDeclMap.erase(Val: VD);
1506 } else {
1507 MasterAddr =
1508 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(D: VD)
1509 : CGM.GetAddrOfGlobal(GD: VD),
1510 CGM.getTypes().ConvertTypeForMem(T: VD->getType()),
1511 getContext().getDeclAlign(D: VD));
1512 }
1513 // Get the address of the threadprivate variable.
1514 Address PrivateAddr = EmitLValue(E: *IRef).getAddress();
1515 if (CopiedVars.size() == 1) {
1516 // At first check if current thread is a master thread. If it is, no
1517 // need to copy data.
1518 CopyBegin = createBasicBlock(name: "copyin.not.master");
1519 CopyEnd = createBasicBlock(name: "copyin.not.master.end");
1520 // TODO: Avoid ptrtoint conversion.
1521 auto *MasterAddrInt = Builder.CreatePtrToInt(
1522 V: MasterAddr.emitRawPointer(CGF&: *this), DestTy: CGM.IntPtrTy);
1523 auto *PrivateAddrInt = Builder.CreatePtrToInt(
1524 V: PrivateAddr.emitRawPointer(CGF&: *this), DestTy: CGM.IntPtrTy);
1525 Builder.CreateCondBr(
1526 Cond: Builder.CreateICmpNE(LHS: MasterAddrInt, RHS: PrivateAddrInt), True: CopyBegin,
1527 False: CopyEnd);
1528 EmitBlock(BB: CopyBegin);
1529 }
1530 const auto *SrcVD =
1531 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1532 const auto *DestVD =
1533 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1534 EmitOMPCopy(OriginalType: Type, DestAddr: PrivateAddr, SrcAddr: MasterAddr, DestVD, SrcVD, Copy: AssignOp);
1535 }
1536 ++IRef;
1537 ++ISrcRef;
1538 ++IDestRef;
1539 }
1540 }
1541 if (CopyEnd) {
1542 // Exit out of copying procedure for non-master thread.
1543 EmitBlock(BB: CopyEnd, /*IsFinished=*/true);
1544 return true;
1545 }
1546 return false;
1547}
1548
1549bool CodeGenFunction::EmitOMPLastprivateClauseInit(
1550 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1551 if (!HaveInsertPoint())
1552 return false;
1553 bool HasAtLeastOneLastprivate = false;
1554 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1555 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1556 if (isOpenMPSimdDirective(DKind: EKind)) {
1557 const auto *LoopDirective = cast<OMPLoopDirective>(Val: &D);
1558 for (const Expr *C : LoopDirective->counters()) {
1559 SIMDLCVs.insert(
1560 V: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: C)->getDecl())->getCanonicalDecl());
1561 }
1562 }
1563 llvm::SmallDenseSet<const ValueDecl *> AlreadyEmittedVars;
1564 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1565 HasAtLeastOneLastprivate = true;
1566 if (isOpenMPTaskLoopDirective(DKind: EKind) && !getLangOpts().OpenMPSimd)
1567 break;
1568 const auto *IRef = C->varlist_begin();
1569 const auto *IDestRef = C->destination_exprs().begin();
1570 for (const Expr *IInit : C->private_copies()) {
1571 // Keep the address of the original variable for future update at the end
1572 // of the loop.
1573 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1574 // Handle BindingDecls with the same level of support as VarDecls.
1575 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
1576 if (AlreadyEmittedVars.insert(V: cast<ValueDecl>(Val: BD->getCanonicalDecl()))
1577 .second) {
1578 const auto *DestVD =
1579 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1580
1581 // Get the original binding address.
1582 Address OrigAddr =
1583 EmitOMPBindingOriginalAddr(BD, Loc: (*IRef)->getExprLoc());
1584 PrivateScope.addPrivate(LocalVD: DestVD, Addr: OrigAddr);
1585 if (IInit) {
1586 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1587 // Emit private VarDecl with copy init.
1588 EmitDecl(D: *VD);
1589 Address VDAddr = GetAddrOfLocalVar(VD);
1590 bool IsRegistered = PrivateScope.addPrivate(LocalVD: BD, Addr: VDAddr);
1591 assert(IsRegistered &&
1592 "lastprivate binding already registered as private");
1593 (void)IsRegistered;
1594 }
1595 }
1596 ++IRef;
1597 ++IDestRef;
1598 continue;
1599 }
1600 const auto *OrigVD = cast<VarDecl>(Val: OrigDecl);
1601 // Taskloops do not require additional initialization, it is done in
1602 // runtime support library.
1603 if (AlreadyEmittedVars.insert(V: OrigVD->getCanonicalDecl()).second) {
1604 const auto *DestVD =
1605 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1606 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1607 /*RefersToEnclosingVariableOrCapture=*/
1608 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
1609 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1610 PrivateScope.addPrivate(LocalVD: DestVD, Addr: EmitLValue(E: &DRE).getAddress());
1611 // Check if the variable is also a firstprivate: in this case IInit is
1612 // not generated. Initialization of this variable will happen in codegen
1613 // for 'firstprivate' clause.
1614 if (IInit && !SIMDLCVs.count(V: OrigVD->getCanonicalDecl())) {
1615 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1616 Address VDAddr = Address::invalid();
1617 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1618 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1619 CGF&: *this, VD: OrigVD);
1620 setAddrOfLocalVar(VD, Addr: VDAddr);
1621 } else {
1622 // Emit private VarDecl with copy init.
1623 EmitDecl(D: *VD);
1624 VDAddr = GetAddrOfLocalVar(VD);
1625 }
1626 bool IsRegistered = PrivateScope.addPrivate(LocalVD: OrigVD, Addr: VDAddr);
1627 assert(IsRegistered &&
1628 "lastprivate var already registered as private");
1629 (void)IsRegistered;
1630 }
1631 }
1632 ++IRef;
1633 ++IDestRef;
1634 }
1635 }
1636 return HasAtLeastOneLastprivate;
1637}
1638
1639void CodeGenFunction::EmitOMPLastprivateClauseFinal(
1640 const OMPExecutableDirective &D, bool NoFinals,
1641 llvm::Value *IsLastIterCond) {
1642 if (!HaveInsertPoint())
1643 return;
1644 // Emit following code:
1645 // if (<IsLastIterCond>) {
1646 // orig_var1 = private_orig_var1;
1647 // ...
1648 // orig_varn = private_orig_varn;
1649 // }
1650 llvm::BasicBlock *ThenBB = nullptr;
1651 llvm::BasicBlock *DoneBB = nullptr;
1652 if (IsLastIterCond) {
1653 // Emit implicit barrier if at least one lastprivate conditional is found
1654 // and this is not a simd mode.
1655 if (!getLangOpts().OpenMPSimd &&
1656 llvm::any_of(Range: D.getClausesOfKind<OMPLastprivateClause>(),
1657 P: [](const OMPLastprivateClause *C) {
1658 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1659 })) {
1660 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: D.getBeginLoc(),
1661 Kind: OMPD_unknown,
1662 /*EmitChecks=*/false,
1663 /*ForceSimpleCall=*/true);
1664 }
1665 ThenBB = createBasicBlock(name: ".omp.lastprivate.then");
1666 DoneBB = createBasicBlock(name: ".omp.lastprivate.done");
1667 Builder.CreateCondBr(Cond: IsLastIterCond, True: ThenBB, False: DoneBB);
1668 EmitBlock(BB: ThenBB);
1669 }
1670 llvm::DenseSet<const ValueDecl *> AlreadyEmittedVars;
1671 llvm::SmallDenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1672 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(Val: &D)) {
1673 auto IC = LoopDirective->counters().begin();
1674 for (const Expr *F : LoopDirective->finals()) {
1675 const auto *D =
1676 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IC)->getDecl())->getCanonicalDecl();
1677 if (NoFinals)
1678 AlreadyEmittedVars.insert(V: D);
1679 else
1680 LoopCountersAndUpdates[D] = F;
1681 ++IC;
1682 }
1683 }
1684 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1685 auto IRef = C->varlist_begin();
1686 auto ISrcRef = C->source_exprs().begin();
1687 auto IDestRef = C->destination_exprs().begin();
1688 for (const Expr *AssignOp : C->assignment_ops()) {
1689 const auto *PrivateDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1690
1691 // For BindingDecls, check if we should use .lastprivate.src or the BD
1692 // itself.
1693 const VarDecl *PrivateVD = nullptr;
1694 const BindingDecl *BD = nullptr;
1695 if ((BD = dyn_cast<BindingDecl>(Val: PrivateDecl))) {
1696 // Check if .lastprivate.src is available (taskloop case).
1697 const auto *SrcVD =
1698 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1699
1700 if (LocalDeclMap.count(Val: SrcVD)) {
1701 // Taskloop case: use .lastprivate.src.
1702 PrivateVD = SrcVD;
1703 } else if (OMPPrivatizedBindings.count(Val: BD)) {
1704 // Parallel for case: BindingDecl is directly privatized.
1705 // Leave PrivateVD as nullptr to handle specially below.
1706 }
1707 } else {
1708 PrivateVD = cast<VarDecl>(Val: PrivateDecl);
1709 }
1710
1711 QualType Type = PrivateVD ? PrivateVD->getType() : BD->getType();
1712 const auto *CanonicalVD =
1713 PrivateVD ? PrivateVD->getCanonicalDecl() : nullptr;
1714
1715 // Check if already emitted.
1716 bool ShouldEmit =
1717 AlreadyEmittedVars
1718 .insert(V: PrivateVD ? static_cast<const ValueDecl *>(CanonicalVD)
1719 : static_cast<const ValueDecl *>(BD))
1720 .second;
1721 if (ShouldEmit) {
1722 // If lastprivate variable is a loop control variable for loop-based
1723 // directive, update its value before copyin back to original
1724 // variable.
1725 if (CanonicalVD) {
1726 if (const Expr *FinalExpr =
1727 LoopCountersAndUpdates.lookup(Val: CanonicalVD))
1728 EmitIgnoredExpr(E: FinalExpr);
1729 }
1730 const auto *SrcVD =
1731 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1732 const auto *DestVD =
1733 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1734
1735 // Get the address of the private variable.
1736 Address PrivateAddr = Address::invalid();
1737 if (PrivateVD) {
1738 PrivateAddr = GetAddrOfLocalVar(VD: PrivateVD);
1739 } else {
1740 auto It = OMPPrivatizedBindings.find(Val: BD);
1741 assert(It != OMPPrivatizedBindings.end() &&
1742 "BindingDecl should be privatized");
1743 PrivateAddr = It->second;
1744 }
1745 if (PrivateVD) {
1746 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1747 PrivateAddr = Address(
1748 Builder.CreateLoad(Addr: PrivateAddr),
1749 CGM.getTypes().ConvertTypeForMem(T: RefTy->getPointeeType()),
1750 CGM.getNaturalTypeAlignment(T: RefTy->getPointeeType()));
1751 }
1752 // Store the last value to the private copy in the last iteration.
1753 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1754 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1755 CGF&: *this, PrivLVal: MakeAddrLValue(Addr: PrivateAddr, T: (*IRef)->getType()), VD: PrivateVD,
1756 Loc: (*IRef)->getExprLoc());
1757 // Get the address of the original variable.
1758 Address OriginalAddr = GetAddrOfLocalVar(VD: DestVD);
1759 EmitOMPCopy(OriginalType: Type, DestAddr: OriginalAddr, SrcAddr: PrivateAddr, DestVD, SrcVD, Copy: AssignOp);
1760 }
1761 ++IRef;
1762 ++ISrcRef;
1763 ++IDestRef;
1764 }
1765 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1766 EmitIgnoredExpr(E: PostUpdate);
1767 }
1768 if (IsLastIterCond)
1769 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
1770}
1771
1772void CodeGenFunction::EmitOMPReductionClauseInit(
1773 const OMPExecutableDirective &D,
1774 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1775 if (!HaveInsertPoint())
1776 return;
1777 SmallVector<const Expr *, 4> Shareds;
1778 SmallVector<const Expr *, 4> Privates;
1779 SmallVector<const Expr *, 4> ReductionOps;
1780 SmallVector<const Expr *, 4> LHSs;
1781 SmallVector<const Expr *, 4> RHSs;
1782 OMPTaskDataTy Data;
1783 SmallVector<const Expr *, 4> TaskLHSs;
1784 SmallVector<const Expr *, 4> TaskRHSs;
1785 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1786 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1787 continue;
1788 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
1789 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
1790 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
1791 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1792 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1793 if (C->getModifier() == OMPC_REDUCTION_task) {
1794 Data.ReductionVars.append(in_start: C->privates().begin(), in_end: C->privates().end());
1795 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
1796 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
1797 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
1798 in_end: C->reduction_ops().end());
1799 TaskLHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1800 TaskRHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1801 }
1802 }
1803 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1804 unsigned Count = 0;
1805 auto *ILHS = LHSs.begin();
1806 auto *IRHS = RHSs.begin();
1807 auto *IPriv = Privates.begin();
1808 for (const Expr *IRef : Shareds) {
1809 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IPriv)->getDecl());
1810 RedCG.emitSharedOrigLValue(CGF&: *this, N: Count);
1811 RedCG.emitAggregateType(CGF&: *this, N: Count);
1812 AutoVarEmission Emission = EmitAutoVarAlloca(var: *PrivateVD);
1813 RedCG.emitInitialization(CGF&: *this, N: Count, PrivateAddr: Emission.getAllocatedAddress(),
1814 SharedAddr: RedCG.getSharedLValue(N: Count).getAddress(),
1815 DefaultInit: [&Emission](CodeGenFunction &CGF) {
1816 CGF.EmitAutoVarInit(emission: Emission);
1817 return true;
1818 });
1819 EmitAutoVarCleanups(emission: Emission);
1820 Address BaseAddr = RedCG.adjustPrivateAddress(
1821 CGF&: *this, N: Count, PrivateAddr: Emission.getAllocatedAddress());
1822 bool IsRegistered =
1823 PrivateScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Count), Addr: BaseAddr);
1824 assert(IsRegistered && "private var already registered as private");
1825 // Silence the warning about unused variable.
1826 (void)IsRegistered;
1827
1828 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
1829 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
1830 QualType Type = PrivateVD->getType();
1831 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(Val: IRef);
1832 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1833 // Store the address of the original variable associated with the LHS
1834 // implicit variable.
1835 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: RedCG.getSharedLValue(N: Count).getAddress());
1836 PrivateScope.addPrivate(LocalVD: RHSVD, Addr: GetAddrOfLocalVar(VD: PrivateVD));
1837 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1838 isa<ArraySubscriptExpr>(Val: IRef)) {
1839 // Store the address of the original variable associated with the LHS
1840 // implicit variable.
1841 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: RedCG.getSharedLValue(N: Count).getAddress());
1842 PrivateScope.addPrivate(LocalVD: RHSVD,
1843 Addr: GetAddrOfLocalVar(VD: PrivateVD).withElementType(
1844 ElemTy: ConvertTypeForMem(T: RHSVD->getType())));
1845 } else {
1846 QualType Type = PrivateVD->getType();
1847 bool IsArray = getContext().getAsArrayType(T: Type) != nullptr;
1848 Address OriginalAddr = RedCG.getSharedLValue(N: Count).getAddress();
1849 // Store the address of the original variable associated with the LHS
1850 // implicit variable.
1851 if (IsArray) {
1852 OriginalAddr =
1853 OriginalAddr.withElementType(ElemTy: ConvertTypeForMem(T: LHSVD->getType()));
1854 }
1855 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: OriginalAddr);
1856 PrivateScope.addPrivate(
1857 LocalVD: RHSVD, Addr: IsArray ? GetAddrOfLocalVar(VD: PrivateVD).withElementType(
1858 ElemTy: ConvertTypeForMem(T: RHSVD->getType()))
1859 : GetAddrOfLocalVar(VD: PrivateVD));
1860 }
1861 ++ILHS;
1862 ++IRHS;
1863 ++IPriv;
1864 ++Count;
1865 }
1866 if (!Data.ReductionVars.empty()) {
1867 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1868 Data.IsReductionWithTaskMod = true;
1869 Data.IsWorksharingReduction = isOpenMPWorksharingDirective(DKind: EKind);
1870 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1871 CGF&: *this, Loc: D.getBeginLoc(), LHSExprs: TaskLHSs, RHSExprs: TaskRHSs, Data);
1872 const Expr *TaskRedRef = nullptr;
1873 switch (EKind) {
1874 case OMPD_parallel:
1875 TaskRedRef = cast<OMPParallelDirective>(Val: D).getTaskReductionRefExpr();
1876 break;
1877 case OMPD_for:
1878 TaskRedRef = cast<OMPForDirective>(Val: D).getTaskReductionRefExpr();
1879 break;
1880 case OMPD_sections:
1881 TaskRedRef = cast<OMPSectionsDirective>(Val: D).getTaskReductionRefExpr();
1882 break;
1883 case OMPD_parallel_for:
1884 TaskRedRef = cast<OMPParallelForDirective>(Val: D).getTaskReductionRefExpr();
1885 break;
1886 case OMPD_parallel_master:
1887 TaskRedRef =
1888 cast<OMPParallelMasterDirective>(Val: D).getTaskReductionRefExpr();
1889 break;
1890 case OMPD_parallel_sections:
1891 TaskRedRef =
1892 cast<OMPParallelSectionsDirective>(Val: D).getTaskReductionRefExpr();
1893 break;
1894 case OMPD_target_parallel:
1895 TaskRedRef =
1896 cast<OMPTargetParallelDirective>(Val: D).getTaskReductionRefExpr();
1897 break;
1898 case OMPD_target_parallel_for:
1899 TaskRedRef =
1900 cast<OMPTargetParallelForDirective>(Val: D).getTaskReductionRefExpr();
1901 break;
1902 case OMPD_distribute_parallel_for:
1903 TaskRedRef =
1904 cast<OMPDistributeParallelForDirective>(Val: D).getTaskReductionRefExpr();
1905 break;
1906 case OMPD_teams_distribute_parallel_for:
1907 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(Val: D)
1908 .getTaskReductionRefExpr();
1909 break;
1910 case OMPD_target_teams_distribute_parallel_for:
1911 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(Val: D)
1912 .getTaskReductionRefExpr();
1913 break;
1914 case OMPD_simd:
1915 case OMPD_for_simd:
1916 case OMPD_section:
1917 case OMPD_single:
1918 case OMPD_master:
1919 case OMPD_critical:
1920 case OMPD_parallel_for_simd:
1921 case OMPD_task:
1922 case OMPD_taskyield:
1923 case OMPD_error:
1924 case OMPD_barrier:
1925 case OMPD_taskwait:
1926 case OMPD_taskgroup:
1927 case OMPD_flush:
1928 case OMPD_depobj:
1929 case OMPD_scan:
1930 case OMPD_ordered_standalone:
1931 case OMPD_ordered_blockassoc:
1932 case OMPD_atomic:
1933 case OMPD_teams:
1934 case OMPD_target:
1935 case OMPD_cancellation_point:
1936 case OMPD_cancel:
1937 case OMPD_target_data:
1938 case OMPD_target_enter_data:
1939 case OMPD_target_exit_data:
1940 case OMPD_taskloop:
1941 case OMPD_taskloop_simd:
1942 case OMPD_master_taskloop:
1943 case OMPD_master_taskloop_simd:
1944 case OMPD_parallel_master_taskloop:
1945 case OMPD_parallel_master_taskloop_simd:
1946 case OMPD_distribute:
1947 case OMPD_target_update:
1948 case OMPD_distribute_parallel_for_simd:
1949 case OMPD_distribute_simd:
1950 case OMPD_target_parallel_for_simd:
1951 case OMPD_target_simd:
1952 case OMPD_teams_distribute:
1953 case OMPD_teams_distribute_simd:
1954 case OMPD_teams_distribute_parallel_for_simd:
1955 case OMPD_target_teams:
1956 case OMPD_target_teams_distribute:
1957 case OMPD_target_teams_distribute_parallel_for_simd:
1958 case OMPD_target_teams_distribute_simd:
1959 case OMPD_declare_target:
1960 case OMPD_end_declare_target:
1961 case OMPD_threadprivate:
1962 case OMPD_allocate:
1963 case OMPD_declare_reduction:
1964 case OMPD_declare_mapper:
1965 case OMPD_declare_simd:
1966 case OMPD_requires:
1967 case OMPD_declare_variant:
1968 case OMPD_begin_declare_variant:
1969 case OMPD_end_declare_variant:
1970 case OMPD_unknown:
1971 default:
1972 llvm_unreachable("Unexpected directive with task reductions.");
1973 }
1974
1975 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: TaskRedRef)->getDecl());
1976 EmitVarDecl(D: *VD);
1977 EmitStoreOfScalar(Value: ReductionDesc, Addr: GetAddrOfLocalVar(VD),
1978 /*Volatile=*/false, Ty: TaskRedRef->getType());
1979 }
1980}
1981
1982void CodeGenFunction::EmitOMPReductionClauseFinal(
1983 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1984 if (!HaveInsertPoint())
1985 return;
1986 llvm::SmallVector<const Expr *, 8> Privates;
1987 llvm::SmallVector<const Expr *, 8> LHSExprs;
1988 llvm::SmallVector<const Expr *, 8> RHSExprs;
1989 llvm::SmallVector<const Expr *, 8> ReductionOps;
1990 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1991 bool HasAtLeastOneReduction = false;
1992 bool IsReductionWithTaskMod = false;
1993 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1994 // Do not emit for inscan reductions.
1995 if (C->getModifier() == OMPC_REDUCTION_inscan)
1996 continue;
1997 HasAtLeastOneReduction = true;
1998 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
1999 LHSExprs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
2000 RHSExprs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
2001 IsPrivateVarReduction.append(in_start: C->private_var_reduction_flags().begin(),
2002 in_end: C->private_var_reduction_flags().end());
2003 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
2004 IsReductionWithTaskMod =
2005 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
2006 }
2007 if (HasAtLeastOneReduction) {
2008 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2009 if (IsReductionWithTaskMod) {
2010 CGM.getOpenMPRuntime().emitTaskReductionFini(
2011 CGF&: *this, Loc: D.getBeginLoc(), IsWorksharingReduction: isOpenMPWorksharingDirective(DKind: EKind));
2012 }
2013 bool TeamsLoopCanBeParallel = false;
2014 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(Val: &D))
2015 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
2016 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
2017 isOpenMPParallelDirective(DKind: EKind) ||
2018 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
2019 bool SimpleReduction = ReductionKind == OMPD_simd;
2020 // Emit nowait reduction if nowait clause is present or directive is a
2021 // parallel directive (it always has implicit barrier).
2022 CGM.getOpenMPRuntime().emitReduction(
2023 CGF&: *this, Loc: D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
2024 Options: {.WithNowait: WithNowait, .SimpleReduction: SimpleReduction, .IsPrivateVarReduction: IsPrivateVarReduction, .ReductionKind: ReductionKind});
2025 }
2026}
2027
2028static void emitPostUpdateForReductionClause(
2029 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2030 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2031 if (!CGF.HaveInsertPoint())
2032 return;
2033 llvm::BasicBlock *DoneBB = nullptr;
2034 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
2035 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
2036 if (!DoneBB) {
2037 if (llvm::Value *Cond = CondGen(CGF)) {
2038 // If the first post-update expression is found, emit conditional
2039 // block if it was requested.
2040 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: ".omp.reduction.pu");
2041 DoneBB = CGF.createBasicBlock(name: ".omp.reduction.pu.done");
2042 CGF.Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
2043 CGF.EmitBlock(BB: ThenBB);
2044 }
2045 }
2046 CGF.EmitIgnoredExpr(E: PostUpdate);
2047 }
2048 }
2049 if (DoneBB)
2050 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
2051}
2052
2053namespace {
2054/// Codegen lambda for appending distribute lower and upper bounds to outlined
2055/// parallel function. This is necessary for combined constructs such as
2056/// 'distribute parallel for'
2057typedef llvm::function_ref<void(CodeGenFunction &,
2058 const OMPExecutableDirective &,
2059 llvm::SmallVectorImpl<llvm::Value *> &)>
2060 CodeGenBoundParametersTy;
2061} // anonymous namespace
2062
2063static void
2064checkForLastprivateConditionalUpdate(CodeGenFunction &CGF,
2065 const OMPExecutableDirective &S) {
2066 if (CGF.getLangOpts().OpenMP < 50)
2067 return;
2068 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
2069 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2070 for (const Expr *Ref : C->varlist()) {
2071 if (!Ref->getType()->isScalarType())
2072 continue;
2073 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2074 if (!DRE)
2075 continue;
2076 // Skip BindingDecls - lastprivate conditional only applies to VarDecls.
2077 const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl());
2078 if (!VD)
2079 continue;
2080 PrivateDecls.insert(V: VD);
2081 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
2082 }
2083 }
2084 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2085 for (const Expr *Ref : C->varlist()) {
2086 if (!Ref->getType()->isScalarType())
2087 continue;
2088 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2089 if (!DRE)
2090 continue;
2091 // Skip BindingDecls - they don't use the same conditional lastprivate
2092 // mechanism.
2093 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
2094 PrivateDecls.insert(V: VD);
2095 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
2096 }
2097 }
2098 }
2099 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
2100 for (const Expr *Ref : C->varlist()) {
2101 if (!Ref->getType()->isScalarType())
2102 continue;
2103 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2104 if (!DRE)
2105 continue;
2106 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
2107 PrivateDecls.insert(V: VD);
2108 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
2109 }
2110 }
2111 }
2112 // Privates should ne analyzed since they are not captured at all.
2113 // Task reductions may be skipped - tasks are ignored.
2114 // Firstprivates do not return value but may be passed by reference - no need
2115 // to check for updated lastprivate conditional.
2116 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2117 for (const Expr *Ref : C->varlist()) {
2118 if (!Ref->getType()->isScalarType())
2119 continue;
2120 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2121 if (!DRE)
2122 continue;
2123 // Only track VarDecl, not BindingDecl.
2124 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl()))
2125 PrivateDecls.insert(V: VD);
2126 }
2127 }
2128 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional(
2129 CGF, D: S, IgnoredDecls: PrivateDecls);
2130}
2131
2132static void emitCommonOMPParallelDirective(
2133 CodeGenFunction &CGF, const OMPExecutableDirective &S,
2134 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
2135 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
2136 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
2137 llvm::Value *NumThreads = nullptr;
2138 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown;
2139 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is as
2140 // if sev-level is fatal."
2141 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal;
2142 clang::Expr *Message = nullptr;
2143 SourceLocation SeverityLoc = SourceLocation();
2144 SourceLocation MessageLoc = SourceLocation();
2145
2146 llvm::Function *OutlinedFn =
2147 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
2148 CGF, D: S, ThreadIDVar: *CS->getCapturedDecl()->param_begin(), InnermostKind,
2149 CodeGen);
2150
2151 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
2152 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
2153 NumThreads = CGF.EmitScalarExpr(E: NumThreadsClause->getNumThreads().front(),
2154 /*IgnoreResultAssign=*/true);
2155 Modifier = NumThreadsClause->getPrescriptivenessModifier();
2156 if (const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {
2157 Message = MessageClause->getMessageString();
2158 MessageLoc = MessageClause->getBeginLoc();
2159 }
2160 if (const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {
2161 Severity = SeverityClause->getSeverityKind();
2162 SeverityLoc = SeverityClause->getBeginLoc();
2163 }
2164 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
2165 CGF, NumThreads, Loc: NumThreadsClause->getBeginLoc(), Modifier, Severity,
2166 SeverityLoc, Message, MessageLoc);
2167 }
2168 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
2169 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
2170 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
2171 CGF, ProcBind: ProcBindClause->getProcBindKind(), Loc: ProcBindClause->getBeginLoc());
2172 }
2173 const Expr *IfCond = nullptr;
2174 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2175 if (C->getNameModifier() == OMPD_unknown ||
2176 C->getNameModifier() == OMPD_parallel) {
2177 IfCond = C->getCondition();
2178 break;
2179 }
2180 }
2181
2182 OMPParallelScope Scope(CGF, S);
2183 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
2184 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
2185 // lower and upper bounds with the pragma 'for' chunking mechanism.
2186 // The following lambda takes care of appending the lower and upper bound
2187 // parameters when necessary
2188 CodeGenBoundParameters(CGF, S, CapturedVars);
2189 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
2190 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, Loc: S.getBeginLoc(), OutlinedFn,
2191 CapturedVars, IfCond, NumThreads,
2192 NumThreadsModifier: Modifier, Severity, Message);
2193}
2194
2195static bool isAllocatableDecl(const VarDecl *VD) {
2196 const VarDecl *CVD = VD->getCanonicalDecl();
2197 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
2198 return false;
2199 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
2200 // Use the default allocation.
2201 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
2202 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
2203 !AA->getAllocator());
2204}
2205
2206static void emitEmptyBoundParameters(CodeGenFunction &,
2207 const OMPExecutableDirective &,
2208 llvm::SmallVectorImpl<llvm::Value *> &) {}
2209
2210static void emitOMPCopyinClause(CodeGenFunction &CGF,
2211 const OMPExecutableDirective &S) {
2212 bool Copyins = CGF.EmitOMPCopyinClause(D: S);
2213 if (Copyins) {
2214 // Emit implicit barrier to synchronize threads and avoid data races on
2215 // propagation master's thread values of threadprivate variables to local
2216 // instances of that variables of all other implicit threads.
2217 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2218 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
2219 /*ForceSimpleCall=*/true);
2220 }
2221}
2222
2223Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable(
2224 CodeGenFunction &CGF, const VarDecl *VD) {
2225 CodeGenModule &CGM = CGF.CGM;
2226 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2227
2228 // Unreachable declaration: nothing can run the allocation.
2229 if (!VD || !CGF.HaveInsertPoint())
2230 return Address::invalid();
2231 const VarDecl *CVD = VD->getCanonicalDecl();
2232 if (!isAllocatableDecl(VD: CVD))
2233 return Address::invalid();
2234 llvm::Value *Size;
2235 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
2236 if (CVD->getType()->isVariablyModifiedType()) {
2237 Size = CGF.getTypeSize(Ty: CVD->getType());
2238 // Align the size: ((size + align - 1) / align) * align
2239 Size = CGF.Builder.CreateNUWAdd(
2240 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
2241 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
2242 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
2243 } else {
2244 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
2245 Size = CGM.getSize(numChars: Sz.alignTo(Align));
2246 }
2247
2248 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
2249 assert(AA->getAllocator() &&
2250 "Expected allocator expression for non-default allocator.");
2251 llvm::Value *Allocator = CGF.EmitScalarExpr(E: AA->getAllocator());
2252 // According to the standard, the original allocator type is a enum (integer).
2253 // Convert to pointer type, if required.
2254 if (Allocator->getType()->isIntegerTy())
2255 Allocator = CGF.Builder.CreateIntToPtr(V: Allocator, DestTy: CGM.VoidPtrTy);
2256 else if (Allocator->getType()->isPointerTy())
2257 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: Allocator,
2258 DestTy: CGM.VoidPtrTy);
2259
2260 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
2261 Loc: CGF.Builder, Size, Allocator,
2262 Name: getNameWithSeparators(Parts: {CVD->getName(), ".void.addr"}, FirstSeparator: ".", Separator: "."));
2263 llvm::CallInst *FreeCI =
2264 OMPBuilder.createOMPFree(Loc: CGF.Builder, Addr, Allocator);
2265
2266 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(Kind: NormalAndEHCleanup, A: FreeCI);
2267 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2268 V: Addr,
2269 DestTy: CGF.ConvertTypeForMem(T: CGM.getContext().getPointerType(T: CVD->getType())),
2270 Name: getNameWithSeparators(Parts: {CVD->getName(), ".addr"}, FirstSeparator: ".", Separator: "."));
2271 return Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
2272}
2273
2274Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2275 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
2276 SourceLocation Loc) {
2277 CodeGenModule &CGM = CGF.CGM;
2278 if (CGM.getLangOpts().OpenMPUseTLS &&
2279 CGM.getContext().getTargetInfo().isTLSSupported())
2280 return VDAddr;
2281
2282 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2283
2284 llvm::Type *VarTy = VDAddr.getElementType();
2285 llvm::Value *Data =
2286 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy);
2287 llvm::ConstantInt *Size = CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy));
2288 std::string Suffix = getNameWithSeparators(Parts: {"cache", ""});
2289 llvm::Twine CacheName = Twine(CGM.getMangledName(GD: VD)).concat(Suffix);
2290
2291 llvm::CallInst *ThreadPrivateCacheCall =
2292 OMPBuilder.createCachedThreadPrivate(Loc: CGF.Builder, Pointer: Data, Size, Name: CacheName);
2293
2294 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
2295}
2296
2297std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(
2298 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
2299 SmallString<128> Buffer;
2300 llvm::raw_svector_ostream OS(Buffer);
2301 StringRef Sep = FirstSeparator;
2302 for (StringRef Part : Parts) {
2303 OS << Sep << Part;
2304 Sep = Separator;
2305 }
2306 return OS.str().str();
2307}
2308
2309void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
2310 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2311 InsertPointTy CodeGenIP, Twine RegionName) {
2312 CGBuilderTy &Builder = CGF.Builder;
2313 Builder.restoreIP(IP: CodeGenIP);
2314 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2315 Suffix: "." + RegionName + ".after");
2316
2317 {
2318 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2319 CGF.EmitStmt(S: RegionBodyStmt);
2320 }
2321
2322 if (Builder.saveIP().isValid())
2323 Builder.CreateBr(Dest: FiniBB);
2324}
2325
2326void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
2327 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2328 InsertPointTy CodeGenIP, Twine RegionName) {
2329 CGBuilderTy &Builder = CGF.Builder;
2330 Builder.restoreIP(IP: CodeGenIP);
2331 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2332 Suffix: "." + RegionName + ".after");
2333
2334 {
2335 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2336 CGF.EmitStmt(S: RegionBodyStmt);
2337 }
2338
2339 if (Builder.saveIP().isValid())
2340 Builder.CreateBr(Dest: FiniBB);
2341}
2342
2343void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
2344 if (CGM.getLangOpts().OpenMPIRBuilder) {
2345 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2346 // Check if we have any if clause associated with the directive.
2347 llvm::Value *IfCond = nullptr;
2348 if (const auto *C = S.getSingleClause<OMPIfClause>())
2349 IfCond = EmitScalarExpr(E: C->getCondition(),
2350 /*IgnoreResultAssign=*/true);
2351
2352 llvm::Value *NumThreads = nullptr;
2353 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
2354 NumThreads = EmitScalarExpr(E: NumThreadsClause->getNumThreads().front(),
2355 /*IgnoreResultAssign=*/true);
2356
2357 ProcBindKind ProcBind = OMP_PROC_BIND_default;
2358 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
2359 ProcBind = ProcBindClause->getProcBindKind();
2360
2361 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2362
2363 // The cleanup callback that finalizes all variables at the given location,
2364 // thus calls destructors etc.
2365 auto FiniCB = [this](InsertPointTy IP) {
2366 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
2367 return llvm::Error::success();
2368 };
2369
2370 // Privatization callback that performs appropriate action for
2371 // shared/private/firstprivate/lastprivate/copyin/... variables.
2372 //
2373 // TODO: This defaults to shared right now.
2374 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
2375 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
2376 // The next line is appropriate only for variables (Val) with the
2377 // data-sharing attribute "shared".
2378 ReplVal = &Val;
2379
2380 return CodeGenIP;
2381 };
2382
2383 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
2384 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
2385
2386 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
2387 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
2388 OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
2389 CGF&: *this, RegionBodyStmt: ParallelRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "parallel");
2390 return llvm::Error::success();
2391 };
2392
2393 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
2394 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
2395 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
2396 AllocaInsertPt->getIterator());
2397 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2398 cantFail(ValOrErr: OMPBuilder.createParallel(
2399 Loc: Builder, AllocaIP, /*DeallocBlocks=*/{}, BodyGenCB, PrivCB, FiniCB,
2400 IfCondition: IfCond, NumThreads, ProcBind, IsCancellable: S.hasCancel()));
2401 Builder.restoreIP(IP: AfterIP);
2402 return;
2403 }
2404
2405 // Emit parallel region as a standalone region.
2406 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2407 Action.Enter(CGF);
2408 OMPPrivateScope PrivateScope(CGF);
2409 emitOMPCopyinClause(CGF, S);
2410 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
2411 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
2412 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
2413 (void)PrivateScope.Privatize();
2414 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_parallel)->getCapturedStmt());
2415 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
2416 };
2417 {
2418 auto LPCRegion =
2419 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
2420 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_parallel, CodeGen,
2421 CodeGenBoundParameters: emitEmptyBoundParameters);
2422 emitPostUpdateForReductionClause(CGF&: *this, D: S,
2423 CondGen: [](CodeGenFunction &) { return nullptr; });
2424 }
2425 // Check for outer lastprivate conditional update.
2426 checkForLastprivateConditionalUpdate(CGF&: *this, S);
2427}
2428
2429void CodeGenFunction::EmitOMPMetaDirective(const OMPMetaDirective &S) {
2430 // TODO: Select the variant at runtime.
2431 EmitStmt(S: S.variants().front());
2432}
2433
2434namespace {
2435/// RAII to handle scopes for loop transformation directives.
2436class OMPTransformDirectiveScopeRAII {
2437 OMPLoopScope *Scope = nullptr;
2438 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
2439 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
2440
2441 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
2442 delete;
2443 OMPTransformDirectiveScopeRAII &
2444 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
2445
2446public:
2447 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
2448 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(Val: S)) {
2449 Scope = new OMPLoopScope(CGF, *Dir);
2450 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
2451 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2452 } else if (const auto *Dir =
2453 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(
2454 Val: S)) {
2455 // For simplicity we reuse the loop scope similarly to what we do with
2456 // OMPCanonicalLoopNestTransformationDirective do by being a subclass
2457 // of OMPLoopBasedDirective.
2458 Scope = new OMPLoopScope(CGF, *Dir);
2459 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
2460 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2461 }
2462 }
2463 ~OMPTransformDirectiveScopeRAII() {
2464 if (!Scope)
2465 return;
2466 delete CapInfoRAII;
2467 delete CGSI;
2468 delete Scope;
2469 }
2470};
2471} // namespace
2472
2473static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
2474 int MaxLevel, int Level = 0) {
2475 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
2476 const Stmt *SimplifiedS = S->IgnoreContainers();
2477 if (const auto *CS = dyn_cast<CompoundStmt>(Val: SimplifiedS)) {
2478 PrettyStackTraceLoc CrashInfo(
2479 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
2480 "LLVM IR generation of compound statement ('{}')");
2481
2482 // Keep track of the current cleanup stack depth, including debug scopes.
2483 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
2484 for (const Stmt *CurStmt : CS->body())
2485 emitBody(CGF, S: CurStmt, NextLoop, MaxLevel, Level);
2486 return;
2487 }
2488
2489 // `tryToFindNextInnerLoop` keeps the intra-tile hint wrapper around, so match
2490 // against the loop it annotates. The tile overshoot guard is emitted
2491 // separately via EmitOMPLoopBody's finals-conditions handling.
2492 if (SimplifiedS == OMPLoopBasedDirective::ignoreIntraTileHint(S: NextLoop)) {
2493 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(Val: SimplifiedS))
2494 SimplifiedS = Dir->getTransformedStmt();
2495 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Val: SimplifiedS))
2496 SimplifiedS = CanonLoop->getLoopStmt();
2497 if (const auto *For = dyn_cast<ForStmt>(Val: SimplifiedS)) {
2498 S = For->getBody();
2499 } else {
2500 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
2501 "Expected canonical for loop or range-based for loop.");
2502 const auto *CXXFor = cast<CXXForRangeStmt>(Val: SimplifiedS);
2503 CGF.EmitStmt(S: CXXFor->getLoopVarStmt());
2504 S = CXXFor->getBody();
2505 }
2506 if (Level + 1 < MaxLevel) {
2507 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2508 CurStmt: S, /*TryImperfectlyNestedLoops=*/true);
2509 emitBody(CGF, S, NextLoop, MaxLevel, Level: Level + 1);
2510 return;
2511 }
2512 }
2513 CGF.EmitStmt(S);
2514}
2515
2516void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
2517 JumpDest LoopExit) {
2518 RunCleanupsScope BodyScope(*this);
2519 // Update counters values on current iteration.
2520 for (const Expr *UE : D.updates())
2521 EmitIgnoredExpr(E: UE);
2522 // Update the linear variables.
2523 // In distribute directives only loop counters may be marked as linear, no
2524 // need to generate the code for them.
2525 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2526 if (!isOpenMPDistributeDirective(DKind: EKind)) {
2527 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2528 for (const Expr *UE : C->updates())
2529 EmitIgnoredExpr(E: UE);
2530 }
2531 }
2532
2533 // On a continue in the body, jump to the end.
2534 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.body.continue");
2535 BreakContinueStack.push_back(Elt: BreakContinue(D, LoopExit, Continue));
2536 for (const Expr *E : D.finals_conditions()) {
2537 if (!E)
2538 continue;
2539 // Check that loop counter in non-rectangular nest fits into the iteration
2540 // space.
2541 llvm::BasicBlock *NextBB = createBasicBlock(name: "omp.body.next");
2542 EmitBranchOnBoolExpr(Cond: E, TrueBlock: NextBB, FalseBlock: Continue.getBlock(),
2543 TrueCount: getProfileCount(S: D.getBody()));
2544 EmitBlock(BB: NextBB);
2545 }
2546
2547 OMPPrivateScope InscanScope(*this);
2548 EmitOMPReductionClauseInit(D, PrivateScope&: InscanScope, /*ForInscan=*/true);
2549 bool IsInscanRegion = InscanScope.Privatize();
2550 if (IsInscanRegion) {
2551 // Need to remember the block before and after scan directive
2552 // to dispatch them correctly depending on the clause used in
2553 // this directive, inclusive or exclusive. For inclusive scan the natural
2554 // order of the blocks is used, for exclusive clause the blocks must be
2555 // executed in reverse order.
2556 OMPBeforeScanBlock = createBasicBlock(name: "omp.before.scan.bb");
2557 OMPAfterScanBlock = createBasicBlock(name: "omp.after.scan.bb");
2558 // No need to allocate inscan exit block, in simd mode it is selected in the
2559 // codegen for the scan directive.
2560 if (EKind != OMPD_simd && !getLangOpts().OpenMPSimd)
2561 OMPScanExitBlock = createBasicBlock(name: "omp.exit.inscan.bb");
2562 OMPScanDispatch = createBasicBlock(name: "omp.inscan.dispatch");
2563 EmitBranch(Block: OMPScanDispatch);
2564 EmitBlock(BB: OMPBeforeScanBlock);
2565 }
2566
2567 // Emit loop variables for C++ range loops.
2568 const Stmt *Body =
2569 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2570 // Emit loop body.
2571 emitBody(CGF&: *this, S: Body,
2572 NextLoop: OMPLoopBasedDirective::tryToFindNextInnerLoop(
2573 CurStmt: Body, /*TryImperfectlyNestedLoops=*/true),
2574 MaxLevel: D.getLoopsNumber());
2575
2576 // Jump to the dispatcher at the end of the loop body.
2577 if (IsInscanRegion)
2578 EmitBranch(Block: OMPScanExitBlock);
2579
2580 // The end (updates/cleanups).
2581 EmitBlock(BB: Continue.getBlock());
2582 BreakContinueStack.pop_back();
2583}
2584
2585using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
2586
2587/// Emit a captured statement and return the function as well as its captured
2588/// closure context.
2589static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF,
2590 const CapturedStmt *S) {
2591 LValue CapStruct = ParentCGF.InitCapturedStruct(S: *S);
2592 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
2593 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2594 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(args: *S);
2595 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
2596 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S: *S);
2597
2598 return {F, CapStruct.getPointer(CGF&: ParentCGF)};
2599}
2600
2601/// Emit a call to a previously captured closure.
2602static llvm::CallInst *
2603emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap,
2604 llvm::ArrayRef<llvm::Value *> Args) {
2605 // Append the closure context to the argument.
2606 SmallVector<llvm::Value *> EffectiveArgs;
2607 EffectiveArgs.reserve(N: Args.size() + 1);
2608 llvm::append_range(C&: EffectiveArgs, R&: Args);
2609 EffectiveArgs.push_back(Elt: Cap.second);
2610
2611 return ParentCGF.Builder.CreateCall(Callee: Cap.first, Args: EffectiveArgs);
2612}
2613
2614llvm::CanonicalLoopInfo *
2615CodeGenFunction::EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth) {
2616 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
2617
2618 // The caller is processing the loop-associated directive processing the \p
2619 // Depth loops nested in \p S. Put the previous pending loop-associated
2620 // directive to the stack. If the current loop-associated directive is a loop
2621 // transformation directive, it will push its generated loops onto the stack
2622 // such that together with the loops left here they form the combined loop
2623 // nest for the parent loop-associated directive.
2624 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
2625 ExpectedOMPLoopDepth = Depth;
2626
2627 EmitStmt(S);
2628 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
2629
2630 // The last added loop is the outermost one.
2631 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
2632
2633 // Pop the \p Depth loops requested by the call from that stack and restore
2634 // the previous context.
2635 OMPLoopNestStack.pop_back_n(NumItems: Depth);
2636 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2637
2638 return Result;
2639}
2640
2641void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
2642 const Stmt *SyntacticalLoop = S->getLoopStmt();
2643 if (!getLangOpts().OpenMPIRBuilder) {
2644 // Ignore if OpenMPIRBuilder is not enabled.
2645 EmitStmt(S: SyntacticalLoop);
2646 return;
2647 }
2648
2649 LexicalScope ForScope(*this, S->getSourceRange());
2650
2651 // Emit init statements. The Distance/LoopVar funcs may reference variable
2652 // declarations they contain.
2653 const Stmt *BodyStmt;
2654 if (const auto *For = dyn_cast<ForStmt>(Val: SyntacticalLoop)) {
2655 if (const Stmt *InitStmt = For->getInit())
2656 EmitStmt(S: InitStmt);
2657 BodyStmt = For->getBody();
2658 } else if (const auto *RangeFor =
2659 dyn_cast<CXXForRangeStmt>(Val: SyntacticalLoop)) {
2660 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2661 EmitStmt(S: RangeStmt);
2662 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2663 EmitStmt(S: BeginStmt);
2664 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2665 EmitStmt(S: EndStmt);
2666 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2667 EmitStmt(S: LoopVarStmt);
2668 BodyStmt = RangeFor->getBody();
2669 } else
2670 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2671
2672 // Emit closure for later use. By-value captures will be captured here.
2673 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2674 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(ParentCGF&: *this, S: DistanceFunc);
2675 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2676 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(ParentCGF&: *this, S: LoopVarFunc);
2677
2678 // Call the distance function to get the number of iterations of the loop to
2679 // come.
2680 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2681 ->getParam(i: 0)
2682 ->getType()
2683 .getNonReferenceType();
2684 RawAddress CountAddr = CreateMemTemp(T: LogicalTy, Name: ".count.addr");
2685 emitCapturedStmtCall(ParentCGF&: *this, Cap: DistanceClosure, Args: {CountAddr.getPointer()});
2686 llvm::Value *DistVal = Builder.CreateLoad(Addr: CountAddr, Name: ".count");
2687
2688 // Emit the loop structure.
2689 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2690 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2691 llvm::Value *IndVar) {
2692 Builder.restoreIP(IP: CodeGenIP);
2693
2694 // Emit the loop body: Convert the logical iteration number to the loop
2695 // variable and emit the body.
2696 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2697 LValue LCVal = EmitLValue(E: LoopVarRef);
2698 Address LoopVarAddress = LCVal.getAddress();
2699 emitCapturedStmtCall(ParentCGF&: *this, Cap: LoopVarClosure,
2700 Args: {LoopVarAddress.emitRawPointer(CGF&: *this), IndVar});
2701
2702 RunCleanupsScope BodyScope(*this);
2703 EmitStmt(S: BodyStmt);
2704 return llvm::Error::success();
2705 };
2706
2707 llvm::CanonicalLoopInfo *CL =
2708 cantFail(ValOrErr: OMPBuilder.createCanonicalLoop(Loc: Builder, BodyGenCB: BodyGen, TripCount: DistVal));
2709
2710 // Finish up the loop.
2711 Builder.restoreIP(IP: CL->getAfterIP());
2712 ForScope.ForceCleanup();
2713
2714 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2715 OMPLoopNestStack.push_back(Elt: CL);
2716}
2717
2718/// A directive's own source range covers only the pragma, so use the range of
2719/// the associated loop, looking through loop transformation directives.
2720static SourceRange getAssociatedLoopRange(const OMPExecutableDirective &S) {
2721 if (!isa<OMPLoopDirective>(Val: S))
2722 return S.getSourceRange();
2723 const Stmt *Loop = S.getRawStmt()->IgnoreContainers();
2724 while (isa<OMPLoopTransformationDirective>(Val: Loop))
2725 Loop = cast<OMPExecutableDirective>(Val: Loop)->getRawStmt()->IgnoreContainers();
2726 return Loop->getSourceRange();
2727}
2728
2729void CodeGenFunction::EmitOMPInnerLoop(
2730 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2731 const Expr *IncExpr,
2732 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2733 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2734 auto LoopExit = getJumpDestInCurrentScope(Name: "omp.inner.for.end");
2735
2736 // Start the loop with a block that tests the condition.
2737 auto CondBlock = createBasicBlock(name: "omp.inner.for.cond");
2738 EmitBlock(BB: CondBlock);
2739 const SourceRange R = getAssociatedLoopRange(S);
2740
2741 // If attributes are attached, push to the basic block with them.
2742 const auto &OMPED = cast<OMPExecutableDirective>(Val: S);
2743 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2744 const Stmt *SS = ICS->getCapturedStmt();
2745 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(Val: SS);
2746 OMPLoopNestStack.clear();
2747 if (AS)
2748 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
2749 Attrs: AS->getAttrs(), StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
2750 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
2751 else
2752 LoopStack.push(Header: CondBlock, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
2753 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
2754
2755 // If there are any cleanups between here and the loop-exit scope,
2756 // create a block to stage a loop exit along.
2757 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2758 if (RequiresCleanup)
2759 ExitBlock = createBasicBlock(name: "omp.inner.for.cond.cleanup");
2760
2761 llvm::BasicBlock *LoopBody = createBasicBlock(name: "omp.inner.for.body");
2762
2763 // Emit condition.
2764 EmitBranchOnBoolExpr(Cond: LoopCond, TrueBlock: LoopBody, FalseBlock: ExitBlock, TrueCount: getProfileCount(S: &S));
2765 if (ExitBlock != LoopExit.getBlock()) {
2766 EmitBlock(BB: ExitBlock);
2767 EmitBranchThroughCleanup(Dest: LoopExit);
2768 }
2769
2770 EmitBlock(BB: LoopBody);
2771 incrementProfileCounter(S: &S);
2772
2773 // Create a block for the increment.
2774 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.inner.for.inc");
2775 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
2776
2777 BodyGen(*this);
2778
2779 // Emit "IV = IV + 1" and a back-edge to the condition block.
2780 EmitBlock(BB: Continue.getBlock());
2781 EmitIgnoredExpr(E: IncExpr);
2782 PostIncGen(*this);
2783 BreakContinueStack.pop_back();
2784 EmitBranch(Block: CondBlock);
2785 LoopStack.pop();
2786 // Emit the fall-through block.
2787 EmitBlock(BB: LoopExit.getBlock());
2788}
2789
2790bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
2791 if (!HaveInsertPoint())
2792 return false;
2793 // Emit inits for the linear variables.
2794 bool HasLinears = false;
2795 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2796 for (const Expr *Init : C->inits()) {
2797 HasLinears = true;
2798 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Init)->getDecl());
2799 if (const auto *Ref =
2800 dyn_cast<DeclRefExpr>(Val: VD->getInit()->IgnoreImpCasts())) {
2801 if (isa<BindingDecl>(Val: Ref->getDecl())) {
2802 EmitVarDecl(D: *VD);
2803 } else {
2804 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
2805 const auto *OrigVD = cast<VarDecl>(Val: Ref->getDecl());
2806 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2807 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
2808 VD->getInit()->getType(), VK_LValue,
2809 VD->getInit()->getExprLoc());
2810 EmitExprAsInit(
2811 init: &DRE, D: VD,
2812 lvalue: MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: VD->getType()),
2813 /*capturedByInit=*/false);
2814 EmitAutoVarCleanups(emission: Emission);
2815 }
2816 } else {
2817 EmitVarDecl(D: *VD);
2818 }
2819 }
2820 // Emit the linear steps for the linear clauses.
2821 // If a step is not constant, it is pre-calculated before the loop.
2822 if (const auto *CS = cast_or_null<BinaryOperator>(Val: C->getCalcStep()))
2823 if (const auto *SaveRef = cast<DeclRefExpr>(Val: CS->getLHS())) {
2824 EmitVarDecl(D: *cast<VarDecl>(Val: SaveRef->getDecl()));
2825 // Emit calculation of the linear step.
2826 EmitIgnoredExpr(E: CS);
2827 }
2828 }
2829 return HasLinears;
2830}
2831
2832void CodeGenFunction::EmitOMPLinearClauseFinal(
2833 const OMPLoopDirective &D,
2834 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2835 if (!HaveInsertPoint())
2836 return;
2837 llvm::BasicBlock *DoneBB = nullptr;
2838 // Emit the final values of the linear variables.
2839 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2840 auto IC = C->varlist_begin();
2841 for (const Expr *F : C->finals()) {
2842 if (!DoneBB) {
2843 if (llvm::Value *Cond = CondGen(*this)) {
2844 // If the first post-update expression is found, emit conditional
2845 // block if it was requested.
2846 llvm::BasicBlock *ThenBB = createBasicBlock(name: ".omp.linear.pu");
2847 DoneBB = createBasicBlock(name: ".omp.linear.pu.done");
2848 Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
2849 EmitBlock(BB: ThenBB);
2850 }
2851 }
2852 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IC)->getDecl();
2853 Address OrigAddr = Address::invalid();
2854 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
2855 OrigAddr = EmitOMPBindingOriginalAddr(BD, Loc: (*IC)->getExprLoc());
2856 } else {
2857 const auto *OrigVD = cast<VarDecl>(Val: OrigDecl);
2858 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2859 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
2860 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2861 OrigAddr = EmitLValue(E: &DRE).getAddress();
2862 }
2863 CodeGenFunction::OMPPrivateScope VarScope(*this);
2864 VarScope.addPrivate(LocalVD: OrigDecl, Addr: OrigAddr);
2865 (void)VarScope.Privatize();
2866 EmitIgnoredExpr(E: F);
2867 ++IC;
2868 }
2869 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2870 EmitIgnoredExpr(E: PostUpdate);
2871 }
2872 if (DoneBB)
2873 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
2874}
2875
2876static void emitAlignedClause(CodeGenFunction &CGF,
2877 const OMPExecutableDirective &D) {
2878 if (!CGF.HaveInsertPoint())
2879 return;
2880 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2881 llvm::APInt ClauseAlignment(64, 0);
2882 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2883 auto *AlignmentCI =
2884 cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AlignmentExpr));
2885 ClauseAlignment = AlignmentCI->getValue();
2886 }
2887 for (const Expr *E : Clause->varlist()) {
2888 llvm::APInt Alignment(ClauseAlignment);
2889 if (Alignment == 0) {
2890 // OpenMP [2.8.1, Description]
2891 // If no optional parameter is specified, implementation-defined default
2892 // alignments for SIMD instructions on the target platforms are assumed.
2893 Alignment =
2894 CGF.getContext()
2895 .toCharUnitsFromBits(BitSize: CGF.getContext().getOpenMPDefaultSimdAlign(
2896 T: E->getType()->getPointeeType()))
2897 .getQuantity();
2898 }
2899 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2900 "alignment is not power of 2");
2901 if (Alignment != 0) {
2902 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2903 CGF.emitAlignmentAssumption(
2904 PtrValue, E, /*No second loc needed*/ AssumptionLoc: SourceLocation(),
2905 Alignment: llvm::ConstantInt::get(Context&: CGF.getLLVMContext(), V: Alignment));
2906 }
2907 }
2908 }
2909}
2910
2911void CodeGenFunction::EmitOMPPrivateLoopCounters(
2912 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
2913 if (!HaveInsertPoint())
2914 return;
2915 auto I = S.private_counters().begin();
2916 for (const Expr *E : S.counters()) {
2917 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
2918 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
2919 // Emit var without initialization.
2920 AutoVarEmission VarEmission = EmitAutoVarAlloca(var: *PrivateVD);
2921 EmitAutoVarCleanups(emission: VarEmission);
2922 LocalDeclMap.erase(Val: PrivateVD);
2923 (void)LoopScope.addPrivate(LocalVD: VD, Addr: VarEmission.getAllocatedAddress());
2924 if (LocalDeclMap.count(Val: VD) || CapturedStmtInfo->lookup(VD) ||
2925 VD->hasGlobalStorage()) {
2926 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2927 LocalDeclMap.count(Val: VD) || CapturedStmtInfo->lookup(VD),
2928 E->getType(), VK_LValue, E->getExprLoc());
2929 (void)LoopScope.addPrivate(LocalVD: PrivateVD, Addr: EmitLValue(E: &DRE).getAddress());
2930 } else {
2931 (void)LoopScope.addPrivate(LocalVD: PrivateVD, Addr: VarEmission.getAllocatedAddress());
2932 }
2933 ++I;
2934 }
2935 // Privatize extra loop counters used in loops for ordered(n) clauses.
2936 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2937 if (!C->getNumForLoops())
2938 continue;
2939 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2940 I < E; ++I) {
2941 const auto *DRE = cast<DeclRefExpr>(Val: C->getLoopCounter(NumLoop: I));
2942 const auto *VD = cast<VarDecl>(Val: DRE->getDecl());
2943 // Override only those variables that can be captured to avoid re-emission
2944 // of the variables declared within the loops.
2945 if (DRE->refersToEnclosingVariableOrCapture()) {
2946 (void)LoopScope.addPrivate(
2947 LocalVD: VD, Addr: CreateMemTemp(T: DRE->getType(), Name: VD->getName()));
2948 }
2949 }
2950 }
2951}
2952
2953static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
2954 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2955 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2956 if (!CGF.HaveInsertPoint())
2957 return;
2958 {
2959 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2960 CGF.EmitOMPPrivateLoopCounters(S, LoopScope&: PreCondScope);
2961 (void)PreCondScope.Privatize();
2962 // Get initial values of real counters.
2963 for (const Expr *I : S.inits()) {
2964 CGF.EmitIgnoredExpr(E: I);
2965 }
2966 }
2967 // Create temp loop control variables with their init values to support
2968 // non-rectangular loops.
2969 CodeGenFunction::OMPMapVars PreCondVars;
2970 for (const Expr *E : S.dependent_counters()) {
2971 if (!E)
2972 continue;
2973 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2974 "dependent counter must not be an iterator.");
2975 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
2976 Address CounterAddr =
2977 CGF.CreateMemTemp(T: VD->getType().getNonReferenceType());
2978 (void)PreCondVars.setVarAddr(CGF, LocalVD: VD, TempAddr: CounterAddr);
2979 }
2980 (void)PreCondVars.apply(CGF);
2981 for (const Expr *E : S.dependent_inits()) {
2982 if (!E)
2983 continue;
2984 CGF.EmitIgnoredExpr(E);
2985 }
2986 // Check that loop is executed at least one time.
2987 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2988 PreCondVars.restore(CGF);
2989}
2990
2991void CodeGenFunction::EmitOMPLinearClause(
2992 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2993 if (!HaveInsertPoint())
2994 return;
2995 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2996 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2997 if (isOpenMPSimdDirective(DKind: EKind)) {
2998 const auto *LoopDirective = cast<OMPLoopDirective>(Val: &D);
2999 for (const Expr *C : LoopDirective->counters()) {
3000 SIMDLCVs.insert(
3001 V: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: C)->getDecl())->getCanonicalDecl());
3002 }
3003 }
3004 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
3005 auto CurPrivate = C->privates().begin();
3006 for (const Expr *E : C->varlist()) {
3007 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
3008 const auto *PrivateVD =
3009 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *CurPrivate)->getDecl());
3010 bool IsSIMDLCV = false;
3011 if (const auto *VarD = dyn_cast<VarDecl>(Val: VD))
3012 IsSIMDLCV = SIMDLCVs.count(V: VarD->getCanonicalDecl());
3013 if (!IsSIMDLCV) {
3014 // Emit private VarDecl with copy init.
3015 EmitVarDecl(D: *PrivateVD);
3016 Address PrivateAddr = GetAddrOfLocalVar(VD: PrivateVD);
3017 bool IsRegistered = PrivateScope.addPrivate(LocalVD: VD, Addr: PrivateAddr);
3018 assert(IsRegistered && "linear var already registered as private");
3019 // Silence the warning about unused variable.
3020 (void)IsRegistered;
3021 } else {
3022 EmitVarDecl(D: *PrivateVD);
3023 }
3024 ++CurPrivate;
3025 }
3026 }
3027}
3028
3029static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
3030 const OMPExecutableDirective &D) {
3031 if (!CGF.HaveInsertPoint())
3032 return;
3033 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
3034 RValue Len = CGF.EmitAnyExpr(E: C->getSimdlen(), aggSlot: AggValueSlot::ignored(),
3035 /*ignoreResult=*/true);
3036 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3037 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
3038 // In presence of finite 'safelen', it may be unsafe to mark all
3039 // the memory instructions parallel, because loop-carried
3040 // dependences of 'safelen' iterations are possible.
3041 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
3042 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
3043 RValue Len = CGF.EmitAnyExpr(E: C->getSafelen(), aggSlot: AggValueSlot::ignored(),
3044 /*ignoreResult=*/true);
3045 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3046 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
3047 // In presence of finite 'safelen', it may be unsafe to mark all
3048 // the memory instructions parallel, because loop-carried
3049 // dependences of 'safelen' iterations are possible.
3050 CGF.LoopStack.setParallel(/*Enable=*/false);
3051 }
3052}
3053
3054// Check for the presence of an `OMPOrderedBlockAssocDirective`,
3055// i.e., `ordered` in `#pragma omp ordered simd`.
3056//
3057// Consider the following source code:
3058// ```
3059// __attribute__((noinline)) void omp_simd_loop(float X[ARRAY_SIZE][ARRAY_SIZE])
3060// {
3061// for (int r = 1; r < ARRAY_SIZE; ++r) {
3062// for (int c = 1; c < ARRAY_SIZE; ++c) {
3063// #pragma omp simd
3064// for (int k = 2; k < ARRAY_SIZE; ++k) {
3065// #pragma omp ordered simd
3066// X[r][k] = X[r][k - 2] + sinf((float)(r / c));
3067// }
3068// }
3069// }
3070// }
3071// ```
3072//
3073// Suppose we are in `CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective
3074// &D)`. By examining `D.dump()` we have the following AST containing
3075// `OMPOrderedBlockAssocDirective`:
3076//
3077// ```
3078// OMPSimdDirective 0x1c32950
3079// `-CapturedStmt 0x1c32028
3080// |-CapturedDecl 0x1c310e8
3081// | |-ForStmt 0x1c31e30
3082// | | |-DeclStmt 0x1c31298
3083// | | | `-VarDecl 0x1c31208 used k 'int' cinit
3084// | | | `-IntegerLiteral 0x1c31278 'int' 2
3085// | | |-<<<NULL>>>
3086// | | |-BinaryOperator 0x1c31308 'int' '<'
3087// | | | |-ImplicitCastExpr 0x1c312f0 'int' <LValueToRValue>
3088// | | | | `-DeclRefExpr 0x1c312b0 'int' lvalue Var 0x1c31208 'k' 'int'
3089// | | | `-IntegerLiteral 0x1c312d0 'int' 256
3090// | | |-UnaryOperator 0x1c31348 'int' prefix '++'
3091// | | | `-DeclRefExpr 0x1c31328 'int' lvalue Var 0x1c31208 'k' 'int'
3092// | | `-CompoundStmt 0x1c31e18
3093// | | `-OMPOrderedBlockAssocDirective 0x1c31dd8
3094// | | |-OMPSimdClause 0x1c31380
3095// | | `-CapturedStmt 0x1c31cd0
3096// ```
3097//
3098// Note the presence of `OMPOrderedBlockAssocDirective` above:
3099// It's (transitively) nested in a `CapturedStmt` representing the pragma
3100// annotated compound statement. Thus, we need to consider this nesting and
3101// include checking the `getCapturedStmt` in this case.
3102static bool hasOrderedBlockAssocDirective(const Stmt *S) {
3103 if (isa<OMPOrderedBlockAssocDirective>(Val: S))
3104 return true;
3105
3106 if (const auto *CS = dyn_cast<CapturedStmt>(Val: S))
3107 return hasOrderedBlockAssocDirective(S: CS->getCapturedStmt());
3108
3109 for (const Stmt *Child : S->children()) {
3110 if (Child && hasOrderedBlockAssocDirective(S: Child))
3111 return true;
3112 }
3113
3114 return false;
3115}
3116
3117static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt,
3118 LoopInfoStack &LoopStack) {
3119 // Check for the presence of an `OMPOrderedBlockAssocDirective`
3120 // i.e., `ordered` in `#pragma omp ordered simd`
3121 bool HasOrderedDirective = hasOrderedBlockAssocDirective(S: &AssociatedStmt);
3122 // If present then conservatively disable loop vectorization
3123 // analogously to how `emitSimdlenSafelenClause` does.
3124 if (HasOrderedDirective)
3125 LoopStack.setParallel(/*Enable=*/false);
3126}
3127
3128void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) {
3129 // Walk clauses and process safelen/lastprivate.
3130 LoopStack.setParallel(/*Enable=*/true);
3131 LoopStack.setVectorizeEnable();
3132 const Stmt *AssociatedStmt = D.getAssociatedStmt();
3133 applyConservativeSimdOrderedDirective(AssociatedStmt: *AssociatedStmt, LoopStack);
3134 emitSimdlenSafelenClause(CGF&: *this, D);
3135 if (const auto *C = D.getSingleClause<OMPOrderClause>())
3136 if (C->getKind() == OMPC_ORDER_concurrent)
3137 LoopStack.setParallel(/*Enable=*/true);
3138 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
3139 if ((EKind == OMPD_simd ||
3140 (getLangOpts().OpenMPSimd && isOpenMPSimdDirective(DKind: EKind))) &&
3141 llvm::any_of(Range: D.getClausesOfKind<OMPReductionClause>(),
3142 P: [](const OMPReductionClause *C) {
3143 return C->getModifier() == OMPC_REDUCTION_inscan;
3144 }))
3145 // Disable parallel access in case of prefix sum.
3146 LoopStack.setParallel(/*Enable=*/false);
3147}
3148
3149void CodeGenFunction::EmitOMPSimdFinal(
3150 const OMPLoopDirective &D,
3151 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
3152 if (!HaveInsertPoint())
3153 return;
3154 llvm::BasicBlock *DoneBB = nullptr;
3155 auto IC = D.counters().begin();
3156 auto IPC = D.private_counters().begin();
3157 for (const Expr *F : D.finals()) {
3158 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: (*IC))->getDecl());
3159 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: (*IPC))->getDecl());
3160 const auto *CED = dyn_cast<OMPCapturedExprDecl>(Val: OrigVD);
3161 if (LocalDeclMap.count(Val: OrigVD) || CapturedStmtInfo->lookup(VD: OrigVD) ||
3162 OrigVD->hasGlobalStorage() || CED) {
3163 if (!DoneBB) {
3164 if (llvm::Value *Cond = CondGen(*this)) {
3165 // If the first post-update expression is found, emit conditional
3166 // block if it was requested.
3167 llvm::BasicBlock *ThenBB = createBasicBlock(name: ".omp.final.then");
3168 DoneBB = createBasicBlock(name: ".omp.final.done");
3169 Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
3170 EmitBlock(BB: ThenBB);
3171 }
3172 }
3173 Address OrigAddr = Address::invalid();
3174 if (CED) {
3175 OrigAddr = EmitLValue(E: CED->getInit()->IgnoreImpCasts()).getAddress();
3176 } else {
3177 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
3178 /*RefersToEnclosingVariableOrCapture=*/false,
3179 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
3180 OrigAddr = EmitLValue(E: &DRE).getAddress();
3181 }
3182 OMPPrivateScope VarScope(*this);
3183 VarScope.addPrivate(LocalVD: OrigVD, Addr: OrigAddr);
3184 (void)VarScope.Privatize();
3185 EmitIgnoredExpr(E: F);
3186 }
3187 ++IC;
3188 ++IPC;
3189 }
3190 if (DoneBB)
3191 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
3192}
3193
3194static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
3195 const OMPLoopDirective &S,
3196 CodeGenFunction::JumpDest LoopExit) {
3197 CGF.EmitOMPLoopBody(D: S, LoopExit);
3198 CGF.EmitStopPoint(S: &S);
3199}
3200
3201/// Emit a helper variable and return corresponding lvalue.
3202static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
3203 const DeclRefExpr *Helper) {
3204 auto VDecl = cast<VarDecl>(Val: Helper->getDecl());
3205 CGF.EmitVarDecl(D: *VDecl);
3206 return CGF.EmitLValue(E: Helper);
3207}
3208
3209static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
3210 const RegionCodeGenTy &SimdInitGen,
3211 const RegionCodeGenTy &BodyCodeGen) {
3212 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
3213 PrePostActionTy &) {
3214 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
3215 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3216 SimdInitGen(CGF);
3217
3218 BodyCodeGen(CGF);
3219 };
3220 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
3221 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3222 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
3223
3224 BodyCodeGen(CGF);
3225 };
3226 const Expr *IfCond = nullptr;
3227 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind())) {
3228 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3229 if (CGF.getLangOpts().OpenMP >= 50 &&
3230 (C->getNameModifier() == OMPD_unknown ||
3231 C->getNameModifier() == OMPD_simd)) {
3232 IfCond = C->getCondition();
3233 break;
3234 }
3235 }
3236 }
3237 if (IfCond) {
3238 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
3239 } else {
3240 RegionCodeGenTy ThenRCG(ThenGen);
3241 ThenRCG(CGF);
3242 }
3243}
3244
3245static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
3246 PrePostActionTy &Action) {
3247 Action.Enter(CGF);
3248 OMPLoopScope PreInitScope(CGF, S);
3249 // if (PreCond) {
3250 // for (IV in 0..LastIteration) BODY;
3251 // <Final counter/linear vars updates>;
3252 // }
3253
3254 // The presence of lower/upper bound variable depends on the actual directive
3255 // kind in the AST node. The variables must be emitted because some of the
3256 // expressions associated with the loop will use them.
3257 OpenMPDirectiveKind DKind = S.getDirectiveKind();
3258 if (isOpenMPDistributeDirective(DKind) ||
3259 isOpenMPWorksharingDirective(DKind) || isOpenMPTaskLoopDirective(DKind) ||
3260 isOpenMPGenericLoopDirective(DKind)) {
3261 (void)EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: S.getLowerBoundVariable()));
3262 (void)EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: S.getUpperBoundVariable()));
3263 }
3264
3265 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3266 // Emit: if (PreCond) - begin.
3267 // If the condition constant folds and can be elided, avoid emitting the
3268 // whole loop.
3269 bool CondConstant;
3270 llvm::BasicBlock *ContBlock = nullptr;
3271 if (CGF.ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
3272 if (!CondConstant)
3273 return;
3274 } else {
3275 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "simd.if.then");
3276 ContBlock = CGF.createBasicBlock(name: "simd.if.end");
3277 emitPreCond(CGF, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
3278 TrueCount: CGF.getProfileCount(S: &S));
3279 CGF.EmitBlock(BB: ThenBlock);
3280 CGF.incrementProfileCounter(S: &S);
3281 }
3282
3283 // Emit the loop iteration variable.
3284 const Expr *IVExpr = S.getIterationVariable();
3285 const auto *IVDecl = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IVExpr)->getDecl());
3286 CGF.EmitVarDecl(D: *IVDecl);
3287 CGF.EmitIgnoredExpr(E: S.getInit());
3288
3289 // Emit the iterations count variable.
3290 // If it is not a variable, Sema decided to calculate iterations count on
3291 // each iteration (e.g., it is foldable into a constant).
3292 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
3293 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
3294 // Emit calculation of the iterations count.
3295 CGF.EmitIgnoredExpr(E: S.getCalcLastIteration());
3296 }
3297
3298 emitAlignedClause(CGF, D: S);
3299 (void)CGF.EmitOMPLinearClauseInit(D: S);
3300 {
3301 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
3302 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
3303 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
3304 CGF.EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
3305 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
3306 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
3307 CGF, S, CGF.EmitLValue(E: S.getIterationVariable()));
3308 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
3309 (void)LoopScope.Privatize();
3310 if (isOpenMPTargetExecutionDirective(DKind: EKind))
3311 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
3312
3313 emitCommonSimdLoop(
3314 CGF, S,
3315 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3316 CGF.EmitOMPSimdInit(D: S);
3317 },
3318 BodyCodeGen: [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3319 CGF.EmitOMPInnerLoop(
3320 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: S.getCond(), IncExpr: S.getInc(),
3321 BodyGen: [&S](CodeGenFunction &CGF) {
3322 emitOMPLoopBodyWithStopPoint(CGF, S,
3323 LoopExit: CodeGenFunction::JumpDest());
3324 },
3325 PostIncGen: [](CodeGenFunction &) {});
3326 });
3327 CGF.EmitOMPSimdFinal(D: S, CondGen: [](CodeGenFunction &) { return nullptr; });
3328 // Emit final copy of the lastprivate variables at the end of loops.
3329 if (HasLastprivateClause)
3330 CGF.EmitOMPLastprivateClauseFinal(D: S, /*NoFinals=*/true);
3331 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_simd);
3332 emitPostUpdateForReductionClause(CGF, D: S,
3333 CondGen: [](CodeGenFunction &) { return nullptr; });
3334 LoopScope.restoreMap();
3335 CGF.EmitOMPLinearClauseFinal(D: S, CondGen: [](CodeGenFunction &) { return nullptr; });
3336 }
3337 // Emit: if (PreCond) - end.
3338 if (ContBlock) {
3339 CGF.EmitBranch(Block: ContBlock);
3340 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
3341 }
3342}
3343
3344// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3345// available for "loop bind(thread)", which maps to "simd".
3346static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S) {
3347 // Check for unsupported clauses
3348 for (OMPClause *C : S.clauses()) {
3349 // Currently only order, simdlen and safelen clauses are supported
3350 if (!(isa<OMPSimdlenClause>(Val: C) || isa<OMPSafelenClause>(Val: C) ||
3351 isa<OMPOrderClause>(Val: C) || isa<OMPAlignedClause>(Val: C)))
3352 return false;
3353 }
3354
3355 // Check if we have a statement with the ordered-blockassoc directive.
3356 // Visit the statement hierarchy to find a compound statement
3357 // with a ordered-blockassoc directive in it.
3358 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Val: S.getRawStmt())) {
3359 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
3360 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
3361 if (!SubStmt)
3362 continue;
3363 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: SubStmt)) {
3364 for (const Stmt *CSSubStmt : CS->children()) {
3365 if (!CSSubStmt)
3366 continue;
3367 if (isa<OMPOrderedBlockAssocDirective>(Val: CSSubStmt)) {
3368 return false;
3369 }
3370 }
3371 }
3372 }
3373 }
3374 }
3375 return true;
3376}
3377
3378static llvm::MapVector<llvm::Value *, llvm::Value *>
3379GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF) {
3380 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
3381 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
3382 llvm::APInt ClauseAlignment(64, 0);
3383 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
3384 auto *AlignmentCI =
3385 cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AlignmentExpr));
3386 ClauseAlignment = AlignmentCI->getValue();
3387 }
3388 for (const Expr *E : Clause->varlist()) {
3389 llvm::APInt Alignment(ClauseAlignment);
3390 if (Alignment == 0) {
3391 // OpenMP [2.8.1, Description]
3392 // If no optional parameter is specified, implementation-defined default
3393 // alignments for SIMD instructions on the target platforms are assumed.
3394 Alignment =
3395 CGF.getContext()
3396 .toCharUnitsFromBits(BitSize: CGF.getContext().getOpenMPDefaultSimdAlign(
3397 T: E->getType()->getPointeeType()))
3398 .getQuantity();
3399 }
3400 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
3401 "alignment is not power of 2");
3402 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
3403 AlignedVars[PtrValue] = CGF.Builder.getInt64(C: Alignment.getSExtValue());
3404 }
3405 }
3406 return AlignedVars;
3407}
3408
3409// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3410// available for "loop bind(thread)", which maps to "simd".
3411static void emitOMPSimdDirective(const OMPLoopDirective &S,
3412 CodeGenFunction &CGF, CodeGenModule &CGM) {
3413 bool UseOMPIRBuilder =
3414 CGM.getLangOpts().OpenMPIRBuilder && isSimdSupportedByOpenMPIRBuilder(S);
3415 if (UseOMPIRBuilder) {
3416 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,
3417 PrePostActionTy &) {
3418 // Use the OpenMPIRBuilder if enabled.
3419 if (UseOMPIRBuilder) {
3420 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
3421 GetAlignedMapping(S, CGF);
3422 // Emit the associated statement and get its loop representation.
3423 const Stmt *Inner = S.getRawStmt();
3424 llvm::CanonicalLoopInfo *CLI =
3425 CGF.EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
3426
3427 llvm::OpenMPIRBuilder &OMPBuilder =
3428 CGM.getOpenMPRuntime().getOMPBuilder();
3429 // Add SIMD specific metadata
3430 llvm::ConstantInt *Simdlen = nullptr;
3431 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
3432 RValue Len = CGF.EmitAnyExpr(E: C->getSimdlen(), aggSlot: AggValueSlot::ignored(),
3433 /*ignoreResult=*/true);
3434 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3435 Simdlen = Val;
3436 }
3437 llvm::ConstantInt *Safelen = nullptr;
3438 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
3439 RValue Len = CGF.EmitAnyExpr(E: C->getSafelen(), aggSlot: AggValueSlot::ignored(),
3440 /*ignoreResult=*/true);
3441 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3442 Safelen = Val;
3443 }
3444 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
3445 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3446 if (C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
3447 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
3448 }
3449 }
3450 // Add simd metadata to the collapsed loop. Do not generate
3451 // another loop for if clause. Support for if clause is done earlier.
3452 OMPBuilder.applySimd(Loop: CLI, AlignedVars,
3453 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
3454 return;
3455 }
3456 };
3457 {
3458 auto LPCRegion =
3459 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
3460 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3461 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd,
3462 CodeGen: CodeGenIRBuilder);
3463 }
3464 return;
3465 }
3466
3467 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3468 CGF.OMPFirstScanLoop = true;
3469 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3470 emitOMPSimdRegion(CGF, S, Action);
3471 };
3472 {
3473 auto LPCRegion =
3474 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
3475 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3476 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd, CodeGen);
3477 }
3478 // Check for outer lastprivate conditional update.
3479 checkForLastprivateConditionalUpdate(CGF, S);
3480}
3481
3482void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
3483 emitOMPSimdDirective(S, CGF&: *this, CGM);
3484}
3485
3486/// Emit a loop-nest transformation followed by its loop variable finalization,
3487/// which restores the loop-exit values as required by OpenMP 6.0.
3488static void emitLoopNestTransformation(
3489 CodeGenFunction &CGF,
3490 const OMPCanonicalLoopNestTransformationDirective &S) {
3491 OMPTransformDirectiveScopeRAII Scope(CGF, &S);
3492 CGF.EmitStmt(S: S.getTransformedStmt());
3493 if (Stmt *Finals = S.getFinals())
3494 CGF.EmitStmt(S: Finals);
3495}
3496
3497void CodeGenFunction::EmitOMPTileDirective(const OMPTileDirective &S) {
3498 emitLoopNestTransformation(CGF&: *this, S);
3499}
3500
3501void CodeGenFunction::EmitOMPStripeDirective(const OMPStripeDirective &S) {
3502 emitLoopNestTransformation(CGF&: *this, S);
3503}
3504
3505void CodeGenFunction::EmitOMPReverseDirective(const OMPReverseDirective &S) {
3506 emitLoopNestTransformation(CGF&: *this, S);
3507}
3508
3509void CodeGenFunction::EmitOMPSplitDirective(const OMPSplitDirective &S) {
3510 emitLoopNestTransformation(CGF&: *this, S);
3511}
3512
3513void CodeGenFunction::EmitOMPInterchangeDirective(
3514 const OMPInterchangeDirective &S) {
3515 emitLoopNestTransformation(CGF&: *this, S);
3516}
3517
3518void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
3519 // Emit the de-sugared statement.
3520 OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
3521 EmitStmt(S: S.getTransformedStmt());
3522 EmitStmt(S: S.getFinals());
3523}
3524
3525void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
3526 // Emit the de-sugared statement
3527 OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
3528
3529 EmitStmt(S: S.getTransformedStmt());
3530 if (Stmt *Finals = S.getFinals())
3531 EmitStmt(S: Finals);
3532}
3533
3534void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
3535 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
3536
3537 if (UseOMPIRBuilder) {
3538 auto DL = SourceLocToDebugLoc(Location: S.getBeginLoc());
3539 const Stmt *Inner = S.getRawStmt();
3540
3541 // Consume nested loop. Clear the entire remaining loop stack because a
3542 // fully unrolled loop is non-transformable. For partial unrolling the
3543 // generated outer loop is pushed back to the stack.
3544 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
3545 OMPLoopNestStack.clear();
3546
3547 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
3548
3549 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
3550 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
3551
3552 if (S.hasClausesOfKind<OMPFullClause>()) {
3553 assert(ExpectedOMPLoopDepth == 0);
3554 OMPBuilder.unrollLoopFull(DL, Loop: CLI);
3555 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3556 uint64_t Factor = 0;
3557 if (Expr *FactorExpr = PartialClause->getFactor()) {
3558 Factor =
3559 FactorExpr->EvaluateKnownConstInt(Ctx: getContext()).getLimitedValue();
3560 assert(Factor >= 1 && "Only positive factors are valid");
3561 }
3562 OMPBuilder.unrollLoopPartial(DL, Loop: CLI, Factor,
3563 UnrolledCLI: NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
3564 } else {
3565 OMPBuilder.unrollLoopHeuristic(DL, Loop: CLI);
3566 }
3567
3568 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3569 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3570 if (UnrolledCLI)
3571 OMPLoopNestStack.push_back(Elt: UnrolledCLI);
3572
3573 return;
3574 }
3575
3576 // This function is only called if the unrolled loop is not consumed by any
3577 // other loop-associated construct. Such a loop-associated construct will have
3578 // used the transformed AST.
3579
3580 // Set the unroll metadata for the next emitted loop.
3581 LoopStack.setUnrollState(LoopAttributes::Enable);
3582
3583 if (S.hasClausesOfKind<OMPFullClause>()) {
3584 LoopStack.setUnrollState(LoopAttributes::Full);
3585 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3586 if (Expr *FactorExpr = PartialClause->getFactor()) {
3587 uint64_t Factor =
3588 FactorExpr->EvaluateKnownConstInt(Ctx: getContext()).getLimitedValue();
3589 assert(Factor >= 1 && "Only positive factors are valid");
3590 LoopStack.setUnrollCount(Factor);
3591 }
3592 }
3593
3594 EmitStmt(S: S.getAssociatedStmt());
3595}
3596
3597void CodeGenFunction::EmitOMPOuterLoop(
3598 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
3599 CodeGenFunction::OMPPrivateScope &LoopScope,
3600 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3601 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
3602 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
3603 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3604
3605 const Expr *IVExpr = S.getIterationVariable();
3606 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3607 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3608
3609 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "omp.dispatch.end");
3610
3611 // Start the loop with a block that tests the condition.
3612 llvm::BasicBlock *CondBlock = createBasicBlock(name: "omp.dispatch.cond");
3613 EmitBlock(BB: CondBlock);
3614 const SourceRange R = getAssociatedLoopRange(S);
3615 OMPLoopNestStack.clear();
3616 LoopStack.push(Header: CondBlock, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
3617 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
3618
3619 llvm::Value *BoolCondVal = nullptr;
3620 if (!DynamicOrOrdered) {
3621 // UB = min(UB, GlobalUB) or
3622 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
3623 // 'distribute parallel for')
3624 EmitIgnoredExpr(E: LoopArgs.EUB);
3625 // IV = LB
3626 EmitIgnoredExpr(E: LoopArgs.Init);
3627 // IV < UB
3628 BoolCondVal = EvaluateExprAsBool(E: LoopArgs.Cond);
3629 } else {
3630 BoolCondVal =
3631 RT.emitForNext(CGF&: *this, Loc: S.getBeginLoc(), IVSize, IVSigned, IL: LoopArgs.IL,
3632 LB: LoopArgs.LB, UB: LoopArgs.UB, ST: LoopArgs.ST);
3633 }
3634
3635 // If there are any cleanups between here and the loop-exit scope,
3636 // create a block to stage a loop exit along.
3637 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
3638 if (LoopScope.requiresCleanups())
3639 ExitBlock = createBasicBlock(name: "omp.dispatch.cleanup");
3640
3641 llvm::BasicBlock *LoopBody = createBasicBlock(name: "omp.dispatch.body");
3642 Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock);
3643 if (ExitBlock != LoopExit.getBlock()) {
3644 EmitBlock(BB: ExitBlock);
3645 EmitBranchThroughCleanup(Dest: LoopExit);
3646 }
3647 EmitBlock(BB: LoopBody);
3648
3649 // Emit "IV = LB" (in case of static schedule, we have already calculated new
3650 // LB for loop condition and emitted it above).
3651 if (DynamicOrOrdered)
3652 EmitIgnoredExpr(E: LoopArgs.Init);
3653
3654 // Create a block for the increment.
3655 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.dispatch.inc");
3656 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
3657
3658 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3659 emitCommonSimdLoop(
3660 CGF&: *this, S,
3661 SimdInitGen: [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3662 // Generate !llvm.loop.parallel metadata for loads and stores for loops
3663 // with dynamic/guided scheduling and without ordered clause.
3664 if (!isOpenMPSimdDirective(DKind: EKind)) {
3665 CGF.LoopStack.setParallel(!IsMonotonic);
3666 if (const auto *C = S.getSingleClause<OMPOrderClause>())
3667 if (C->getKind() == OMPC_ORDER_concurrent)
3668 CGF.LoopStack.setParallel(/*Enable=*/true);
3669 } else {
3670 CGF.EmitOMPSimdInit(D: S);
3671 }
3672 },
3673 BodyCodeGen: [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3674 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3675 SourceLocation Loc = S.getBeginLoc();
3676 // when 'distribute' is not combined with a 'for':
3677 // while (idx <= UB) { BODY; ++idx; }
3678 // when 'distribute' is combined with a 'for'
3679 // (e.g. 'distribute parallel for')
3680 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3681 CGF.EmitOMPInnerLoop(
3682 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: LoopArgs.Cond, IncExpr: LoopArgs.IncExpr,
3683 BodyGen: [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3684 CodeGenLoop(CGF, S, LoopExit);
3685 },
3686 PostIncGen: [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3687 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3688 });
3689 });
3690
3691 EmitBlock(BB: Continue.getBlock());
3692 BreakContinueStack.pop_back();
3693 if (!DynamicOrOrdered) {
3694 // Emit "LB = LB + Stride", "UB = UB + Stride".
3695 EmitIgnoredExpr(E: LoopArgs.NextLB);
3696 EmitIgnoredExpr(E: LoopArgs.NextUB);
3697 }
3698
3699 EmitBranch(Block: CondBlock);
3700 OMPLoopNestStack.clear();
3701 LoopStack.pop();
3702 // Emit the fall-through block.
3703 EmitBlock(BB: LoopExit.getBlock());
3704
3705 // Tell the runtime we are done.
3706 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3707 if (!DynamicOrOrdered)
3708 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
3709 DKind: LoopArgs.DKind);
3710 };
3711 OMPCancelStack.emitExit(CGF&: *this, Kind: EKind, CodeGen);
3712}
3713
3714void CodeGenFunction::EmitOMPForOuterLoop(
3715 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
3716 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
3717 const OMPLoopArguments &LoopArgs,
3718 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3719 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3720
3721 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
3722 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind: ScheduleKind.Schedule);
3723
3724 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
3725 LoopArgs.Chunk != nullptr)) &&
3726 "static non-chunked schedule does not need outer loop");
3727
3728 // Emit outer loop.
3729 //
3730 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3731 // When schedule(dynamic,chunk_size) is specified, the iterations are
3732 // distributed to threads in the team in chunks as the threads request them.
3733 // Each thread executes a chunk of iterations, then requests another chunk,
3734 // until no chunks remain to be distributed. Each chunk contains chunk_size
3735 // iterations, except for the last chunk to be distributed, which may have
3736 // fewer iterations. When no chunk_size is specified, it defaults to 1.
3737 //
3738 // When schedule(guided,chunk_size) is specified, the iterations are assigned
3739 // to threads in the team in chunks as the executing threads request them.
3740 // Each thread executes a chunk of iterations, then requests another chunk,
3741 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
3742 // each chunk is proportional to the number of unassigned iterations divided
3743 // by the number of threads in the team, decreasing to 1. For a chunk_size
3744 // with value k (greater than 1), the size of each chunk is determined in the
3745 // same way, with the restriction that the chunks do not contain fewer than k
3746 // iterations (except for the last chunk to be assigned, which may have fewer
3747 // than k iterations).
3748 //
3749 // When schedule(auto) is specified, the decision regarding scheduling is
3750 // delegated to the compiler and/or runtime system. The programmer gives the
3751 // implementation the freedom to choose any possible mapping of iterations to
3752 // threads in the team.
3753 //
3754 // When schedule(runtime) is specified, the decision regarding scheduling is
3755 // deferred until run time, and the schedule and chunk size are taken from the
3756 // run-sched-var ICV. If the ICV is set to auto, the schedule is
3757 // implementation defined
3758 //
3759 // __kmpc_dispatch_init();
3760 // while(__kmpc_dispatch_next(&LB, &UB)) {
3761 // idx = LB;
3762 // while (idx <= UB) { BODY; ++idx;
3763 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
3764 // } // inner loop
3765 // }
3766 // __kmpc_dispatch_deinit();
3767 //
3768 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3769 // When schedule(static, chunk_size) is specified, iterations are divided into
3770 // chunks of size chunk_size, and the chunks are assigned to the threads in
3771 // the team in a round-robin fashion in the order of the thread number.
3772 //
3773 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
3774 // while (idx <= UB) { BODY; ++idx; } // inner loop
3775 // LB = LB + ST;
3776 // UB = UB + ST;
3777 // }
3778 //
3779
3780 const Expr *IVExpr = S.getIterationVariable();
3781 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3782 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3783
3784 if (DynamicOrOrdered) {
3785 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3786 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
3787 llvm::Value *LBVal = DispatchBounds.first;
3788 llvm::Value *UBVal = DispatchBounds.second;
3789 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3790 LoopArgs.Chunk};
3791 RT.emitForDispatchInit(CGF&: *this, Loc: S.getBeginLoc(), ScheduleKind, IVSize,
3792 IVSigned, Ordered, DispatchValues: DipatchRTInputValues);
3793 } else {
3794 CGOpenMPRuntime::StaticRTInput StaticInit(
3795 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3796 LoopArgs.ST, LoopArgs.Chunk);
3797 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3798 RT.emitForStaticInit(CGF&: *this, Loc: S.getBeginLoc(), DKind: EKind, ScheduleKind,
3799 Values: StaticInit);
3800 }
3801
3802 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3803 const unsigned IVSize,
3804 const bool IVSigned) {
3805 if (Ordered) {
3806 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3807 IVSigned);
3808 }
3809 };
3810
3811 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3812 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3813 OuterLoopArgs.IncExpr = S.getInc();
3814 OuterLoopArgs.Init = S.getInit();
3815 OuterLoopArgs.Cond = S.getCond();
3816 OuterLoopArgs.NextLB = S.getNextLowerBound();
3817 OuterLoopArgs.NextUB = S.getNextUpperBound();
3818 OuterLoopArgs.DKind = LoopArgs.DKind;
3819 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, LoopArgs: OuterLoopArgs,
3820 CodeGenLoop: emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3821 if (DynamicOrOrdered) {
3822 RT.emitForDispatchDeinit(CGF&: *this, Loc: S.getBeginLoc());
3823 }
3824}
3825
3826static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
3827 const unsigned IVSize, const bool IVSigned) {}
3828
3829void CodeGenFunction::EmitOMPDistributeOuterLoop(
3830 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3831 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3832 const CodeGenLoopTy &CodeGenLoopContent) {
3833
3834 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3835
3836 // Emit outer loop.
3837 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3838 // dynamic
3839 //
3840
3841 const Expr *IVExpr = S.getIterationVariable();
3842 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3843 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3844 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3845
3846 CGOpenMPRuntime::StaticRTInput StaticInit(
3847 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3848 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3849 RT.emitDistributeStaticInit(CGF&: *this, Loc: S.getBeginLoc(), SchedKind: ScheduleKind, Values: StaticInit);
3850
3851 // for combined 'distribute' and 'for' the increment expression of distribute
3852 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3853 Expr *IncExpr;
3854 if (isOpenMPLoopBoundSharingDirective(Kind: EKind))
3855 IncExpr = S.getDistInc();
3856 else
3857 IncExpr = S.getInc();
3858
3859 // this routine is shared by 'omp distribute parallel for' and
3860 // 'omp distribute': select the right EUB expression depending on the
3861 // directive
3862 OMPLoopArguments OuterLoopArgs;
3863 OuterLoopArgs.LB = LoopArgs.LB;
3864 OuterLoopArgs.UB = LoopArgs.UB;
3865 OuterLoopArgs.ST = LoopArgs.ST;
3866 OuterLoopArgs.IL = LoopArgs.IL;
3867 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3868 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3869 ? S.getCombinedEnsureUpperBound()
3870 : S.getEnsureUpperBound();
3871 OuterLoopArgs.IncExpr = IncExpr;
3872 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3873 ? S.getCombinedInit()
3874 : S.getInit();
3875 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3876 ? S.getCombinedCond()
3877 : S.getCond();
3878 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3879 ? S.getCombinedNextLowerBound()
3880 : S.getNextLowerBound();
3881 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3882 ? S.getCombinedNextUpperBound()
3883 : S.getNextUpperBound();
3884 OuterLoopArgs.DKind = OMPD_distribute;
3885
3886 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3887 LoopScope, LoopArgs: OuterLoopArgs, CodeGenLoop: CodeGenLoopContent,
3888 CodeGenOrdered: emitEmptyOrdered);
3889}
3890
3891static std::pair<LValue, LValue>
3892emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
3893 const OMPExecutableDirective &S) {
3894 const OMPLoopDirective &LS = cast<OMPLoopDirective>(Val: S);
3895 LValue LB =
3896 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getLowerBoundVariable()));
3897 LValue UB =
3898 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getUpperBoundVariable()));
3899
3900 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3901 // parallel for') we need to use the 'distribute'
3902 // chunk lower and upper bounds rather than the whole loop iteration
3903 // space. These are parameters to the outlined function for 'parallel'
3904 // and we copy the bounds of the previous schedule into the
3905 // the current ones.
3906 LValue PrevLB = CGF.EmitLValue(E: LS.getPrevLowerBoundVariable());
3907 LValue PrevUB = CGF.EmitLValue(E: LS.getPrevUpperBoundVariable());
3908 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3909 lvalue: PrevLB, Loc: LS.getPrevLowerBoundVariable()->getExprLoc());
3910 PrevLBVal = CGF.EmitScalarConversion(
3911 Src: PrevLBVal, SrcTy: LS.getPrevLowerBoundVariable()->getType(),
3912 DstTy: LS.getIterationVariable()->getType(),
3913 Loc: LS.getPrevLowerBoundVariable()->getExprLoc());
3914 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3915 lvalue: PrevUB, Loc: LS.getPrevUpperBoundVariable()->getExprLoc());
3916 PrevUBVal = CGF.EmitScalarConversion(
3917 Src: PrevUBVal, SrcTy: LS.getPrevUpperBoundVariable()->getType(),
3918 DstTy: LS.getIterationVariable()->getType(),
3919 Loc: LS.getPrevUpperBoundVariable()->getExprLoc());
3920
3921 CGF.EmitStoreOfScalar(value: PrevLBVal, lvalue: LB);
3922 CGF.EmitStoreOfScalar(value: PrevUBVal, lvalue: UB);
3923
3924 return {LB, UB};
3925}
3926
3927/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3928/// we need to use the LB and UB expressions generated by the worksharing
3929/// code generation support, whereas in non combined situations we would
3930/// just emit 0 and the LastIteration expression
3931/// This function is necessary due to the difference of the LB and UB
3932/// types for the RT emission routines for 'for_static_init' and
3933/// 'for_dispatch_init'
3934static std::pair<llvm::Value *, llvm::Value *>
3935emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
3936 const OMPExecutableDirective &S,
3937 Address LB, Address UB) {
3938 const OMPLoopDirective &LS = cast<OMPLoopDirective>(Val: S);
3939 const Expr *IVExpr = LS.getIterationVariable();
3940 // when implementing a dynamic schedule for a 'for' combined with a
3941 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3942 // is not normalized as each team only executes its own assigned
3943 // distribute chunk
3944 QualType IteratorTy = IVExpr->getType();
3945 llvm::Value *LBVal =
3946 CGF.EmitLoadOfScalar(Addr: LB, /*Volatile=*/false, Ty: IteratorTy, Loc: S.getBeginLoc());
3947 llvm::Value *UBVal =
3948 CGF.EmitLoadOfScalar(Addr: UB, /*Volatile=*/false, Ty: IteratorTy, Loc: S.getBeginLoc());
3949 return {LBVal, UBVal};
3950}
3951
3952static void emitDistributeParallelForDistributeInnerBoundParams(
3953 CodeGenFunction &CGF, const OMPExecutableDirective &S,
3954 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
3955 const auto &Dir = cast<OMPLoopDirective>(Val: S);
3956 LValue LB =
3957 CGF.EmitLValue(E: cast<DeclRefExpr>(Val: Dir.getCombinedLowerBoundVariable()));
3958 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3959 V: CGF.Builder.CreateLoad(Addr: LB.getAddress()), DestTy: CGF.SizeTy, /*isSigned=*/false);
3960 CapturedVars.push_back(Elt: LBCast);
3961 LValue UB =
3962 CGF.EmitLValue(E: cast<DeclRefExpr>(Val: Dir.getCombinedUpperBoundVariable()));
3963
3964 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3965 V: CGF.Builder.CreateLoad(Addr: UB.getAddress()), DestTy: CGF.SizeTy, /*isSigned=*/false);
3966 CapturedVars.push_back(Elt: UBCast);
3967}
3968
3969static void
3970emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
3971 const OMPLoopDirective &S,
3972 CodeGenFunction::JumpDest LoopExit) {
3973 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3974 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,
3975 PrePostActionTy &Action) {
3976 Action.Enter(CGF);
3977 bool HasCancel = false;
3978 if (!isOpenMPSimdDirective(DKind: EKind)) {
3979 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &S))
3980 HasCancel = D->hasCancel();
3981 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(Val: &S))
3982 HasCancel = D->hasCancel();
3983 else if (const auto *D =
3984 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &S))
3985 HasCancel = D->hasCancel();
3986 }
3987 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3988 CGF.EmitOMPWorksharingLoop(S, EUB: S.getPrevEnsureUpperBound(),
3989 CodeGenLoopBounds: emitDistributeParallelForInnerBounds,
3990 CGDispatchBounds: emitDistributeParallelForDispatchBounds);
3991 };
3992
3993 emitCommonOMPParallelDirective(
3994 CGF, S, InnermostKind: isOpenMPSimdDirective(DKind: EKind) ? OMPD_for_simd : OMPD_for,
3995 CodeGen: CGInlinedWorksharingLoop,
3996 CodeGenBoundParameters: emitDistributeParallelForDistributeInnerBoundParams);
3997}
3998
3999void CodeGenFunction::EmitOMPDistributeParallelForDirective(
4000 const OMPDistributeParallelForDirective &S) {
4001 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4002 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
4003 IncExpr: S.getDistInc());
4004 };
4005 OMPLexicalScope Scope(*this, S, OMPD_parallel);
4006 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_distribute, CodeGen);
4007}
4008
4009void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
4010 const OMPDistributeParallelForSimdDirective &S) {
4011 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4012 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
4013 IncExpr: S.getDistInc());
4014 };
4015 OMPLexicalScope Scope(*this, S, OMPD_parallel);
4016 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_distribute, CodeGen);
4017}
4018
4019void CodeGenFunction::EmitOMPDistributeSimdDirective(
4020 const OMPDistributeSimdDirective &S) {
4021 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4022 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
4023 };
4024 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4025 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_simd, CodeGen);
4026}
4027
4028void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
4029 CodeGenModule &CGM, StringRef ParentName, const OMPLoopDirective &S) {
4030 // Emit SPMD target parallel for region as a standalone region.
4031 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4032 emitOMPSimdRegion(CGF, S, Action);
4033 };
4034 llvm::Function *Fn;
4035 llvm::Constant *Addr;
4036 // Emit target region as a standalone region.
4037 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4038 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
4039 assert(Fn && Addr && "Target device function emission failed.");
4040}
4041
4042void CodeGenFunction::EmitOMPTargetSimdDirective(const OMPLoopDirective &S) {
4043 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4044 emitOMPSimdRegion(CGF, S, Action);
4045 };
4046 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
4047}
4048
4049namespace {
4050struct ScheduleKindModifiersTy {
4051 OpenMPScheduleClauseKind Kind;
4052 OpenMPScheduleClauseModifier M1;
4053 OpenMPScheduleClauseModifier M2;
4054 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
4055 OpenMPScheduleClauseModifier M1,
4056 OpenMPScheduleClauseModifier M2)
4057 : Kind(Kind), M1(M1), M2(M2) {}
4058};
4059} // namespace
4060
4061bool CodeGenFunction::EmitOMPWorksharingLoop(
4062 const OMPLoopDirective &S, Expr *EUB,
4063 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4064 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
4065 // Emit the loop iteration variable.
4066 const auto *IVExpr = cast<DeclRefExpr>(Val: S.getIterationVariable());
4067 const auto *IVDecl = cast<VarDecl>(Val: IVExpr->getDecl());
4068 EmitVarDecl(D: *IVDecl);
4069
4070 // Emit the iterations count variable.
4071 // If it is not a variable, Sema decided to calculate iterations count on each
4072 // iteration (e.g., it is foldable into a constant).
4073 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
4074 EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
4075 // Emit calculation of the iterations count.
4076 EmitIgnoredExpr(E: S.getCalcLastIteration());
4077 }
4078
4079 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
4080
4081 bool HasLastprivateClause;
4082 // Check pre-condition.
4083 {
4084 OMPLoopScope PreInitScope(*this, S);
4085 // Skip the entire loop if we don't meet the precondition.
4086 // If the condition constant folds and can be elided, avoid emitting the
4087 // whole loop.
4088 bool CondConstant;
4089 llvm::BasicBlock *ContBlock = nullptr;
4090 if (ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
4091 if (!CondConstant)
4092 return false;
4093 } else {
4094 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "omp.precond.then");
4095 ContBlock = createBasicBlock(name: "omp.precond.end");
4096 emitPreCond(CGF&: *this, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
4097 TrueCount: getProfileCount(S: &S));
4098 EmitBlock(BB: ThenBlock);
4099 incrementProfileCounter(S: &S);
4100 }
4101
4102 RunCleanupsScope DoacrossCleanupScope(*this);
4103 bool Ordered = false;
4104 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
4105 if (OrderedClause->getNumForLoops())
4106 RT.emitDoacrossInit(CGF&: *this, D: S, NumIterations: OrderedClause->getLoopNumIterations());
4107 else
4108 Ordered = true;
4109 }
4110
4111 emitAlignedClause(CGF&: *this, D: S);
4112 bool HasLinears = EmitOMPLinearClauseInit(D: S);
4113 // Emit helper vars inits.
4114
4115 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
4116 LValue LB = Bounds.first;
4117 LValue UB = Bounds.second;
4118 LValue ST =
4119 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()));
4120 LValue IL =
4121 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()));
4122
4123 // Emit 'then' code.
4124 {
4125 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4126 OMPPrivateScope LoopScope(*this);
4127 if (EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope) || HasLinears) {
4128 // Emit implicit barrier to synchronize threads and avoid data races on
4129 // initialization of firstprivate variables and post-update of
4130 // lastprivate variables.
4131 CGM.getOpenMPRuntime().emitBarrierCall(
4132 CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4133 /*ForceSimpleCall=*/true);
4134 }
4135 EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
4136 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
4137 *this, S, EmitLValue(E: S.getIterationVariable()));
4138 HasLastprivateClause = EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
4139 EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
4140 EmitOMPPrivateLoopCounters(S, LoopScope);
4141 EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
4142 (void)LoopScope.Privatize();
4143 if (isOpenMPTargetExecutionDirective(DKind: EKind))
4144 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF&: *this, D: S);
4145
4146 // Detect the loop schedule kind and chunk.
4147 const Expr *ChunkExpr = nullptr;
4148 OpenMPScheduleTy ScheduleKind;
4149 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
4150 ScheduleKind.Schedule = C->getScheduleKind();
4151 ScheduleKind.M1 = C->getFirstScheduleModifier();
4152 ScheduleKind.M2 = C->getSecondScheduleModifier();
4153 ChunkExpr = C->getChunkSize();
4154 } else {
4155 // Default behaviour for schedule clause.
4156 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
4157 CGF&: *this, S, ScheduleKind&: ScheduleKind.Schedule, ChunkExpr);
4158 }
4159 bool HasChunkSizeOne = false;
4160 llvm::Value *Chunk = nullptr;
4161 if (ChunkExpr) {
4162 Chunk = EmitScalarExpr(E: ChunkExpr);
4163 Chunk = EmitScalarConversion(Src: Chunk, SrcTy: ChunkExpr->getType(),
4164 DstTy: S.getIterationVariable()->getType(),
4165 Loc: S.getBeginLoc());
4166 Expr::EvalResult Result;
4167 if (ChunkExpr->EvaluateAsInt(Result, Ctx: getContext())) {
4168 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
4169 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
4170 }
4171 }
4172 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
4173 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
4174 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
4175 // If the static schedule kind is specified or if the ordered clause is
4176 // specified, and if no monotonic modifier is specified, the effect will
4177 // be as if the monotonic modifier was specified.
4178 bool StaticChunkedOne =
4179 RT.isStaticChunked(ScheduleKind: ScheduleKind.Schedule,
4180 /* Chunked */ Chunk != nullptr) &&
4181 HasChunkSizeOne && isOpenMPLoopBoundSharingDirective(Kind: EKind);
4182 // GPU combined `distribute parallel for`: emit a single
4183 // for_static_init with the fused distr_static_chunk + static_chunkone
4184 // schedule (enum 93). The surrounding EmitOMPDistributeLoop must skip
4185 // its distribute_static_init under the same conditions. Both sites are
4186 // guarded by canEmitGPUFusedDistSchedule() alone so they cannot
4187 // disagree; the assert guards the invariant that makes this safe today,
4188 // aka that the implicit GPU default schedule is always static chunk-one.
4189 ScheduleKind.UseFusedDistChunkSchedule =
4190 canEmitGPUFusedDistSchedule(CGM, S, DKind: EKind);
4191 assert((!ScheduleKind.UseFusedDistChunkSchedule || StaticChunkedOne) &&
4192 "fused distribute schedule requires a static chunk-one schedule");
4193 bool IsMonotonic =
4194 Ordered ||
4195 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
4196 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
4197 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
4198 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
4199 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
4200 if ((RT.isStaticNonchunked(ScheduleKind: ScheduleKind.Schedule,
4201 /* Chunked */ Chunk != nullptr) ||
4202 StaticChunkedOne) &&
4203 !Ordered) {
4204 JumpDest LoopExit =
4205 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
4206 emitCommonSimdLoop(
4207 CGF&: *this, S,
4208 SimdInitGen: [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
4209 if (isOpenMPSimdDirective(DKind: EKind)) {
4210 CGF.EmitOMPSimdInit(D: S);
4211 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
4212 if (C->getKind() == OMPC_ORDER_concurrent)
4213 CGF.LoopStack.setParallel(/*Enable=*/true);
4214 }
4215 },
4216 BodyCodeGen: [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
4217 &S, ScheduleKind, LoopExit, EKind,
4218 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
4219 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
4220 // When no chunk_size is specified, the iteration space is divided
4221 // into chunks that are approximately equal in size, and at most
4222 // one chunk is distributed to each thread. Note that the size of
4223 // the chunks is unspecified in this case.
4224 CGOpenMPRuntime::StaticRTInput StaticInit(
4225 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
4226 UB.getAddress(), ST.getAddress(),
4227 StaticChunkedOne ? Chunk : nullptr);
4228 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4229 CGF, Loc: S.getBeginLoc(), DKind: EKind, ScheduleKind, Values: StaticInit);
4230 // UB = min(UB, GlobalUB);
4231 if (!StaticChunkedOne)
4232 CGF.EmitIgnoredExpr(E: S.getEnsureUpperBound());
4233 // IV = LB;
4234 CGF.EmitIgnoredExpr(E: S.getInit());
4235 // For unchunked static schedule generate:
4236 //
4237 // while (idx <= UB) {
4238 // BODY;
4239 // ++idx;
4240 // }
4241 //
4242 // For static schedule with chunk one:
4243 //
4244 // while (IV <= PrevUB) {
4245 // BODY;
4246 // IV += ST;
4247 // }
4248 CGF.EmitOMPInnerLoop(
4249 S, RequiresCleanup: LoopScope.requiresCleanups(),
4250 LoopCond: StaticChunkedOne ? S.getCombinedParForInDistCond()
4251 : S.getCond(),
4252 IncExpr: StaticChunkedOne ? S.getDistInc() : S.getInc(),
4253 BodyGen: [&S, LoopExit](CodeGenFunction &CGF) {
4254 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
4255 },
4256 PostIncGen: [](CodeGenFunction &) {});
4257 });
4258 EmitBlock(BB: LoopExit.getBlock());
4259 // Tell the runtime we are done.
4260 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4261 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
4262 DKind: OMPD_for);
4263 };
4264 OMPCancelStack.emitExit(CGF&: *this, Kind: EKind, CodeGen);
4265 } else {
4266 // Emit the outer loop, which requests its work chunk [LB..UB] from
4267 // runtime and runs the inner loop to process it.
4268 OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
4269 ST.getAddress(), IL.getAddress(), Chunk,
4270 EUB);
4271 LoopArguments.DKind = OMPD_for;
4272 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
4273 LoopArgs: LoopArguments, CGDispatchBounds);
4274 }
4275 if (isOpenMPSimdDirective(DKind: EKind)) {
4276 EmitOMPSimdFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4277 return CGF.Builder.CreateIsNotNull(
4278 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4279 });
4280 }
4281 EmitOMPReductionClauseFinal(
4282 D: S, /*ReductionKind=*/isOpenMPSimdDirective(DKind: EKind)
4283 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
4284 : /*Parallel only*/ OMPD_parallel);
4285 // Emit post-update of the reduction variables if IsLastIter != 0.
4286 emitPostUpdateForReductionClause(
4287 CGF&: *this, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4288 return CGF.Builder.CreateIsNotNull(
4289 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4290 });
4291 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4292 if (HasLastprivateClause)
4293 EmitOMPLastprivateClauseFinal(
4294 D: S, NoFinals: isOpenMPSimdDirective(DKind: EKind),
4295 IsLastIterCond: Builder.CreateIsNotNull(Arg: EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
4296 LoopScope.restoreMap();
4297 EmitOMPLinearClauseFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4298 return CGF.Builder.CreateIsNotNull(
4299 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4300 });
4301 }
4302 DoacrossCleanupScope.ForceCleanup();
4303 // We're now done with the loop, so jump to the continuation block.
4304 if (ContBlock) {
4305 EmitBranch(Block: ContBlock);
4306 EmitBlock(BB: ContBlock, /*IsFinished=*/true);
4307 }
4308 }
4309 return HasLastprivateClause;
4310}
4311
4312/// The following two functions generate expressions for the loop lower
4313/// and upper bounds in case of static and dynamic (dispatch) schedule
4314/// of the associated 'for' or 'distribute' loop.
4315static std::pair<LValue, LValue>
4316emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4317 const auto &LS = cast<OMPLoopDirective>(Val: S);
4318 LValue LB =
4319 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getLowerBoundVariable()));
4320 LValue UB =
4321 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getUpperBoundVariable()));
4322 return {LB, UB};
4323}
4324
4325/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
4326/// consider the lower and upper bound expressions generated by the
4327/// worksharing loop support, but we use 0 and the iteration space size as
4328/// constants
4329static std::pair<llvm::Value *, llvm::Value *>
4330emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
4331 Address LB, Address UB) {
4332 const auto &LS = cast<OMPLoopDirective>(Val: S);
4333 const Expr *IVExpr = LS.getIterationVariable();
4334 const unsigned IVSize = CGF.getContext().getTypeSize(T: IVExpr->getType());
4335 llvm::Value *LBVal = CGF.Builder.getIntN(N: IVSize, C: 0);
4336 llvm::Value *UBVal = CGF.EmitScalarExpr(E: LS.getLastIteration());
4337 return {LBVal, UBVal};
4338}
4339
4340/// Emits internal temp array declarations for the directive with inscan
4341/// reductions.
4342/// The code is the following:
4343/// \code
4344/// size num_iters = <num_iters>;
4345/// <type> buffer[num_iters];
4346/// \endcode
4347static void emitScanBasedDirectiveDecls(
4348 CodeGenFunction &CGF, const OMPLoopDirective &S,
4349 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4350 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4351 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4352 SmallVector<const Expr *, 4> Shareds;
4353 SmallVector<const Expr *, 4> Privates;
4354 SmallVector<const Expr *, 4> ReductionOps;
4355 SmallVector<const Expr *, 4> CopyArrayTemps;
4356 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4357 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4358 "Only inscan reductions are expected.");
4359 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
4360 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4361 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
4362 CopyArrayTemps.append(in_start: C->copy_array_temps().begin(),
4363 in_end: C->copy_array_temps().end());
4364 }
4365 {
4366 // Emit buffers for each reduction variables.
4367 // ReductionCodeGen is required to emit correctly the code for array
4368 // reductions.
4369 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
4370 unsigned Count = 0;
4371 auto *ITA = CopyArrayTemps.begin();
4372 for (const Expr *IRef : Privates) {
4373 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IRef)->getDecl());
4374 // Emit variably modified arrays, used for arrays/array sections
4375 // reductions.
4376 if (PrivateVD->getType()->isVariablyModifiedType()) {
4377 RedCG.emitSharedOrigLValue(CGF, N: Count);
4378 RedCG.emitAggregateType(CGF, N: Count);
4379 }
4380 CodeGenFunction::OpaqueValueMapping DimMapping(
4381 CGF,
4382 cast<OpaqueValueExpr>(
4383 Val: cast<VariableArrayType>(Val: (*ITA)->getType()->getAsArrayTypeUnsafe())
4384 ->getSizeExpr()),
4385 RValue::get(V: OMPScanNumIterations));
4386 // Emit temp buffer.
4387 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ITA)->getDecl()));
4388 ++ITA;
4389 ++Count;
4390 }
4391 }
4392}
4393
4394/// Copies final inscan reductions values to the original variables.
4395/// The code is the following:
4396/// \code
4397/// <orig_var> = buffer[num_iters-1];
4398/// \endcode
4399static void emitScanBasedDirectiveFinals(
4400 CodeGenFunction &CGF, const OMPLoopDirective &S,
4401 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4402 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4403 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4404 SmallVector<const Expr *, 4> Shareds;
4405 SmallVector<const Expr *, 4> LHSs;
4406 SmallVector<const Expr *, 4> RHSs;
4407 SmallVector<const Expr *, 4> Privates;
4408 SmallVector<const Expr *, 4> CopyOps;
4409 SmallVector<const Expr *, 4> CopyArrayElems;
4410 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4411 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4412 "Only inscan reductions are expected.");
4413 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
4414 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
4415 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
4416 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4417 CopyOps.append(in_start: C->copy_ops().begin(), in_end: C->copy_ops().end());
4418 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
4419 in_end: C->copy_array_elems().end());
4420 }
4421 // Create temp var and copy LHS value to this temp value.
4422 // LHS = TMP[LastIter];
4423 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
4424 LHS: OMPScanNumIterations,
4425 RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1, /*isSigned=*/IsSigned: false));
4426 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
4427 const Expr *PrivateExpr = Privates[I];
4428 const Expr *OrigExpr = Shareds[I];
4429 const Expr *CopyArrayElem = CopyArrayElems[I];
4430 CodeGenFunction::OpaqueValueMapping IdxMapping(
4431 CGF,
4432 cast<OpaqueValueExpr>(
4433 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4434 RValue::get(V: OMPLast));
4435 LValue DestLVal = CGF.EmitLValue(E: OrigExpr);
4436 LValue SrcLVal = CGF.EmitLValue(E: CopyArrayElem);
4437 CGF.EmitOMPCopy(
4438 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
4439 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
4440 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
4441 }
4442}
4443
4444/// Emits the code for the directive with inscan reductions.
4445/// The code is the following:
4446/// \code
4447/// #pragma omp ...
4448/// for (i: 0..<num_iters>) {
4449/// <input phase>;
4450/// buffer[i] = red;
4451/// }
4452/// #pragma omp master // in parallel region
4453/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
4454/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
4455/// buffer[i] op= buffer[i-pow(2,k)];
4456/// #pragma omp barrier // in parallel region
4457/// #pragma omp ...
4458/// for (0..<num_iters>) {
4459/// red = InclusiveScan ? buffer[i] : buffer[i-1];
4460/// <scan phase>;
4461/// }
4462/// \endcode
4463static void emitScanBasedDirective(
4464 CodeGenFunction &CGF, const OMPLoopDirective &S,
4465 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
4466 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
4467 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
4468 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4469 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4470 SmallVector<const Expr *, 4> Privates;
4471 SmallVector<const Expr *, 4> ReductionOps;
4472 SmallVector<const Expr *, 4> LHSs;
4473 SmallVector<const Expr *, 4> RHSs;
4474 SmallVector<const Expr *, 4> CopyArrayElems;
4475 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4476 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4477 "Only inscan reductions are expected.");
4478 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4479 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
4480 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
4481 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
4482 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
4483 in_end: C->copy_array_elems().end());
4484 }
4485 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
4486 {
4487 // Emit loop with input phase:
4488 // #pragma omp ...
4489 // for (i: 0..<num_iters>) {
4490 // <input phase>;
4491 // buffer[i] = red;
4492 // }
4493 CGF.OMPFirstScanLoop = true;
4494 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4495 FirstGen(CGF);
4496 }
4497 // #pragma omp barrier // in parallel region
4498 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
4499 &ReductionOps,
4500 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
4501 Action.Enter(CGF);
4502 // Emit prefix reduction:
4503 // #pragma omp master // in parallel region
4504 // for (int k = 0; k <= ceil(log2(n)); ++k)
4505 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
4506 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(name: "omp.outer.log.scan.body");
4507 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "omp.outer.log.scan.exit");
4508 llvm::Function *F =
4509 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::log2, Tys: CGF.DoubleTy);
4510 llvm::Value *Arg =
4511 CGF.Builder.CreateUIToFP(V: OMPScanNumIterations, DestTy: CGF.DoubleTy);
4512 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(callee: F, args: Arg);
4513 F = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ceil, Tys: CGF.DoubleTy);
4514 LogVal = CGF.EmitNounwindRuntimeCall(callee: F, args: LogVal);
4515 LogVal = CGF.Builder.CreateFPToUI(V: LogVal, DestTy: CGF.IntTy);
4516 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
4517 LHS: OMPScanNumIterations, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1));
4518 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: S.getBeginLoc());
4519 CGF.EmitBlock(BB: LoopBB);
4520 auto *Counter = CGF.Builder.CreatePHI(Ty: CGF.IntTy, NumReservedValues: 2);
4521 // size pow2k = 1;
4522 auto *Pow2K = CGF.Builder.CreatePHI(Ty: CGF.SizeTy, NumReservedValues: 2);
4523 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0), BB: InputBB);
4524 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), BB: InputBB);
4525 // for (size i = n - 1; i >= 2 ^ k; --i)
4526 // tmp[i] op= tmp[i-pow2k];
4527 llvm::BasicBlock *InnerLoopBB =
4528 CGF.createBasicBlock(name: "omp.inner.log.scan.body");
4529 llvm::BasicBlock *InnerExitBB =
4530 CGF.createBasicBlock(name: "omp.inner.log.scan.exit");
4531 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
4532 CGF.Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
4533 CGF.EmitBlock(BB: InnerLoopBB);
4534 auto *IVal = CGF.Builder.CreatePHI(Ty: CGF.SizeTy, NumReservedValues: 2);
4535 IVal->addIncoming(V: NMin1, BB: LoopBB);
4536 {
4537 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
4538 auto *ILHS = LHSs.begin();
4539 auto *IRHS = RHSs.begin();
4540 for (const Expr *CopyArrayElem : CopyArrayElems) {
4541 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
4542 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
4543 Address LHSAddr = Address::invalid();
4544 {
4545 CodeGenFunction::OpaqueValueMapping IdxMapping(
4546 CGF,
4547 cast<OpaqueValueExpr>(
4548 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4549 RValue::get(V: IVal));
4550 LHSAddr = CGF.EmitLValue(E: CopyArrayElem).getAddress();
4551 }
4552 PrivScope.addPrivate(LocalVD: LHSVD, Addr: LHSAddr);
4553 Address RHSAddr = Address::invalid();
4554 {
4555 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(LHS: IVal, RHS: Pow2K);
4556 CodeGenFunction::OpaqueValueMapping IdxMapping(
4557 CGF,
4558 cast<OpaqueValueExpr>(
4559 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4560 RValue::get(V: OffsetIVal));
4561 RHSAddr = CGF.EmitLValue(E: CopyArrayElem).getAddress();
4562 }
4563 PrivScope.addPrivate(LocalVD: RHSVD, Addr: RHSAddr);
4564 ++ILHS;
4565 ++IRHS;
4566 }
4567 PrivScope.Privatize();
4568 CGF.CGM.getOpenMPRuntime().emitReduction(
4569 CGF, Loc: S.getEndLoc(), Privates, LHSExprs: LHSs, RHSExprs: RHSs, ReductionOps,
4570 Options: {/*WithNowait=*/true, /*SimpleReduction=*/true,
4571 /*IsPrivateVarReduction*/ {}, .ReductionKind: OMPD_unknown});
4572 }
4573 llvm::Value *NextIVal =
4574 CGF.Builder.CreateNUWSub(LHS: IVal, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1));
4575 IVal->addIncoming(V: NextIVal, BB: CGF.Builder.GetInsertBlock());
4576 CmpI = CGF.Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
4577 CGF.Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
4578 CGF.EmitBlock(BB: InnerExitBB);
4579 llvm::Value *Next =
4580 CGF.Builder.CreateNUWAdd(LHS: Counter, RHS: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1));
4581 Counter->addIncoming(V: Next, BB: CGF.Builder.GetInsertBlock());
4582 // pow2k <<= 1;
4583 llvm::Value *NextPow2K =
4584 CGF.Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
4585 Pow2K->addIncoming(V: NextPow2K, BB: CGF.Builder.GetInsertBlock());
4586 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
4587 CGF.Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
4588 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: S.getEndLoc());
4589 CGF.EmitBlock(BB: ExitBB);
4590 };
4591 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4592 if (isOpenMPParallelDirective(DKind: EKind)) {
4593 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
4594 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4595 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4596 /*ForceSimpleCall=*/true);
4597 } else {
4598 RegionCodeGenTy RCG(CodeGen);
4599 RCG(CGF);
4600 }
4601
4602 CGF.OMPFirstScanLoop = false;
4603 SecondGen(CGF);
4604}
4605
4606static bool emitWorksharingDirective(CodeGenFunction &CGF,
4607 const OMPLoopDirective &S,
4608 bool HasCancel) {
4609 bool HasLastprivates;
4610 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4611 if (llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
4612 P: [](const OMPReductionClause *C) {
4613 return C->getModifier() == OMPC_REDUCTION_inscan;
4614 })) {
4615 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4616 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4617 OMPLoopScope LoopScope(CGF, S);
4618 return CGF.EmitScalarExpr(E: S.getNumIterations());
4619 };
4620 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {
4621 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4622 (void)CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4623 CodeGenLoopBounds: emitForLoopBounds,
4624 CGDispatchBounds: emitDispatchForLoopBounds);
4625 // Emit an implicit barrier at the end.
4626 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc: S.getBeginLoc(),
4627 Kind: OMPD_for);
4628 };
4629 const auto &&SecondGen = [&S, HasCancel, EKind,
4630 &HasLastprivates](CodeGenFunction &CGF) {
4631 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4632 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4633 CodeGenLoopBounds: emitForLoopBounds,
4634 CGDispatchBounds: emitDispatchForLoopBounds);
4635 };
4636 if (!isOpenMPParallelDirective(DKind: EKind))
4637 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
4638 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
4639 if (!isOpenMPParallelDirective(DKind: EKind))
4640 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
4641 } else {
4642 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4643 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4644 CodeGenLoopBounds: emitForLoopBounds,
4645 CGDispatchBounds: emitDispatchForLoopBounds);
4646 }
4647 return HasLastprivates;
4648}
4649
4650// Pass OMPLoopDirective (instead of OMPForDirective) to make this check
4651// available for "loop bind(parallel)", which maps to "for".
4652static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S,
4653 bool HasCancel) {
4654 if (HasCancel)
4655 return false;
4656 for (OMPClause *C : S.clauses()) {
4657 if (isa<OMPNowaitClause, OMPBindClause>(Val: C))
4658 continue;
4659
4660 if (auto *SC = dyn_cast<OMPScheduleClause>(Val: C)) {
4661 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4662 return false;
4663 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4664 return false;
4665 switch (SC->getScheduleKind()) {
4666 case OMPC_SCHEDULE_auto:
4667 case OMPC_SCHEDULE_dynamic:
4668 case OMPC_SCHEDULE_runtime:
4669 case OMPC_SCHEDULE_guided:
4670 case OMPC_SCHEDULE_static:
4671 continue;
4672 case OMPC_SCHEDULE_unknown:
4673 return false;
4674 }
4675 }
4676
4677 return false;
4678 }
4679
4680 return true;
4681}
4682
4683static llvm::omp::ScheduleKind
4684convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind) {
4685 switch (ScheduleClauseKind) {
4686 case OMPC_SCHEDULE_unknown:
4687 return llvm::omp::OMP_SCHEDULE_Default;
4688 case OMPC_SCHEDULE_auto:
4689 return llvm::omp::OMP_SCHEDULE_Auto;
4690 case OMPC_SCHEDULE_dynamic:
4691 return llvm::omp::OMP_SCHEDULE_Dynamic;
4692 case OMPC_SCHEDULE_guided:
4693 return llvm::omp::OMP_SCHEDULE_Guided;
4694 case OMPC_SCHEDULE_runtime:
4695 return llvm::omp::OMP_SCHEDULE_Runtime;
4696 case OMPC_SCHEDULE_static:
4697 return llvm::omp::OMP_SCHEDULE_Static;
4698 }
4699 llvm_unreachable("Unhandled schedule kind");
4700}
4701
4702// Pass OMPLoopDirective (instead of OMPForDirective) to make this function
4703// available for "loop bind(parallel)", which maps to "for".
4704static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF,
4705 CodeGenModule &CGM, bool HasCancel) {
4706 bool HasLastprivates = false;
4707 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder &&
4708 isForSupportedByOpenMPIRBuilder(S, HasCancel);
4709 auto &&CodeGen = [&S, &CGM, HasCancel, &HasLastprivates,
4710 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
4711 // Use the OpenMPIRBuilder if enabled.
4712 if (UseOMPIRBuilder) {
4713 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
4714
4715 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4716 llvm::Value *ChunkSize = nullptr;
4717 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
4718 SchedKind =
4719 convertClauseKindToSchedKind(ScheduleClauseKind: SchedClause->getScheduleKind());
4720 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4721 ChunkSize = CGF.EmitScalarExpr(E: ChunkSizeExpr);
4722 }
4723
4724 // Emit the associated statement and get its loop representation.
4725 const Stmt *Inner = S.getRawStmt();
4726 llvm::CanonicalLoopInfo *CLI =
4727 CGF.EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
4728
4729 llvm::OpenMPIRBuilder &OMPBuilder =
4730 CGM.getOpenMPRuntime().getOMPBuilder();
4731 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4732 CGF.AllocaInsertPt->getIterator());
4733 cantFail(ValOrErr: OMPBuilder.applyWorkshareLoop(
4734 DL: CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4735 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
4736 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
4737 /*HasOrderedClause=*/false));
4738 return;
4739 }
4740
4741 HasLastprivates = emitWorksharingDirective(CGF, S, HasCancel);
4742 };
4743 {
4744 auto LPCRegion =
4745 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
4746 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
4747 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_for, CodeGen,
4748 HasCancel);
4749 }
4750
4751 if (!UseOMPIRBuilder) {
4752 // Emit an implicit barrier at the end.
4753 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4754 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc: S.getBeginLoc(), Kind: OMPD_for);
4755 }
4756 // Check for outer lastprivate conditional update.
4757 checkForLastprivateConditionalUpdate(CGF, S);
4758}
4759
4760void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
4761 return emitOMPForDirective(S, CGF&: *this, CGM, HasCancel: S.hasCancel());
4762}
4763
4764void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
4765 bool HasLastprivates = false;
4766 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4767 PrePostActionTy &) {
4768 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4769 };
4770 {
4771 auto LPCRegion =
4772 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4773 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4774 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_simd, CodeGen);
4775 }
4776
4777 // Emit an implicit barrier at the end.
4778 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4779 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_for);
4780 // Check for outer lastprivate conditional update.
4781 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4782}
4783
4784static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
4785 const Twine &Name,
4786 llvm::Value *Init = nullptr) {
4787 LValue LVal = CGF.MakeAddrLValue(Addr: CGF.CreateMemTemp(T: Ty, Name), T: Ty);
4788 if (Init)
4789 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Init), Dst: LVal, /*isInit*/ true);
4790 return LVal;
4791}
4792
4793void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
4794 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4795 const auto *CS = dyn_cast<CompoundStmt>(Val: CapturedStmt);
4796 bool HasLastprivates = false;
4797 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4798 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4799 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4800 const ASTContext &C = CGF.getContext();
4801 QualType KmpInt32Ty =
4802 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4803 // Emit helper vars inits.
4804 LValue LB = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.lb.",
4805 Init: CGF.Builder.getInt32(C: 0));
4806 llvm::ConstantInt *GlobalUBVal = CS != nullptr
4807 ? CGF.Builder.getInt32(C: CS->size() - 1)
4808 : CGF.Builder.getInt32(C: 0);
4809 LValue UB =
4810 createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.ub.", Init: GlobalUBVal);
4811 LValue ST = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.st.",
4812 Init: CGF.Builder.getInt32(C: 1));
4813 LValue IL = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.il.",
4814 Init: CGF.Builder.getInt32(C: 0));
4815 // Loop counter.
4816 LValue IV = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.iv.");
4817 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4818 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4819 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4820 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4821 // Generate condition for loop.
4822 BinaryOperator *Cond = BinaryOperator::Create(
4823 C, lhs: &IVRefExpr, rhs: &UBRefExpr, opc: BO_LE, ResTy: C.BoolTy, VK: VK_PRValue, OK: OK_Ordinary,
4824 opLoc: S.getBeginLoc(), FPFeatures: FPOptionsOverride());
4825 // Increment for loop counter.
4826 UnaryOperator *Inc = UnaryOperator::Create(
4827 C, input: &IVRefExpr, opc: UO_PreInc, type: KmpInt32Ty, VK: VK_PRValue, OK: OK_Ordinary,
4828 l: S.getBeginLoc(), CanOverflow: true, FPFeatures: FPOptionsOverride());
4829 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4830 // Iterate through all sections and emit a switch construct:
4831 // switch (IV) {
4832 // case 0:
4833 // <SectionStmt[0]>;
4834 // break;
4835 // ...
4836 // case <NumSection> - 1:
4837 // <SectionStmt[<NumSection> - 1]>;
4838 // break;
4839 // }
4840 // .omp.sections.exit:
4841 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".omp.sections.exit");
4842 llvm::SwitchInst *SwitchStmt =
4843 CGF.Builder.CreateSwitch(V: CGF.EmitLoadOfScalar(lvalue: IV, Loc: S.getBeginLoc()),
4844 Dest: ExitBB, NumCases: CS == nullptr ? 1 : CS->size());
4845 if (CS) {
4846 unsigned CaseNumber = 0;
4847 for (const Stmt *SubStmt : CS->children()) {
4848 auto CaseBB = CGF.createBasicBlock(name: ".omp.sections.case");
4849 CGF.EmitBlock(BB: CaseBB);
4850 SwitchStmt->addCase(OnVal: CGF.Builder.getInt32(C: CaseNumber), Dest: CaseBB);
4851 CGF.EmitStmt(S: SubStmt);
4852 CGF.EmitBranch(Block: ExitBB);
4853 ++CaseNumber;
4854 }
4855 } else {
4856 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(name: ".omp.sections.case");
4857 CGF.EmitBlock(BB: CaseBB);
4858 SwitchStmt->addCase(OnVal: CGF.Builder.getInt32(C: 0), Dest: CaseBB);
4859 CGF.EmitStmt(S: CapturedStmt);
4860 CGF.EmitBranch(Block: ExitBB);
4861 }
4862 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
4863 };
4864
4865 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4866 if (CGF.EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope)) {
4867 // Emit implicit barrier to synchronize threads and avoid data races on
4868 // initialization of firstprivate variables and post-update of lastprivate
4869 // variables.
4870 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4871 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4872 /*ForceSimpleCall=*/true);
4873 }
4874 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
4875 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4876 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
4877 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
4878 (void)LoopScope.Privatize();
4879 if (isOpenMPTargetExecutionDirective(DKind: EKind))
4880 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
4881
4882 // Emit static non-chunked loop.
4883 OpenMPScheduleTy ScheduleKind;
4884 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4885 CGOpenMPRuntime::StaticRTInput StaticInit(
4886 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
4887 LB.getAddress(), UB.getAddress(), ST.getAddress());
4888 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, Loc: S.getBeginLoc(), DKind: EKind,
4889 ScheduleKind, Values: StaticInit);
4890 // UB = min(UB, GlobalUB);
4891 llvm::Value *UBVal = CGF.EmitLoadOfScalar(lvalue: UB, Loc: S.getBeginLoc());
4892 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4893 C: CGF.Builder.CreateICmpSLT(LHS: UBVal, RHS: GlobalUBVal), True: UBVal, False: GlobalUBVal);
4894 CGF.EmitStoreOfScalar(value: MinUBGlobalUB, lvalue: UB);
4895 // IV = LB;
4896 CGF.EmitStoreOfScalar(value: CGF.EmitLoadOfScalar(lvalue: LB, Loc: S.getBeginLoc()), lvalue: IV);
4897 // while (idx <= UB) { BODY; ++idx; }
4898 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, LoopCond: Cond, IncExpr: Inc, BodyGen,
4899 PostIncGen: [](CodeGenFunction &) {});
4900 // Tell the runtime we are done.
4901 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4902 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
4903 DKind: OMPD_sections);
4904 };
4905 CGF.OMPCancelStack.emitExit(CGF, Kind: EKind, CodeGen);
4906 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
4907 // Emit post-update of the reduction variables if IsLastIter != 0.
4908 emitPostUpdateForReductionClause(CGF, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4909 return CGF.Builder.CreateIsNotNull(
4910 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4911 });
4912
4913 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4914 if (HasLastprivates)
4915 CGF.EmitOMPLastprivateClauseFinal(
4916 D: S, /*NoFinals=*/false,
4917 IsLastIterCond: CGF.Builder.CreateIsNotNull(
4918 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
4919 };
4920
4921 bool HasCancel = false;
4922 if (auto *OSD = dyn_cast<OMPSectionsDirective>(Val: &S))
4923 HasCancel = OSD->hasCancel();
4924 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &S))
4925 HasCancel = OPSD->hasCancel();
4926 OMPCancelStackRAII CancelRegion(*this, EKind, HasCancel);
4927 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_sections, CodeGen,
4928 HasCancel);
4929 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4930 // clause. Otherwise the barrier will be generated by the codegen for the
4931 // directive.
4932 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4933 // Emit implicit barrier to synchronize threads and avoid data races on
4934 // initialization of firstprivate variables.
4935 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(),
4936 Kind: OMPD_unknown);
4937 }
4938}
4939
4940void CodeGenFunction::EmitOMPScopeDirective(const OMPScopeDirective &S) {
4941 {
4942 // Emit code for 'scope' region
4943 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4944 Action.Enter(CGF);
4945 OMPPrivateScope PrivateScope(CGF);
4946 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
4947 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
4948 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
4949 (void)PrivateScope.Privatize();
4950 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
4951 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
4952 };
4953 auto LPCRegion =
4954 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4955 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4956 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_scope, CodeGen);
4957 }
4958 // Emit an implicit barrier at the end.
4959 if (!S.getSingleClause<OMPNowaitClause>()) {
4960 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_scope);
4961 }
4962 // Check for outer lastprivate conditional update.
4963 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4964}
4965
4966void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
4967 if (CGM.getLangOpts().OpenMPIRBuilder) {
4968 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4969 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4970 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4971
4972 auto FiniCB = [](InsertPointTy IP) {
4973 // Don't FinalizeOMPRegion because this is done inside of OMPIRBuilder for
4974 // sections.
4975 return llvm::Error::success();
4976 };
4977
4978 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4979 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4980 const auto *CS = dyn_cast<CompoundStmt>(Val: CapturedStmt);
4981 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4982 if (CS) {
4983 for (const Stmt *SubStmt : CS->children()) {
4984 auto SectionCB = [this, SubStmt](
4985 InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4986 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4987 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(CGF&: *this, RegionBodyStmt: SubStmt, AllocaIP: AllocIP,
4988 CodeGenIP, RegionName: "section");
4989 return llvm::Error::success();
4990 };
4991 SectionCBVector.push_back(Elt: SectionCB);
4992 }
4993 } else {
4994 auto SectionCB =
4995 [this, CapturedStmt](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4996 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4997 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4998 CGF&: *this, RegionBodyStmt: CapturedStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "section");
4999 return llvm::Error::success();
5000 };
5001 SectionCBVector.push_back(Elt: SectionCB);
5002 }
5003
5004 // Privatization callback that performs appropriate action for
5005 // shared/private/firstprivate/lastprivate/copyin/... variables.
5006 //
5007 // TODO: This defaults to shared right now.
5008 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5009 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
5010 // The next line is appropriate only for variables (Val) with the
5011 // data-sharing attribute "shared".
5012 ReplVal = &Val;
5013
5014 return CodeGenIP;
5015 };
5016
5017 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
5018 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
5019 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
5020 AllocaInsertPt->getIterator());
5021 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5022 cantFail(ValOrErr: OMPBuilder.createSections(
5023 Loc: Builder, AllocaIP, SectionCBs: SectionCBVector, PrivCB, FiniCB, IsCancellable: S.hasCancel(),
5024 IsNowait: S.getSingleClause<OMPNowaitClause>()));
5025 Builder.restoreIP(IP: AfterIP);
5026 return;
5027 }
5028 {
5029 auto LPCRegion =
5030 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5031 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5032 EmitSections(S);
5033 }
5034 // Emit an implicit barrier at the end.
5035 if (!S.getSingleClause<OMPNowaitClause>()) {
5036 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(),
5037 Kind: OMPD_sections);
5038 }
5039 // Check for outer lastprivate conditional update.
5040 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5041}
5042
5043void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
5044 if (CGM.getLangOpts().OpenMPIRBuilder) {
5045 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5046 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5047
5048 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
5049 auto FiniCB = [this](InsertPointTy IP) {
5050 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5051 return llvm::Error::success();
5052 };
5053
5054 auto BodyGenCB = [SectionRegionBodyStmt,
5055 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5056 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5057 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5058 CGF&: *this, RegionBodyStmt: SectionRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "section");
5059 return llvm::Error::success();
5060 };
5061
5062 LexicalScope Scope(*this, S.getSourceRange());
5063 EmitStopPoint(S: &S);
5064 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5065 cantFail(ValOrErr: OMPBuilder.createSection(Loc: Builder, BodyGenCB, FiniCB));
5066 Builder.restoreIP(IP: AfterIP);
5067
5068 return;
5069 }
5070 LexicalScope Scope(*this, S.getSourceRange());
5071 EmitStopPoint(S: &S);
5072 EmitStmt(S: S.getAssociatedStmt());
5073}
5074
5075void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
5076 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
5077 llvm::SmallVector<const Expr *, 8> DestExprs;
5078 llvm::SmallVector<const Expr *, 8> SrcExprs;
5079 llvm::SmallVector<const Expr *, 8> AssignmentOps;
5080 // Check if there are any 'copyprivate' clauses associated with this
5081 // 'single' construct.
5082 // Build a list of copyprivate variables along with helper expressions
5083 // (<source>, <destination>, <destination>=<source> expressions)
5084 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
5085 CopyprivateVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5086 DestExprs.append(in_start: C->destination_exprs().begin(),
5087 in_end: C->destination_exprs().end());
5088 SrcExprs.append(in_start: C->source_exprs().begin(), in_end: C->source_exprs().end());
5089 AssignmentOps.append(in_start: C->assignment_ops().begin(),
5090 in_end: C->assignment_ops().end());
5091 }
5092 // Emit code for 'single' region along with 'copyprivate' clauses
5093 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5094 Action.Enter(CGF);
5095 OMPPrivateScope SingleScope(CGF);
5096 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope&: SingleScope);
5097 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: SingleScope);
5098 (void)SingleScope.Privatize();
5099 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
5100 };
5101 {
5102 auto LPCRegion =
5103 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5104 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5105 CGM.getOpenMPRuntime().emitSingleRegion(CGF&: *this, SingleOpGen: CodeGen, Loc: S.getBeginLoc(),
5106 CopyprivateVars, DestExprs,
5107 SrcExprs, AssignmentOps);
5108 }
5109 // Emit an implicit barrier at the end (to avoid data race on firstprivate
5110 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
5111 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
5112 CGM.getOpenMPRuntime().emitBarrierCall(
5113 CGF&: *this, Loc: S.getBeginLoc(),
5114 Kind: S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
5115 }
5116 // Check for outer lastprivate conditional update.
5117 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5118}
5119
5120static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
5121 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5122 Action.Enter(CGF);
5123 CGF.EmitStmt(S: S.getRawStmt());
5124 };
5125 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
5126}
5127
5128void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
5129 if (CGM.getLangOpts().OpenMPIRBuilder) {
5130 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5131 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5132
5133 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
5134
5135 auto FiniCB = [this](InsertPointTy IP) {
5136 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5137 return llvm::Error::success();
5138 };
5139
5140 auto BodyGenCB = [MasterRegionBodyStmt,
5141 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5142 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5143 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5144 CGF&: *this, RegionBodyStmt: MasterRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "master");
5145 return llvm::Error::success();
5146 };
5147
5148 LexicalScope Scope(*this, S.getSourceRange());
5149 EmitStopPoint(S: &S);
5150 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5151 cantFail(ValOrErr: OMPBuilder.createMaster(Loc: Builder, BodyGenCB, FiniCB));
5152 Builder.restoreIP(IP: AfterIP);
5153
5154 return;
5155 }
5156 LexicalScope Scope(*this, S.getSourceRange());
5157 EmitStopPoint(S: &S);
5158 emitMaster(CGF&: *this, S);
5159}
5160
5161static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
5162 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5163 Action.Enter(CGF);
5164 CGF.EmitStmt(S: S.getRawStmt());
5165 };
5166 Expr *Filter = nullptr;
5167 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
5168 Filter = FilterClause->getThreadID();
5169 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: CodeGen, Loc: S.getBeginLoc(),
5170 Filter);
5171}
5172
5173void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {
5174 if (CGM.getLangOpts().OpenMPIRBuilder) {
5175 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5176 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5177
5178 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
5179 const Expr *Filter = nullptr;
5180 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
5181 Filter = FilterClause->getThreadID();
5182 llvm::Value *FilterVal = Filter
5183 ? EmitScalarExpr(E: Filter, IgnoreResultAssign: CGM.Int32Ty)
5184 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
5185
5186 auto FiniCB = [this](InsertPointTy IP) {
5187 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5188 return llvm::Error::success();
5189 };
5190
5191 auto BodyGenCB = [MaskedRegionBodyStmt,
5192 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5193 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5194 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5195 CGF&: *this, RegionBodyStmt: MaskedRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "masked");
5196 return llvm::Error::success();
5197 };
5198
5199 LexicalScope Scope(*this, S.getSourceRange());
5200 EmitStopPoint(S: &S);
5201 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
5202 ValOrErr: OMPBuilder.createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal));
5203 Builder.restoreIP(IP: AfterIP);
5204
5205 return;
5206 }
5207 LexicalScope Scope(*this, S.getSourceRange());
5208 EmitStopPoint(S: &S);
5209 emitMasked(CGF&: *this, S);
5210}
5211
5212void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
5213 if (CGM.getLangOpts().OpenMPIRBuilder) {
5214 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5215 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5216
5217 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
5218 const Expr *Hint = nullptr;
5219 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
5220 Hint = HintClause->getHint();
5221
5222 // TODO: This is slightly different from what's currently being done in
5223 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
5224 // about typing is final.
5225 llvm::Value *HintInst = nullptr;
5226 if (Hint)
5227 HintInst =
5228 Builder.CreateIntCast(V: EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, isSigned: false);
5229
5230 auto FiniCB = [this](InsertPointTy IP) {
5231 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5232 return llvm::Error::success();
5233 };
5234
5235 auto BodyGenCB = [CriticalRegionBodyStmt,
5236 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5237 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5238 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5239 CGF&: *this, RegionBodyStmt: CriticalRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "critical");
5240 return llvm::Error::success();
5241 };
5242
5243 LexicalScope Scope(*this, S.getSourceRange());
5244 EmitStopPoint(S: &S);
5245 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5246 cantFail(ValOrErr: OMPBuilder.createCritical(Loc: Builder, BodyGenCB, FiniCB,
5247 CriticalName: S.getDirectiveName().getAsString(),
5248 HintInst));
5249 Builder.restoreIP(IP: AfterIP);
5250
5251 return;
5252 }
5253
5254 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5255 Action.Enter(CGF);
5256 CGF.EmitStmt(S: S.getAssociatedStmt());
5257 };
5258 const Expr *Hint = nullptr;
5259 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
5260 Hint = HintClause->getHint();
5261 LexicalScope Scope(*this, S.getSourceRange());
5262 EmitStopPoint(S: &S);
5263 CGM.getOpenMPRuntime().emitCriticalRegion(CGF&: *this,
5264 CriticalName: S.getDirectiveName().getAsString(),
5265 CriticalOpGen: CodeGen, Loc: S.getBeginLoc(), Hint);
5266}
5267
5268void CodeGenFunction::EmitOMPParallelForDirective(
5269 const OMPParallelForDirective &S) {
5270 // Emit directive as a combined directive that consists of two implicit
5271 // directives: 'parallel' with 'for' directive.
5272 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5273 Action.Enter(CGF);
5274 emitOMPCopyinClause(CGF, S);
5275 (void)emitWorksharingDirective(CGF, S, HasCancel: S.hasCancel());
5276 };
5277 {
5278 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5279 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
5280 CGCapturedStmtInfo CGSI(CR_OpenMP);
5281 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5282 OMPLoopScope LoopScope(CGF, S);
5283 return CGF.EmitScalarExpr(E: S.getNumIterations());
5284 };
5285 bool IsInscan = llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
5286 P: [](const OMPReductionClause *C) {
5287 return C->getModifier() == OMPC_REDUCTION_inscan;
5288 });
5289 if (IsInscan)
5290 emitScanBasedDirectiveDecls(CGF&: *this, S, NumIteratorsGen);
5291 auto LPCRegion =
5292 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5293 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for, CodeGen,
5294 CodeGenBoundParameters: emitEmptyBoundParameters);
5295 if (IsInscan)
5296 emitScanBasedDirectiveFinals(CGF&: *this, S, NumIteratorsGen);
5297 }
5298 // Check for outer lastprivate conditional update.
5299 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5300}
5301
5302void CodeGenFunction::EmitOMPParallelForSimdDirective(
5303 const OMPParallelForSimdDirective &S) {
5304 // Emit directive as a combined directive that consists of two implicit
5305 // directives: 'parallel' with 'for' directive.
5306 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5307 Action.Enter(CGF);
5308 emitOMPCopyinClause(CGF, S);
5309 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
5310 };
5311 {
5312 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5313 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
5314 CGCapturedStmtInfo CGSI(CR_OpenMP);
5315 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5316 OMPLoopScope LoopScope(CGF, S);
5317 return CGF.EmitScalarExpr(E: S.getNumIterations());
5318 };
5319 bool IsInscan = llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
5320 P: [](const OMPReductionClause *C) {
5321 return C->getModifier() == OMPC_REDUCTION_inscan;
5322 });
5323 if (IsInscan)
5324 emitScanBasedDirectiveDecls(CGF&: *this, S, NumIteratorsGen);
5325 auto LPCRegion =
5326 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5327 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for_simd, CodeGen,
5328 CodeGenBoundParameters: emitEmptyBoundParameters);
5329 if (IsInscan)
5330 emitScanBasedDirectiveFinals(CGF&: *this, S, NumIteratorsGen);
5331 }
5332 // Check for outer lastprivate conditional update.
5333 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5334}
5335
5336void CodeGenFunction::EmitOMPParallelMasterDirective(
5337 const OMPParallelMasterDirective &S) {
5338 // Emit directive as a combined directive that consists of two implicit
5339 // directives: 'parallel' with 'master' directive.
5340 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5341 Action.Enter(CGF);
5342 OMPPrivateScope PrivateScope(CGF);
5343 emitOMPCopyinClause(CGF, S);
5344 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
5345 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
5346 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
5347 (void)PrivateScope.Privatize();
5348 emitMaster(CGF, S);
5349 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
5350 };
5351 {
5352 auto LPCRegion =
5353 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5354 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master, CodeGen,
5355 CodeGenBoundParameters: emitEmptyBoundParameters);
5356 emitPostUpdateForReductionClause(CGF&: *this, D: S,
5357 CondGen: [](CodeGenFunction &) { return nullptr; });
5358 }
5359 // Check for outer lastprivate conditional update.
5360 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5361}
5362
5363void CodeGenFunction::EmitOMPParallelMaskedDirective(
5364 const OMPParallelMaskedDirective &S) {
5365 // Emit directive as a combined directive that consists of two implicit
5366 // directives: 'parallel' with 'masked' directive.
5367 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5368 Action.Enter(CGF);
5369 OMPPrivateScope PrivateScope(CGF);
5370 emitOMPCopyinClause(CGF, S);
5371 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
5372 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
5373 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
5374 (void)PrivateScope.Privatize();
5375 emitMasked(CGF, S);
5376 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
5377 };
5378 {
5379 auto LPCRegion =
5380 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5381 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked, CodeGen,
5382 CodeGenBoundParameters: emitEmptyBoundParameters);
5383 emitPostUpdateForReductionClause(CGF&: *this, D: S,
5384 CondGen: [](CodeGenFunction &) { return nullptr; });
5385 }
5386 // Check for outer lastprivate conditional update.
5387 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5388}
5389
5390void CodeGenFunction::EmitOMPParallelSectionsDirective(
5391 const OMPParallelSectionsDirective &S) {
5392 // Emit directive as a combined directive that consists of two implicit
5393 // directives: 'parallel' with 'sections' directive.
5394 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5395 Action.Enter(CGF);
5396 emitOMPCopyinClause(CGF, S);
5397 CGF.EmitSections(S);
5398 };
5399 {
5400 auto LPCRegion =
5401 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5402 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_sections, CodeGen,
5403 CodeGenBoundParameters: emitEmptyBoundParameters);
5404 }
5405 // Check for outer lastprivate conditional update.
5406 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5407}
5408
5409namespace {
5410/// Get the list of variables declared in the context of the untied tasks.
5411class CheckVarsEscapingUntiedTaskDeclContext final
5412 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
5413 llvm::SmallVector<const VarDecl *, 4> PrivateDecls;
5414
5415public:
5416 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
5417 ~CheckVarsEscapingUntiedTaskDeclContext() = default;
5418 void VisitDeclStmt(const DeclStmt *S) {
5419 if (!S)
5420 return;
5421 // Need to privatize only local vars, static locals can be processed as is.
5422 for (const Decl *D : S->decls()) {
5423 if (const auto *VD = dyn_cast_or_null<VarDecl>(Val: D))
5424 if (VD->hasLocalStorage())
5425 PrivateDecls.push_back(Elt: VD);
5426 }
5427 }
5428 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
5429 void VisitCapturedStmt(const CapturedStmt *) {}
5430 void VisitLambdaExpr(const LambdaExpr *) {}
5431 void VisitBlockExpr(const BlockExpr *) {}
5432 void VisitStmt(const Stmt *S) {
5433 if (!S)
5434 return;
5435 for (const Stmt *Child : S->children())
5436 if (Child)
5437 Visit(S: Child);
5438 }
5439
5440 /// Swaps list of vars with the provided one.
5441 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
5442};
5443} // anonymous namespace
5444
5445static void buildDependences(const OMPExecutableDirective &S,
5446 OMPTaskDataTy &Data) {
5447
5448 // First look for 'omp_all_memory' and add this first.
5449 bool OmpAllMemory = false;
5450 if (llvm::any_of(
5451 Range: S.getClausesOfKind<OMPDependClause>(), P: [](const OMPDependClause *C) {
5452 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
5453 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
5454 })) {
5455 OmpAllMemory = true;
5456 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
5457 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
5458 // simplify.
5459 OMPTaskDataTy::DependData &DD =
5460 Data.Dependences.emplace_back(Args: OMPC_DEPEND_outallmemory,
5461 /*IteratorExpr=*/Args: nullptr);
5462 // Add a nullptr Expr to simplify the codegen in emitDependData.
5463 DD.DepExprs.push_back(Elt: nullptr);
5464 }
5465 // Add remaining dependences skipping any 'out' or 'inout' if they are
5466 // overridden by 'omp_all_memory'.
5467 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
5468 OpenMPDependClauseKind Kind = C->getDependencyKind();
5469 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
5470 continue;
5471 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
5472 continue;
5473 OMPTaskDataTy::DependData &DD =
5474 Data.Dependences.emplace_back(Args: C->getDependencyKind(), Args: C->getModifier());
5475 DD.DepExprs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5476 }
5477}
5478
5479void CodeGenFunction::EmitOMPTaskBasedDirective(
5480 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
5481 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
5482 OMPTaskDataTy &Data) {
5483 // Emit outlined function for task construct.
5484 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CapturedRegion);
5485 auto I = CS->getCapturedDecl()->param_begin();
5486 auto PartId = std::next(x: I);
5487 auto TaskT = std::next(x: I, n: 4);
5488 // Check if the task is final
5489 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
5490 // If the condition constant folds and can be elided, try to avoid emitting
5491 // the condition and the dead arm of the if/else.
5492 const Expr *Cond = Clause->getCondition();
5493 bool CondConstant;
5494 if (ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant))
5495 Data.Final.setInt(CondConstant);
5496 else
5497 Data.Final.setPointer(EvaluateExprAsBool(E: Cond));
5498 } else {
5499 // By default the task is not final.
5500 Data.Final.setInt(/*IntVal=*/false);
5501 }
5502 // Check if the task has 'priority' clause.
5503 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
5504 const Expr *Prio = Clause->getPriority();
5505 Data.Priority.setInt(/*IntVal=*/true);
5506 Data.Priority.setPointer(EmitScalarConversion(
5507 Src: EmitScalarExpr(E: Prio), SrcTy: Prio->getType(),
5508 DstTy: getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
5509 Loc: Prio->getExprLoc()));
5510 }
5511 // The first function argument for tasks is a thread id, the second one is a
5512 // part id (0 for tied tasks, >=0 for untied task).
5513 llvm::DenseSet<const ValueDecl *> EmittedAsPrivate;
5514 // Get list of private variables.
5515 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
5516 auto IRef = C->varlist_begin();
5517 for (const Expr *IInit : C->private_copies()) {
5518 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
5519 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
5520 .second) {
5521 Data.PrivateVars.push_back(Elt: *IRef);
5522 Data.PrivateCopies.push_back(Elt: IInit);
5523 }
5524 ++IRef;
5525 }
5526 }
5527 EmittedAsPrivate.clear();
5528 // Get list of firstprivate variables.
5529 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5530 auto IRef = C->varlist_begin();
5531 auto IElemInitRef = C->inits().begin();
5532 for (const Expr *IInit : C->private_copies()) {
5533 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
5534 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
5535 .second) {
5536 Data.FirstprivateVars.push_back(Elt: *IRef);
5537 Data.FirstprivateCopies.push_back(Elt: IInit);
5538 Data.FirstprivateInits.push_back(Elt: *IElemInitRef);
5539 }
5540 ++IRef;
5541 ++IElemInitRef;
5542 }
5543 }
5544 // Get list of lastprivate variables (for taskloops).
5545 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
5546 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateSrcsOrigs;
5547 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
5548 auto IRef = C->varlist_begin();
5549 auto ID = C->destination_exprs().begin();
5550 auto IS = C->source_exprs().begin();
5551 for (const Expr *IInit : C->private_copies()) {
5552 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
5553 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
5554 .second) {
5555 Data.LastprivateVars.push_back(Elt: *IRef);
5556 Data.LastprivateCopies.push_back(Elt: IInit);
5557 }
5558 LastprivateDstsOrigs.insert(KV: std::make_pair(
5559 x: cast<DeclRefExpr>(Val: *ID)->getDecl(), y: cast<DeclRefExpr>(Val: *IRef)));
5560 LastprivateSrcsOrigs.insert(KV: std::make_pair(
5561 x: cast<DeclRefExpr>(Val: *IS)->getDecl(), y: cast<DeclRefExpr>(Val: *IRef)));
5562 ++IRef;
5563 ++ID;
5564 ++IS;
5565 }
5566 }
5567 SmallVector<const Expr *, 4> LHSs;
5568 SmallVector<const Expr *, 4> RHSs;
5569 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
5570 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5571 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5572 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
5573 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
5574 in_end: C->reduction_ops().end());
5575 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
5576 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
5577 }
5578 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
5579 CGF&: *this, Loc: S.getBeginLoc(), LHSExprs: LHSs, RHSExprs: RHSs, Data);
5580 // Build list of dependences.
5581 buildDependences(S, Data);
5582 // Get list of local vars for untied tasks.
5583 if (!Data.Tied) {
5584 CheckVarsEscapingUntiedTaskDeclContext Checker;
5585 Checker.Visit(S: S.getInnermostCapturedStmt()->getCapturedStmt());
5586 Data.PrivateLocals.append(in_start: Checker.getPrivateDecls().begin(),
5587 in_end: Checker.getPrivateDecls().end());
5588 }
5589 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5590 &LastprivateSrcsOrigs, CapturedRegion](
5591 CodeGenFunction &CGF, PrePostActionTy &Action) {
5592 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
5593 std::pair<Address, Address>>
5594 UntiedLocalVars;
5595 // Set proper addresses for generated private copies.
5596 OMPPrivateScope Scope(CGF);
5597 // Generate debug info for variables present in shared clause.
5598 if (auto *DI = CGF.getDebugInfo()) {
5599 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5600 CGF.CapturedStmtInfo->getCaptureFields();
5601 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5602 if (CaptureFields.size() && ContextValue) {
5603 unsigned CharWidth = CGF.getContext().getCharWidth();
5604 // The shared variables are packed together as members of structure.
5605 // So the address of each shared variable can be computed by adding
5606 // offset of it (within record) to the base address of record. For each
5607 // shared variable, debug intrinsic llvm.dbg.declare is generated with
5608 // appropriate expressions (DIExpression).
5609 // Ex:
5610 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
5611 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5612 // metadata !svar1,
5613 // metadata !DIExpression(DW_OP_deref))
5614 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5615 // metadata !svar2,
5616 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
5617 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5618 const VarDecl *SharedVar = It->first;
5619 RecordDecl *CaptureRecord = It->second->getParent();
5620 const ASTRecordLayout &Layout =
5621 CGF.getContext().getASTRecordLayout(D: CaptureRecord);
5622 unsigned Offset =
5623 Layout.getFieldOffset(FieldNo: It->second->getFieldIndex()) / CharWidth;
5624 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5625 (void)DI->EmitDeclareOfAutoVariable(Decl: SharedVar, AI: ContextValue,
5626 Builder&: CGF.Builder, UsePointerValue: false);
5627 // Get the call dbg.declare instruction we just created and update
5628 // its DIExpression to add offset to base address.
5629 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,
5630 unsigned Offset) {
5631 SmallVector<uint64_t, 8> Ops;
5632 // Add offset to the base address if non zero.
5633 if (Offset) {
5634 Ops.push_back(Elt: llvm::dwarf::DW_OP_plus_uconst);
5635 Ops.push_back(Elt: Offset);
5636 }
5637 Ops.push_back(Elt: llvm::dwarf::DW_OP_deref);
5638 Declare->setExpression(llvm::DIExpression::get(Context&: Ctx, Elements: Ops));
5639 };
5640 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
5641 // If we're emitting using the new debug info format into a block
5642 // without a terminator, the record will be "trailing".
5643 assert(!Last.isTerminator() && "unexpected terminator");
5644 if (auto *Marker =
5645 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5646 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5647 C: llvm::filterDbgVars(R: Marker->getDbgRecordRange()))) {
5648 UpdateExpr(Last.getContext(), &DVR, Offset);
5649 break;
5650 }
5651 }
5652 }
5653 }
5654 }
5655 llvm::SmallVector<std::pair<const ValueDecl *, Address>, 16>
5656 FirstprivatePtrs;
5657 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
5658 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
5659 enum { PrivatesParam = 2, CopyFnParam = 3 };
5660 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5661 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: CopyFnParam)));
5662 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(
5663 VD: CS->getCapturedDecl()->getParam(i: PrivatesParam)));
5664 // Map privates.
5665 llvm::SmallVector<std::pair<const ValueDecl *, Address>, 16> PrivatePtrs;
5666 llvm::SmallVector<llvm::Value *, 16> CallArgs;
5667 llvm::SmallVector<llvm::Type *, 4> ParamTypes;
5668 CallArgs.push_back(Elt: PrivatesPtr);
5669 ParamTypes.push_back(Elt: PrivatesPtr->getType());
5670 for (const Expr *E : Data.PrivateVars) {
5671 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
5672 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5673 T: CGF.getContext().getPointerType(T: E->getType()), Name: ".priv.ptr.addr");
5674 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5675 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5676 ParamTypes.push_back(Elt: PrivatePtr.getType());
5677 }
5678 for (const Expr *E : Data.FirstprivateVars) {
5679 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
5680 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5681 T: CGF.getContext().getPointerType(T: E->getType()),
5682 Name: ".firstpriv.ptr.addr");
5683 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5684 FirstprivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5685 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5686 ParamTypes.push_back(Elt: PrivatePtr.getType());
5687 }
5688 for (const Expr *E : Data.LastprivateVars) {
5689 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
5690 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5691 T: CGF.getContext().getPointerType(T: E->getType()),
5692 Name: ".lastpriv.ptr.addr");
5693 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5694 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5695 ParamTypes.push_back(Elt: PrivatePtr.getType());
5696 }
5697 for (const VarDecl *VD : Data.PrivateLocals) {
5698 QualType Ty = VD->getType().getNonReferenceType();
5699 if (VD->getType()->isLValueReferenceType())
5700 Ty = CGF.getContext().getPointerType(T: Ty);
5701 if (isAllocatableDecl(VD))
5702 Ty = CGF.getContext().getPointerType(T: Ty);
5703 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5704 T: CGF.getContext().getPointerType(T: Ty), Name: ".local.ptr.addr");
5705 auto Result = UntiedLocalVars.insert(
5706 KV: std::make_pair(x&: VD, y: std::make_pair(x&: PrivatePtr, y: Address::invalid())));
5707 // If key exists update in place.
5708 if (Result.second == false)
5709 *Result.first = std::make_pair(
5710 x&: VD, y: std::make_pair(x&: PrivatePtr, y: Address::invalid()));
5711 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5712 ParamTypes.push_back(Elt: PrivatePtr.getType());
5713 }
5714 auto *CopyFnTy = llvm::FunctionType::get(Result: CGF.Builder.getVoidTy(),
5715 Params: ParamTypes, /*isVarArg=*/false);
5716 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5717 CGF, Loc: S.getBeginLoc(), OutlinedFn: {CopyFnTy, CopyFn}, Args: CallArgs);
5718 for (const auto &Pair : LastprivateDstsOrigs) {
5719 const auto *OrigDecl = Pair.second->getDecl();
5720 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
5721 // For BindingDecls, emit the binding's LValue directly.
5722 Address OrigAddr =
5723 CGF.EmitOMPBindingOriginalAddr(BD, Loc: Pair.second->getExprLoc());
5724 Scope.addPrivate(LocalVD: Pair.first, Addr: OrigAddr);
5725 } else {
5726 const auto *OrigVD = cast<VarDecl>(Val: OrigDecl);
5727 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
5728 /*RefersToEnclosingVariableOrCapture=*/
5729 CGF.CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
5730 Pair.second->getType(), VK_LValue,
5731 Pair.second->getExprLoc());
5732 Scope.addPrivate(LocalVD: Pair.first, Addr: CGF.EmitLValue(E: &DRE).getAddress());
5733 }
5734 }
5735 for (const auto &Pair : PrivatePtrs) {
5736 Address Replacement = Address(
5737 CGF.Builder.CreateLoad(Addr: Pair.second),
5738 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5739 CGF.getContext().getDeclAlign(D: Pair.first));
5740 Scope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5741
5742 // For BindingDecls with lastprivate, also map the .lastprivate.src
5743 // pseudo-variable to the same private address.
5744 if (isa<BindingDecl>(Val: Pair.first)) {
5745 for (const auto &SrcPair : LastprivateSrcsOrigs) {
5746 if (SrcPair.second->getDecl() == Pair.first) {
5747 Scope.addPrivate(LocalVD: SrcPair.first, Addr: Replacement);
5748 break;
5749 }
5750 }
5751 }
5752
5753 if (auto *DI = CGF.getDebugInfo())
5754 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5755 // Only emit debug info for VarDecls, not BindingDecls.
5756 if (const auto *VD = dyn_cast<VarDecl>(Val: Pair.first))
5757 (void)DI->EmitDeclareOfAutoVariable(
5758 Decl: VD, AI: Pair.second.getBasePointer(), Builder&: CGF.Builder,
5759 /*UsePointerValue*/ true);
5760 }
5761 // Adjust mapping for internal locals by mapping actual memory instead of
5762 // a pointer to this memory.
5763 for (auto &Pair : UntiedLocalVars) {
5764 QualType VDType = Pair.first->getType().getNonReferenceType();
5765 if (Pair.first->getType()->isLValueReferenceType())
5766 VDType = CGF.getContext().getPointerType(T: VDType);
5767 if (isAllocatableDecl(VD: Pair.first)) {
5768 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Pair.second.first);
5769 Address Replacement(
5770 Ptr,
5771 CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: VDType)),
5772 CGF.getPointerAlign());
5773 Pair.second.first = Replacement;
5774 Ptr = CGF.Builder.CreateLoad(Addr: Replacement);
5775 Replacement = Address(Ptr, CGF.ConvertTypeForMem(T: VDType),
5776 CGF.getContext().getDeclAlign(D: Pair.first));
5777 Pair.second.second = Replacement;
5778 } else {
5779 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Pair.second.first);
5780 Address Replacement(Ptr, CGF.ConvertTypeForMem(T: VDType),
5781 CGF.getContext().getDeclAlign(D: Pair.first));
5782 Pair.second.first = Replacement;
5783 }
5784 }
5785 }
5786 if (Data.Reductions) {
5787 OMPPrivateScope FirstprivateScope(CGF);
5788 for (const auto &Pair : FirstprivatePtrs) {
5789 Address Replacement(
5790 CGF.Builder.CreateLoad(Addr: Pair.second),
5791 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5792 CGF.getContext().getDeclAlign(D: Pair.first));
5793 FirstprivateScope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5794 }
5795 (void)FirstprivateScope.Privatize();
5796 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5797 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5798 Data.ReductionCopies, Data.ReductionOps);
5799 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5800 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: 9)));
5801 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5802 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5803 RedCG.emitAggregateType(CGF, N: Cnt);
5804 // FIXME: This must removed once the runtime library is fixed.
5805 // Emit required threadprivate variables for
5806 // initializer/combiner/finalizer.
5807 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5808 RCG&: RedCG, N: Cnt);
5809 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5810 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5811 Replacement = Address(
5812 CGF.EmitScalarConversion(Src: Replacement.emitRawPointer(CGF),
5813 SrcTy: CGF.getContext().VoidPtrTy,
5814 DstTy: CGF.getContext().getPointerType(
5815 T: Data.ReductionCopies[Cnt]->getType()),
5816 Loc: Data.ReductionCopies[Cnt]->getExprLoc()),
5817 CGF.ConvertTypeForMem(T: Data.ReductionCopies[Cnt]->getType()),
5818 Replacement.getAlignment());
5819 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5820 Scope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5821 }
5822 }
5823 // Privatize all private variables except for in_reduction items.
5824 (void)Scope.Privatize();
5825 SmallVector<const Expr *, 4> InRedVars;
5826 SmallVector<const Expr *, 4> InRedPrivs;
5827 SmallVector<const Expr *, 4> InRedOps;
5828 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5829 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5830 auto IPriv = C->privates().begin();
5831 auto IRed = C->reduction_ops().begin();
5832 auto ITD = C->taskgroup_descriptors().begin();
5833 for (const Expr *Ref : C->varlist()) {
5834 InRedVars.emplace_back(Args&: Ref);
5835 InRedPrivs.emplace_back(Args: *IPriv);
5836 InRedOps.emplace_back(Args: *IRed);
5837 TaskgroupDescriptors.emplace_back(Args: *ITD);
5838 std::advance(i&: IPriv, n: 1);
5839 std::advance(i&: IRed, n: 1);
5840 std::advance(i&: ITD, n: 1);
5841 }
5842 }
5843 // Privatize in_reduction items here, because taskgroup descriptors must be
5844 // privatized earlier.
5845 OMPPrivateScope InRedScope(CGF);
5846 if (!InRedVars.empty()) {
5847 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5848 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5849 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5850 RedCG.emitAggregateType(CGF, N: Cnt);
5851 // The taskgroup descriptor variable is always implicit firstprivate and
5852 // privatized already during processing of the firstprivates.
5853 // FIXME: This must removed once the runtime library is fixed.
5854 // Emit required threadprivate variables for
5855 // initializer/combiner/finalizer.
5856 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5857 RCG&: RedCG, N: Cnt);
5858 llvm::Value *ReductionsPtr;
5859 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5860 ReductionsPtr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: TRExpr),
5861 Loc: TRExpr->getExprLoc());
5862 } else {
5863 ReductionsPtr = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
5864 }
5865 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5866 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5867 Replacement = Address(
5868 CGF.EmitScalarConversion(
5869 Src: Replacement.emitRawPointer(CGF), SrcTy: CGF.getContext().VoidPtrTy,
5870 DstTy: CGF.getContext().getPointerType(T: InRedPrivs[Cnt]->getType()),
5871 Loc: InRedPrivs[Cnt]->getExprLoc()),
5872 CGF.ConvertTypeForMem(T: InRedPrivs[Cnt]->getType()),
5873 Replacement.getAlignment());
5874 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5875 InRedScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5876 }
5877 }
5878 (void)InRedScope.Privatize();
5879
5880 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,
5881 UntiedLocalVars);
5882 Action.Enter(CGF);
5883 BodyGen(CGF);
5884 };
5885 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
5886 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5887 D: S, ThreadIDVar: *I, PartIDVar: *PartId, TaskTVar: *TaskT, InnermostKind: EKind, CodeGen, Tied: Data.Tied, NumberOfParts&: Data.NumberOfParts);
5888 OMPLexicalScope Scope(*this, S, std::nullopt,
5889 !isOpenMPParallelDirective(DKind: EKind) &&
5890 !isOpenMPSimdDirective(DKind: EKind));
5891 TaskGen(*this, OutlinedFn, Data);
5892}
5893
5894static ImplicitParamDecl *
5895createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
5896 QualType Ty, CapturedDecl *CD,
5897 SourceLocation Loc) {
5898 auto *OrigVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: Ty,
5899 ParamKind: ImplicitParamKind::Other);
5900 auto *OrigRef = DeclRefExpr::Create(
5901 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: OrigVD,
5902 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: Ty, VK: VK_LValue);
5903 auto *PrivateVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: Ty,
5904 ParamKind: ImplicitParamKind::Other);
5905 auto *PrivateRef = DeclRefExpr::Create(
5906 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: PrivateVD,
5907 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: Ty, VK: VK_LValue);
5908 QualType ElemType = C.getBaseElementType(QT: Ty);
5909 auto *InitVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: ElemType,
5910 ParamKind: ImplicitParamKind::Other);
5911 auto *InitRef = DeclRefExpr::Create(
5912 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: InitVD,
5913 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: ElemType, VK: VK_LValue);
5914 PrivateVD->setInitStyle(VarDecl::CInit);
5915 PrivateVD->setInit(ImplicitCastExpr::Create(Context: C, T: ElemType, Kind: CK_LValueToRValue,
5916 Operand: InitRef, /*BasePath=*/nullptr,
5917 Cat: VK_PRValue, FPO: FPOptionsOverride()));
5918 Data.FirstprivateVars.emplace_back(Args&: OrigRef);
5919 Data.FirstprivateCopies.emplace_back(Args&: PrivateRef);
5920 Data.FirstprivateInits.emplace_back(Args&: InitRef);
5921 return OrigVD;
5922}
5923
5924void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
5925 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
5926 OMPTargetDataInfo &InputInfo) {
5927 // Emit outlined function for task construct.
5928 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_task);
5929 Address CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
5930 CanQualType SharedsTy =
5931 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
5932 auto I = CS->getCapturedDecl()->param_begin();
5933 auto PartId = std::next(x: I);
5934 auto TaskT = std::next(x: I, n: 4);
5935 OMPTaskDataTy Data;
5936 // The task is not final.
5937 Data.Final.setInt(/*IntVal=*/false);
5938 // Get list of firstprivate variables.
5939 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5940 auto IRef = C->varlist_begin();
5941 auto IElemInitRef = C->inits().begin();
5942 for (auto *IInit : C->private_copies()) {
5943 Data.FirstprivateVars.push_back(Elt: *IRef);
5944 Data.FirstprivateCopies.push_back(Elt: IInit);
5945 Data.FirstprivateInits.push_back(Elt: *IElemInitRef);
5946 ++IRef;
5947 ++IElemInitRef;
5948 }
5949 }
5950 SmallVector<const Expr *, 4> LHSs;
5951 SmallVector<const Expr *, 4> RHSs;
5952 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5953 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5954 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5955 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
5956 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
5957 in_end: C->reduction_ops().end());
5958 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
5959 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
5960 }
5961 OMPPrivateScope TargetScope(*this);
5962 VarDecl *BPVD = nullptr;
5963 VarDecl *PVD = nullptr;
5964 VarDecl *SVD = nullptr;
5965 VarDecl *MVD = nullptr;
5966 if (InputInfo.NumberOfTargetItems > 0) {
5967 auto *CD = CapturedDecl::Create(
5968 C&: getContext(), DC: getContext().getTranslationUnitDecl(), /*NumParams=*/0);
5969 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
5970 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
5971 EltTy: getContext().VoidPtrTy, ArySize: ArrSize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5972 /*IndexTypeQuals=*/0);
5973 BPVD = createImplicitFirstprivateForType(
5974 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5975 PVD = createImplicitFirstprivateForType(
5976 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5977 QualType SizesType = getContext().getConstantArrayType(
5978 EltTy: getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
5979 ArySize: ArrSize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5980 /*IndexTypeQuals=*/0);
5981 SVD = createImplicitFirstprivateForType(C&: getContext(), Data, Ty: SizesType, CD,
5982 Loc: S.getBeginLoc());
5983 TargetScope.addPrivate(LocalVD: BPVD, Addr: InputInfo.BasePointersArray);
5984 TargetScope.addPrivate(LocalVD: PVD, Addr: InputInfo.PointersArray);
5985 TargetScope.addPrivate(LocalVD: SVD, Addr: InputInfo.SizesArray);
5986 // If there is no user-defined mapper, the mapper array will be nullptr. In
5987 // this case, we don't need to privatize it.
5988 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5989 Val: InputInfo.MappersArray.emitRawPointer(CGF&: *this))) {
5990 MVD = createImplicitFirstprivateForType(
5991 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5992 TargetScope.addPrivate(LocalVD: MVD, Addr: InputInfo.MappersArray);
5993 }
5994 }
5995 (void)TargetScope.Privatize();
5996 buildDependences(S, Data);
5997 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
5998 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5999 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
6000 // Set proper addresses for generated private copies.
6001 OMPPrivateScope Scope(CGF);
6002 if (!Data.FirstprivateVars.empty()) {
6003 enum { PrivatesParam = 2, CopyFnParam = 3 };
6004 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
6005 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: CopyFnParam)));
6006 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(
6007 VD: CS->getCapturedDecl()->getParam(i: PrivatesParam)));
6008 // Map privates.
6009 llvm::SmallVector<std::pair<const ValueDecl *, Address>, 16> PrivatePtrs;
6010 llvm::SmallVector<llvm::Value *, 16> CallArgs;
6011 llvm::SmallVector<llvm::Type *, 4> ParamTypes;
6012 CallArgs.push_back(Elt: PrivatesPtr);
6013 ParamTypes.push_back(Elt: PrivatesPtr->getType());
6014 for (const Expr *E : Data.FirstprivateVars) {
6015 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
6016 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
6017 T: CGF.getContext().getPointerType(T: E->getType()),
6018 Name: ".firstpriv.ptr.addr");
6019 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
6020 CallArgs.push_back(Elt: PrivatePtr.getPointer());
6021 ParamTypes.push_back(Elt: PrivatePtr.getType());
6022 }
6023 auto *CopyFnTy = llvm::FunctionType::get(Result: CGF.Builder.getVoidTy(),
6024 Params: ParamTypes, /*isVarArg=*/false);
6025 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
6026 CGF, Loc: S.getBeginLoc(), OutlinedFn: {CopyFnTy, CopyFn}, Args: CallArgs);
6027 for (const auto &Pair : PrivatePtrs) {
6028 Address Replacement(
6029 CGF.Builder.CreateLoad(Addr: Pair.second),
6030 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
6031 CGF.getContext().getDeclAlign(D: Pair.first));
6032 Scope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
6033 }
6034 }
6035 CGF.processInReduction(S, Data, CGF, CS, Scope);
6036 if (InputInfo.NumberOfTargetItems > 0) {
6037 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
6038 Addr: CGF.GetAddrOfLocalVar(VD: BPVD), /*Index=*/0);
6039 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
6040 Addr: CGF.GetAddrOfLocalVar(VD: PVD), /*Index=*/0);
6041 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
6042 Addr: CGF.GetAddrOfLocalVar(VD: SVD), /*Index=*/0);
6043 // If MVD is nullptr, the mapper array is not privatized
6044 if (MVD)
6045 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
6046 Addr: CGF.GetAddrOfLocalVar(VD: MVD), /*Index=*/0);
6047 }
6048
6049 Action.Enter(CGF);
6050 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
6051 auto *TL = S.getSingleClause<OMPThreadLimitClause>();
6052 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
6053 needsTaskBasedThreadLimit(DKind: EKind) && TL) {
6054 // Emit __kmpc_set_thread_limit() to set the thread_limit for the task
6055 // enclosing this target region. This will indirectly set the thread_limit
6056 // for every applicable construct within target region.
6057 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
6058 CGF, ThreadLimit: TL->getThreadLimit().front(), Loc: S.getBeginLoc());
6059 }
6060 BodyGen(CGF);
6061 };
6062 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
6063 D: S, ThreadIDVar: *I, PartIDVar: *PartId, TaskTVar: *TaskT, InnermostKind: EKind, CodeGen, /*Tied=*/true,
6064 NumberOfParts&: Data.NumberOfParts);
6065 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
6066 IntegerLiteral IfCond(getContext(), TrueOrFalse,
6067 getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
6068 SourceLocation());
6069 CGM.getOpenMPRuntime().emitTaskCall(CGF&: *this, Loc: S.getBeginLoc(), D: S, TaskFunction: OutlinedFn,
6070 SharedsTy, Shareds: CapturedStruct, IfCond: &IfCond, Data);
6071}
6072
6073void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
6074 OMPTaskDataTy &Data,
6075 CodeGenFunction &CGF,
6076 const CapturedStmt *CS,
6077 OMPPrivateScope &Scope) {
6078 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
6079 if (Data.Reductions) {
6080 OpenMPDirectiveKind CapturedRegion = EKind;
6081 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
6082 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
6083 Data.ReductionCopies, Data.ReductionOps);
6084 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
6085 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: 4)));
6086 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
6087 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
6088 RedCG.emitAggregateType(CGF, N: Cnt);
6089 // FIXME: This must removed once the runtime library is fixed.
6090 // Emit required threadprivate variables for
6091 // initializer/combiner/finalizer.
6092 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
6093 RCG&: RedCG, N: Cnt);
6094 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
6095 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
6096 Replacement = Address(
6097 CGF.EmitScalarConversion(Src: Replacement.emitRawPointer(CGF),
6098 SrcTy: CGF.getContext().VoidPtrTy,
6099 DstTy: CGF.getContext().getPointerType(
6100 T: Data.ReductionCopies[Cnt]->getType()),
6101 Loc: Data.ReductionCopies[Cnt]->getExprLoc()),
6102 CGF.ConvertTypeForMem(T: Data.ReductionCopies[Cnt]->getType()),
6103 Replacement.getAlignment());
6104 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
6105 Scope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
6106 }
6107 }
6108 (void)Scope.Privatize();
6109 SmallVector<const Expr *, 4> InRedVars;
6110 SmallVector<const Expr *, 4> InRedPrivs;
6111 SmallVector<const Expr *, 4> InRedOps;
6112 SmallVector<const Expr *, 4> TaskgroupDescriptors;
6113 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
6114 auto IPriv = C->privates().begin();
6115 auto IRed = C->reduction_ops().begin();
6116 auto ITD = C->taskgroup_descriptors().begin();
6117 for (const Expr *Ref : C->varlist()) {
6118 InRedVars.emplace_back(Args&: Ref);
6119 InRedPrivs.emplace_back(Args: *IPriv);
6120 InRedOps.emplace_back(Args: *IRed);
6121 TaskgroupDescriptors.emplace_back(Args: *ITD);
6122 std::advance(i&: IPriv, n: 1);
6123 std::advance(i&: IRed, n: 1);
6124 std::advance(i&: ITD, n: 1);
6125 }
6126 }
6127 OMPPrivateScope InRedScope(CGF);
6128 if (!InRedVars.empty()) {
6129 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
6130 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
6131 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
6132 RedCG.emitAggregateType(CGF, N: Cnt);
6133 // FIXME: This must removed once the runtime library is fixed.
6134 // Emit required threadprivate variables for
6135 // initializer/combiner/finalizer.
6136 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
6137 RCG&: RedCG, N: Cnt);
6138 llvm::Value *ReductionsPtr;
6139 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
6140 ReductionsPtr =
6141 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: TRExpr), Loc: TRExpr->getExprLoc());
6142 } else {
6143 ReductionsPtr = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6144 }
6145 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
6146 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
6147 Replacement = Address(
6148 CGF.EmitScalarConversion(
6149 Src: Replacement.emitRawPointer(CGF), SrcTy: CGF.getContext().VoidPtrTy,
6150 DstTy: CGF.getContext().getPointerType(T: InRedPrivs[Cnt]->getType()),
6151 Loc: InRedPrivs[Cnt]->getExprLoc()),
6152 CGF.ConvertTypeForMem(T: InRedPrivs[Cnt]->getType()),
6153 Replacement.getAlignment());
6154 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
6155 InRedScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
6156 }
6157 }
6158 (void)InRedScope.Privatize();
6159}
6160
6161void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
6162 // Emit outlined function for task construct.
6163 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_task);
6164 Address CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
6165 CanQualType SharedsTy =
6166 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
6167 const Expr *IfCond = nullptr;
6168 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
6169 if (C->getNameModifier() == OMPD_unknown ||
6170 C->getNameModifier() == OMPD_task) {
6171 IfCond = C->getCondition();
6172 break;
6173 }
6174 }
6175
6176 OMPTaskDataTy Data;
6177 // Check if we should emit tied or untied task.
6178 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
6179 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
6180 CGF.EmitStmt(S: CS->getCapturedStmt());
6181 };
6182 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
6183 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
6184 const OMPTaskDataTy &Data) {
6185 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, Loc: S.getBeginLoc(), D: S, TaskFunction: OutlinedFn,
6186 SharedsTy, Shareds: CapturedStruct, IfCond,
6187 Data);
6188 };
6189 auto LPCRegion =
6190 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
6191 EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_task, BodyGen, TaskGen, Data);
6192}
6193
6194void CodeGenFunction::EmitOMPTaskyieldDirective(
6195 const OMPTaskyieldDirective &S) {
6196 CGM.getOpenMPRuntime().emitTaskyieldCall(CGF&: *this, Loc: S.getBeginLoc());
6197}
6198
6199void CodeGenFunction::EmitOMPErrorDirective(const OMPErrorDirective &S) {
6200 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();
6201 Expr *ME = MC ? MC->getMessageString() : nullptr;
6202 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();
6203 bool IsFatal = false;
6204 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
6205 IsFatal = true;
6206 CGM.getOpenMPRuntime().emitErrorCall(CGF&: *this, Loc: S.getBeginLoc(), ME, IsFatal);
6207}
6208
6209void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
6210 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_barrier);
6211}
6212
6213void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
6214 OMPTaskDataTy Data;
6215 // Build list of dependences
6216 buildDependences(S, Data);
6217 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
6218 CGM.getOpenMPRuntime().emitTaskwaitCall(CGF&: *this, Loc: S.getBeginLoc(), Data);
6219}
6220
6221static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T) {
6222 return T.clauses().empty();
6223}
6224
6225void CodeGenFunction::EmitOMPTaskgroupDirective(
6226 const OMPTaskgroupDirective &S) {
6227 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6228 if (CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(T: S)) {
6229 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6230 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6231 InsertPointTy AllocaIP(AllocaInsertPt->getIterator());
6232
6233 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
6234 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
6235 Builder.restoreIP(IP: CodeGenIP);
6236 EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
6237 return llvm::Error::success();
6238 };
6239 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
6240 CGCapturedStmtRAII CapInfoRAII(*this, CapturedStmtInfo ? CapturedStmtInfo
6241 : &CapStmtInfo);
6242 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
6243 cantFail(ValOrErr: OMPBuilder.createTaskgroup(Loc: Builder, AllocaIP,
6244 /*DeallocBlocks=*/{}, BodyGenCB));
6245 Builder.restoreIP(IP: AfterIP);
6246 return;
6247 }
6248 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6249 Action.Enter(CGF);
6250 if (const Expr *E = S.getReductionRef()) {
6251 SmallVector<const Expr *, 4> LHSs;
6252 SmallVector<const Expr *, 4> RHSs;
6253 OMPTaskDataTy Data;
6254 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
6255 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
6256 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
6257 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
6258 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
6259 in_end: C->reduction_ops().end());
6260 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
6261 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
6262 }
6263 llvm::Value *ReductionDesc =
6264 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, Loc: S.getBeginLoc(),
6265 LHSExprs: LHSs, RHSExprs: RHSs, Data);
6266 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
6267 CGF.EmitVarDecl(D: *VD);
6268 CGF.EmitStoreOfScalar(Value: ReductionDesc, Addr: CGF.GetAddrOfLocalVar(VD),
6269 /*Volatile=*/false, Ty: E->getType());
6270 }
6271 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
6272 };
6273 CGM.getOpenMPRuntime().emitTaskgroupRegion(CGF&: *this, TaskgroupOpGen: CodeGen, Loc: S.getBeginLoc());
6274}
6275
6276void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
6277 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
6278 ? llvm::AtomicOrdering::NotAtomic
6279 : llvm::AtomicOrdering::AcquireRelease;
6280 CGM.getOpenMPRuntime().emitFlush(
6281 CGF&: *this,
6282 Vars: [&S]() -> ArrayRef<const Expr *> {
6283 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
6284 return llvm::ArrayRef(FlushClause->varlist_begin(),
6285 FlushClause->varlist_end());
6286 return {};
6287 }(),
6288 Loc: S.getBeginLoc(), AO);
6289}
6290
6291void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {
6292 const auto *DO = S.getSingleClause<OMPDepobjClause>();
6293 LValue DOLVal = EmitLValue(E: DO->getDepobj());
6294 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
6295 // Build list and emit dependences
6296 OMPTaskDataTy Data;
6297 buildDependences(S, Data);
6298 for (auto &Dep : Data.Dependences) {
6299 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
6300 CGF&: *this, Dependencies: Dep, Loc: DC->getBeginLoc());
6301 EmitStoreOfScalar(value: DepAddr.emitRawPointer(CGF&: *this), lvalue: DOLVal);
6302 }
6303 return;
6304 }
6305 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
6306 CGM.getOpenMPRuntime().emitDestroyClause(CGF&: *this, DepobjLVal: DOLVal, Loc: DC->getBeginLoc());
6307 return;
6308 }
6309 if (const auto *UC = S.getSingleClause<OMPUpdateDependObjectsClause>()) {
6310 CGM.getOpenMPRuntime().emitUpdateDependObjectsClause(
6311 CGF&: *this, DepobjLVal: DOLVal, NewDepKind: UC->getDependencyKind(), Loc: UC->getBeginLoc());
6312 return;
6313 }
6314}
6315
6316void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) {
6317 if (!OMPParentLoopDirectiveForScan)
6318 return;
6319 const OMPExecutableDirective &ParentDir = *OMPParentLoopDirectiveForScan;
6320 bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();
6321 SmallVector<const Expr *, 4> Shareds;
6322 SmallVector<const Expr *, 4> Privates;
6323 SmallVector<const Expr *, 4> LHSs;
6324 SmallVector<const Expr *, 4> RHSs;
6325 SmallVector<const Expr *, 4> ReductionOps;
6326 SmallVector<const Expr *, 4> CopyOps;
6327 SmallVector<const Expr *, 4> CopyArrayTemps;
6328 SmallVector<const Expr *, 4> CopyArrayElems;
6329 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
6330 if (C->getModifier() != OMPC_REDUCTION_inscan)
6331 continue;
6332 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
6333 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
6334 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
6335 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
6336 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
6337 CopyOps.append(in_start: C->copy_ops().begin(), in_end: C->copy_ops().end());
6338 CopyArrayTemps.append(in_start: C->copy_array_temps().begin(),
6339 in_end: C->copy_array_temps().end());
6340 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
6341 in_end: C->copy_array_elems().end());
6342 }
6343 if (ParentDir.getDirectiveKind() == OMPD_simd ||
6344 (getLangOpts().OpenMPSimd &&
6345 isOpenMPSimdDirective(DKind: ParentDir.getDirectiveKind()))) {
6346 // For simd directive and simd-based directives in simd only mode, use the
6347 // following codegen:
6348 // int x = 0;
6349 // #pragma omp simd reduction(inscan, +: x)
6350 // for (..) {
6351 // <first part>
6352 // #pragma omp scan inclusive(x)
6353 // <second part>
6354 // }
6355 // is transformed to:
6356 // int x = 0;
6357 // for (..) {
6358 // int x_priv = 0;
6359 // <first part>
6360 // x = x_priv + x;
6361 // x_priv = x;
6362 // <second part>
6363 // }
6364 // and
6365 // int x = 0;
6366 // #pragma omp simd reduction(inscan, +: x)
6367 // for (..) {
6368 // <first part>
6369 // #pragma omp scan exclusive(x)
6370 // <second part>
6371 // }
6372 // to
6373 // int x = 0;
6374 // for (..) {
6375 // int x_priv = 0;
6376 // <second part>
6377 // int temp = x;
6378 // x = x_priv + x;
6379 // x_priv = temp;
6380 // <first part>
6381 // }
6382 llvm::BasicBlock *OMPScanReduce = createBasicBlock(name: "omp.inscan.reduce");
6383 EmitBranch(Block: IsInclusive
6384 ? OMPScanReduce
6385 : BreakContinueStack.back().ContinueBlock.getBlock());
6386 EmitBlock(BB: OMPScanDispatch);
6387 {
6388 // New scope for correct construction/destruction of temp variables for
6389 // exclusive scan.
6390 LexicalScope Scope(*this, S.getSourceRange());
6391 EmitBranch(Block: IsInclusive ? OMPBeforeScanBlock : OMPAfterScanBlock);
6392 EmitBlock(BB: OMPScanReduce);
6393 if (!IsInclusive) {
6394 // Create temp var and copy LHS value to this temp value.
6395 // TMP = LHS;
6396 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6397 const Expr *PrivateExpr = Privates[I];
6398 const Expr *TempExpr = CopyArrayTemps[I];
6399 EmitAutoVarDecl(
6400 D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: TempExpr)->getDecl()));
6401 LValue DestLVal = EmitLValue(E: TempExpr);
6402 LValue SrcLVal = EmitLValue(E: LHSs[I]);
6403 EmitOMPCopy(OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(),
6404 SrcAddr: SrcLVal.getAddress(),
6405 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6406 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()),
6407 Copy: CopyOps[I]);
6408 }
6409 }
6410 CGM.getOpenMPRuntime().emitReduction(
6411 CGF&: *this, Loc: ParentDir.getEndLoc(), Privates, LHSExprs: LHSs, RHSExprs: RHSs, ReductionOps,
6412 Options: {/*WithNowait=*/true, /*SimpleReduction=*/true,
6413 /*IsPrivateVarReduction*/ {}, .ReductionKind: OMPD_simd});
6414 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6415 const Expr *PrivateExpr = Privates[I];
6416 LValue DestLVal;
6417 LValue SrcLVal;
6418 if (IsInclusive) {
6419 DestLVal = EmitLValue(E: RHSs[I]);
6420 SrcLVal = EmitLValue(E: LHSs[I]);
6421 } else {
6422 const Expr *TempExpr = CopyArrayTemps[I];
6423 DestLVal = EmitLValue(E: RHSs[I]);
6424 SrcLVal = EmitLValue(E: TempExpr);
6425 }
6426 EmitOMPCopy(
6427 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6428 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6429 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6430 }
6431 }
6432 EmitBranch(Block: IsInclusive ? OMPAfterScanBlock : OMPBeforeScanBlock);
6433 OMPScanExitBlock = IsInclusive
6434 ? BreakContinueStack.back().ContinueBlock.getBlock()
6435 : OMPScanReduce;
6436 EmitBlock(BB: OMPAfterScanBlock);
6437 return;
6438 }
6439 if (!IsInclusive) {
6440 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6441 EmitBlock(BB: OMPScanExitBlock);
6442 }
6443 if (OMPFirstScanLoop) {
6444 // Emit buffer[i] = red; at the end of the input phase.
6445 const auto *IVExpr = cast<OMPLoopDirective>(Val: ParentDir)
6446 .getIterationVariable()
6447 ->IgnoreParenImpCasts();
6448 LValue IdxLVal = EmitLValue(E: IVExpr);
6449 llvm::Value *IdxVal = EmitLoadOfScalar(lvalue: IdxLVal, Loc: IVExpr->getExprLoc());
6450 IdxVal = Builder.CreateIntCast(V: IdxVal, DestTy: SizeTy, /*isSigned=*/false);
6451 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6452 const Expr *PrivateExpr = Privates[I];
6453 const Expr *OrigExpr = Shareds[I];
6454 const Expr *CopyArrayElem = CopyArrayElems[I];
6455 OpaqueValueMapping IdxMapping(
6456 *this,
6457 cast<OpaqueValueExpr>(
6458 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
6459 RValue::get(V: IdxVal));
6460 LValue DestLVal = EmitLValue(E: CopyArrayElem);
6461 LValue SrcLVal = EmitLValue(E: OrigExpr);
6462 EmitOMPCopy(
6463 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6464 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6465 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6466 }
6467 }
6468 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6469 if (IsInclusive) {
6470 EmitBlock(BB: OMPScanExitBlock);
6471 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6472 }
6473 EmitBlock(BB: OMPScanDispatch);
6474 if (!OMPFirstScanLoop) {
6475 // Emit red = buffer[i]; at the entrance to the scan phase.
6476 const auto *IVExpr = cast<OMPLoopDirective>(Val: ParentDir)
6477 .getIterationVariable()
6478 ->IgnoreParenImpCasts();
6479 LValue IdxLVal = EmitLValue(E: IVExpr);
6480 llvm::Value *IdxVal = EmitLoadOfScalar(lvalue: IdxLVal, Loc: IVExpr->getExprLoc());
6481 IdxVal = Builder.CreateIntCast(V: IdxVal, DestTy: SizeTy, /*isSigned=*/false);
6482 llvm::BasicBlock *ExclusiveExitBB = nullptr;
6483 if (!IsInclusive) {
6484 llvm::BasicBlock *ContBB = createBasicBlock(name: "omp.exclusive.dec");
6485 ExclusiveExitBB = createBasicBlock(name: "omp.exclusive.copy.exit");
6486 llvm::Value *Cmp = Builder.CreateIsNull(Arg: IdxVal);
6487 Builder.CreateCondBr(Cond: Cmp, True: ExclusiveExitBB, False: ContBB);
6488 EmitBlock(BB: ContBB);
6489 // Use idx - 1 iteration for exclusive scan.
6490 IdxVal = Builder.CreateNUWSub(LHS: IdxVal, RHS: llvm::ConstantInt::get(Ty: SizeTy, V: 1));
6491 }
6492 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6493 const Expr *PrivateExpr = Privates[I];
6494 const Expr *OrigExpr = Shareds[I];
6495 const Expr *CopyArrayElem = CopyArrayElems[I];
6496 OpaqueValueMapping IdxMapping(
6497 *this,
6498 cast<OpaqueValueExpr>(
6499 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
6500 RValue::get(V: IdxVal));
6501 LValue SrcLVal = EmitLValue(E: CopyArrayElem);
6502 LValue DestLVal = EmitLValue(E: OrigExpr);
6503 EmitOMPCopy(
6504 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6505 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6506 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6507 }
6508 if (!IsInclusive) {
6509 EmitBlock(BB: ExclusiveExitBB);
6510 }
6511 }
6512 EmitBranch(Block: (OMPFirstScanLoop == IsInclusive) ? OMPBeforeScanBlock
6513 : OMPAfterScanBlock);
6514 EmitBlock(BB: OMPAfterScanBlock);
6515}
6516
6517void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
6518 const CodeGenLoopTy &CodeGenLoop,
6519 Expr *IncExpr) {
6520 // Emit the loop iteration variable.
6521 const auto *IVExpr = cast<DeclRefExpr>(Val: S.getIterationVariable());
6522 const auto *IVDecl = cast<VarDecl>(Val: IVExpr->getDecl());
6523 EmitVarDecl(D: *IVDecl);
6524
6525 // Emit the iterations count variable.
6526 // If it is not a variable, Sema decided to calculate iterations count on each
6527 // iteration (e.g., it is foldable into a constant).
6528 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
6529 EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
6530 // Emit calculation of the iterations count.
6531 EmitIgnoredExpr(E: S.getCalcLastIteration());
6532 }
6533
6534 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
6535
6536 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
6537 bool HasLastprivateClause = false;
6538 // Check pre-condition.
6539 {
6540 OMPLoopScope PreInitScope(*this, S);
6541 // Skip the entire loop if we don't meet the precondition.
6542 // If the condition constant folds and can be elided, avoid emitting the
6543 // whole loop.
6544 bool CondConstant;
6545 llvm::BasicBlock *ContBlock = nullptr;
6546 if (ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
6547 if (!CondConstant)
6548 return;
6549 } else {
6550 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "omp.precond.then");
6551 ContBlock = createBasicBlock(name: "omp.precond.end");
6552 emitPreCond(CGF&: *this, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
6553 TrueCount: getProfileCount(S: &S));
6554 EmitBlock(BB: ThenBlock);
6555 incrementProfileCounter(S: &S);
6556 }
6557
6558 emitAlignedClause(CGF&: *this, D: S);
6559 // Emit 'then' code.
6560 {
6561 // Emit helper vars inits.
6562
6563 LValue LB = EmitOMPHelperVar(
6564 CGF&: *this, Helper: cast<DeclRefExpr>(Val: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6565 ? S.getCombinedLowerBoundVariable()
6566 : S.getLowerBoundVariable()));
6567 LValue UB = EmitOMPHelperVar(
6568 CGF&: *this, Helper: cast<DeclRefExpr>(Val: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6569 ? S.getCombinedUpperBoundVariable()
6570 : S.getUpperBoundVariable()));
6571 LValue ST =
6572 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()));
6573 LValue IL =
6574 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()));
6575
6576 OMPPrivateScope LoopScope(*this);
6577 if (EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope)) {
6578 // Emit implicit barrier to synchronize threads and avoid data races
6579 // on initialization of firstprivate variables and post-update of
6580 // lastprivate variables.
6581 CGM.getOpenMPRuntime().emitBarrierCall(
6582 CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
6583 /*ForceSimpleCall=*/true);
6584 }
6585 EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
6586 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()) &&
6587 !isOpenMPParallelDirective(DKind: S.getDirectiveKind()) &&
6588 !isOpenMPTeamsDirective(DKind: S.getDirectiveKind()))
6589 EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
6590 HasLastprivateClause = EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
6591 EmitOMPPrivateLoopCounters(S, LoopScope);
6592 (void)LoopScope.Privatize();
6593 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
6594 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF&: *this, D: S);
6595
6596 // Detect the distribute schedule kind and chunk.
6597 llvm::Value *Chunk = nullptr;
6598 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
6599 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
6600 ScheduleKind = C->getDistScheduleKind();
6601 if (const Expr *Ch = C->getChunkSize()) {
6602 Chunk = EmitScalarExpr(E: Ch);
6603 Chunk = EmitScalarConversion(Src: Chunk, SrcTy: Ch->getType(),
6604 DstTy: S.getIterationVariable()->getType(),
6605 Loc: S.getBeginLoc());
6606 }
6607 } else {
6608 // Default behaviour for dist_schedule clause.
6609 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6610 CGF&: *this, S, ScheduleKind, Chunk);
6611 }
6612 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
6613 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
6614
6615 // GPU fused schedule: omit the outer distribute loop and let the inner
6616 // worksharing loop schedule the flattened team/thread iteration space.
6617 if (canEmitGPUFusedDistSchedule(CGM, S, DKind: EKind)) {
6618 JumpDest LoopExit =
6619 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
6620 CodeGenLoop(*this, S, LoopExit);
6621 EmitBlock(BB: LoopExit.getBlock());
6622 } else {
6623 // OpenMP [2.10.8, distribute Construct, Description]
6624 // If dist_schedule is specified, kind must be static. If specified,
6625 // iterations are divided into chunks of size chunk_size, chunks are
6626 // assigned to the teams of the league in a round-robin fashion in the
6627 // order of the team number. When no chunk_size is specified, the
6628 // iteration space is divided into chunks that are approximately equal
6629 // in size, and at most one chunk is distributed to each team of the
6630 // league. The size of the chunks is unspecified in this case.
6631 bool StaticChunked =
6632 RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
6633 isOpenMPLoopBoundSharingDirective(Kind: EKind);
6634 if (RT.isStaticNonchunked(ScheduleKind,
6635 /* Chunked */ Chunk != nullptr) ||
6636 StaticChunked) {
6637 CGOpenMPRuntime::StaticRTInput StaticInit(
6638 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
6639 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6640 StaticChunked ? Chunk : nullptr);
6641 RT.emitDistributeStaticInit(CGF&: *this, Loc: S.getBeginLoc(), SchedKind: ScheduleKind,
6642 Values: StaticInit);
6643 JumpDest LoopExit =
6644 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
6645 // UB = min(UB, GlobalUB);
6646 EmitIgnoredExpr(E: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6647 ? S.getCombinedEnsureUpperBound()
6648 : S.getEnsureUpperBound());
6649 // IV = LB;
6650 EmitIgnoredExpr(E: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6651 ? S.getCombinedInit()
6652 : S.getInit());
6653
6654 const Expr *Cond = isOpenMPLoopBoundSharingDirective(Kind: EKind)
6655 ? S.getCombinedCond()
6656 : S.getCond();
6657
6658 if (StaticChunked)
6659 Cond = S.getCombinedDistCond();
6660
6661 // For static unchunked schedules generate:
6662 //
6663 // 1. For distribute alone, codegen
6664 // while (idx <= UB) {
6665 // BODY;
6666 // ++idx;
6667 // }
6668 //
6669 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
6670 // while (idx <= UB) {
6671 // <CodeGen rest of pragma>(LB, UB);
6672 // idx += ST;
6673 // }
6674 //
6675 // For static chunk one schedule generate:
6676 //
6677 // while (IV <= GlobalUB) {
6678 // <CodeGen rest of pragma>(LB, UB);
6679 // LB += ST;
6680 // UB += ST;
6681 // UB = min(UB, GlobalUB);
6682 // IV = LB;
6683 // }
6684 //
6685 emitCommonSimdLoop(
6686 CGF&: *this, S,
6687 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6688 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()))
6689 CGF.EmitOMPSimdInit(D: S);
6690 },
6691 BodyCodeGen: [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
6692 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
6693 CGF.EmitOMPInnerLoop(
6694 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: Cond, IncExpr,
6695 BodyGen: [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6696 CodeGenLoop(CGF, S, LoopExit);
6697 },
6698 PostIncGen: [&S, StaticChunked](CodeGenFunction &CGF) {
6699 if (StaticChunked) {
6700 CGF.EmitIgnoredExpr(E: S.getCombinedNextLowerBound());
6701 CGF.EmitIgnoredExpr(E: S.getCombinedNextUpperBound());
6702 CGF.EmitIgnoredExpr(E: S.getCombinedEnsureUpperBound());
6703 CGF.EmitIgnoredExpr(E: S.getCombinedInit());
6704 }
6705 });
6706 });
6707 EmitBlock(BB: LoopExit.getBlock());
6708 // Tell the runtime we are done.
6709 RT.emitForStaticFinish(CGF&: *this, Loc: S.getEndLoc(), DKind: OMPD_distribute);
6710 } else {
6711 // Emit the outer loop, which requests its work chunk [LB..UB] from
6712 // runtime and runs the inner loop to process it.
6713 const OMPLoopArguments LoopArguments = {
6714 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6715 IL.getAddress(), Chunk};
6716 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArgs: LoopArguments,
6717 CodeGenLoopContent: CodeGenLoop);
6718 }
6719 }
6720 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind())) {
6721 EmitOMPSimdFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
6722 return CGF.Builder.CreateIsNotNull(
6723 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
6724 });
6725 }
6726 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()) &&
6727 !isOpenMPParallelDirective(DKind: S.getDirectiveKind()) &&
6728 !isOpenMPTeamsDirective(DKind: S.getDirectiveKind())) {
6729 EmitOMPReductionClauseFinal(D: S, ReductionKind: OMPD_simd);
6730 // Emit post-update of the reduction variables if IsLastIter != 0.
6731 emitPostUpdateForReductionClause(
6732 CGF&: *this, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
6733 return CGF.Builder.CreateIsNotNull(
6734 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
6735 });
6736 }
6737 // Emit final copy of the lastprivate variables if IsLastIter != 0.
6738 if (HasLastprivateClause) {
6739 EmitOMPLastprivateClauseFinal(
6740 D: S, /*NoFinals=*/false,
6741 IsLastIterCond: Builder.CreateIsNotNull(Arg: EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
6742 }
6743 }
6744
6745 // We're now done with the loop, so jump to the continuation block.
6746 if (ContBlock) {
6747 EmitBranch(Block: ContBlock);
6748 EmitBlock(BB: ContBlock, IsFinished: true);
6749 }
6750 }
6751}
6752
6753// Pass OMPLoopDirective (instead of OMPDistributeDirective) to make this
6754// function available for "loop bind(teams)", which maps to "distribute".
6755static void emitOMPDistributeDirective(const OMPLoopDirective &S,
6756 CodeGenFunction &CGF,
6757 CodeGenModule &CGM) {
6758 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6759 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
6760 };
6761 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
6762 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute, CodeGen);
6763}
6764
6765void CodeGenFunction::EmitOMPDistributeDirective(
6766 const OMPDistributeDirective &S) {
6767 emitOMPDistributeDirective(S, CGF&: *this, CGM);
6768}
6769
6770static llvm::Function *
6771emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S,
6772 const OMPExecutableDirective &D) {
6773 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
6774 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
6775 CGF.CapturedStmtInfo = &CapStmtInfo;
6776 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(S: *S, D);
6777 Fn->setDoesNotRecurse();
6778 return Fn;
6779}
6780
6781template <typename T>
6782static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
6783 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6784 llvm::OpenMPIRBuilder &OMPBuilder) {
6785
6786 unsigned NumLoops = C->getNumLoops();
6787 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth(
6788 /*DestWidth=*/64, /*Signed=*/1);
6789 llvm::SmallVector<llvm::Value *> StoreValues;
6790 for (unsigned I = 0; I < NumLoops; I++) {
6791 const Expr *CounterVal = C->getLoopData(I);
6792 assert(CounterVal);
6793 llvm::Value *StoreValue = CGF.EmitScalarConversion(
6794 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
6795 Loc: CounterVal->getExprLoc());
6796 StoreValues.emplace_back(Args&: StoreValue);
6797 }
6798 OMPDoacrossKind<T> ODK;
6799 bool IsDependSource = ODK.isSource(C);
6800 CGF.Builder.restoreIP(
6801 IP: OMPBuilder.createOrderedDepend(Loc: CGF.Builder, AllocaIP, NumLoops,
6802 StoreValues, Name: ".cnt.addr", IsDependSource));
6803}
6804
6805void CodeGenFunction::EmitOMPOrderedStandaloneDirective(
6806 const OMPOrderedStandaloneDirective &S) {
6807 assert((S.hasClausesOfKind<OMPDependClause>() ||
6808 S.hasClausesOfKind<OMPDoacrossClause>()) &&
6809 "Standalone ordered directive should have either depend or doacross "
6810 "clause");
6811 // The ordered-standalone directive.
6812 assert(!S.hasAssociatedStmt() && "No associated statement must be in "
6813 "ordered depend|doacross construct.");
6814
6815 if (CGM.getLangOpts().OpenMPIRBuilder) {
6816 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6817 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6818
6819 InsertPointTy AllocaIP(AllocaInsertPt->getIterator());
6820 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6821 emitRestoreIP(CGF&: *this, C: DC, AllocaIP, OMPBuilder);
6822 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6823 emitRestoreIP(CGF&: *this, C: DC, AllocaIP, OMPBuilder);
6824 return;
6825 }
6826
6827 if (S.hasClausesOfKind<OMPDependClause>()) {
6828 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6829 CGM.getOpenMPRuntime().emitDoacrossOrdered(CGF&: *this, C: DC);
6830 } else if (S.hasClausesOfKind<OMPDoacrossClause>()) {
6831 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6832 CGM.getOpenMPRuntime().emitDoacrossOrdered(CGF&: *this, C: DC);
6833 }
6834}
6835
6836void CodeGenFunction::EmitOMPOrderedBlockAssocDirective(
6837 const OMPOrderedBlockAssocDirective &S) {
6838 if (CGM.getLangOpts().OpenMPIRBuilder) {
6839 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6840 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6841
6842 // The ordered directive with threads or simd clause, or without clause.
6843 // Without clause, it behaves as if the threads clause is specified.
6844 const auto *C = S.getSingleClause<OMPSIMDClause>();
6845
6846 auto FiniCB = [this](InsertPointTy IP) {
6847 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
6848 return llvm::Error::success();
6849 };
6850
6851 auto BodyGenCB = [&S, C, this](InsertPointTy AllocIP,
6852 InsertPointTy CodeGenIP,
6853 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
6854 Builder.restoreIP(IP: CodeGenIP);
6855
6856 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6857 if (C) {
6858 llvm::BasicBlock *CodeGenBB = CodeGenIP.getNodeParent();
6859 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6860 Builder, /*CreateBranch=*/false, Suffix: ".ordered.after");
6861 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
6862 GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
6863 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, S: CS, D: S);
6864 assert(S.getBeginLoc().isValid() &&
6865 "Outlined function call location must be valid.");
6866 ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: S.getBeginLoc());
6867 OMPBuilderCBHelpers::EmitCaptureStmt(CGF&: *this, CodeGenIPBB: CodeGenBB, FiniBB&: *FiniBB,
6868 Fn: OutlinedFn, Args: CapturedVars);
6869 } else {
6870 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
6871 CGF&: *this, RegionBodyStmt: CS->getCapturedStmt(), AllocaIP: AllocIP, CodeGenIP, RegionName: "ordered");
6872 }
6873 return llvm::Error::success();
6874 };
6875
6876 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6877 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6878 ValOrErr: OMPBuilder.createOrderedThreadsSimd(Loc: Builder, BodyGenCB, FiniCB, IsThreads: !C));
6879 Builder.restoreIP(IP: AfterIP);
6880 return;
6881 }
6882
6883 const auto *C = S.getSingleClause<OMPSIMDClause>();
6884 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6885 PrePostActionTy &Action) {
6886 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6887 if (C) {
6888 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
6889 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
6890 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, S: CS, D: S);
6891 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc: S.getBeginLoc(),
6892 OutlinedFn, Args: CapturedVars);
6893 } else {
6894 Action.Enter(CGF);
6895 CGF.EmitStmt(S: CS->getCapturedStmt());
6896 }
6897 };
6898 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6899 CGM.getOpenMPRuntime().emitOrderedRegion(CGF&: *this, OrderedOpGen: CodeGen, Loc: S.getBeginLoc(), IsThreads: !C);
6900}
6901
6902static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
6903 QualType SrcType, QualType DestType,
6904 SourceLocation Loc) {
6905 assert(CGF.hasScalarEvaluationKind(DestType) &&
6906 "DestType must have scalar evaluation kind.");
6907 assert(!Val.isAggregate() && "Must be a scalar or complex.");
6908 return Val.isScalar() ? CGF.EmitScalarConversion(Src: Val.getScalarVal(), SrcTy: SrcType,
6909 DstTy: DestType, Loc)
6910 : CGF.EmitComplexToScalarConversion(
6911 Src: Val.getComplexVal(), SrcTy: SrcType, DstTy: DestType, Loc);
6912}
6913
6914static CodeGenFunction::ComplexPairTy
6915convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
6916 QualType DestType, SourceLocation Loc) {
6917 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
6918 "DestType must have complex evaluation kind.");
6919 CodeGenFunction::ComplexPairTy ComplexVal;
6920 if (Val.isScalar()) {
6921 // Convert the input element to the element type of the complex.
6922 QualType DestElementType =
6923 DestType->castAs<ComplexType>()->getElementType();
6924 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
6925 Src: Val.getScalarVal(), SrcTy: SrcType, DstTy: DestElementType, Loc);
6926 ComplexVal = CodeGenFunction::ComplexPairTy(
6927 ScalarVal, llvm::Constant::getNullValue(Ty: ScalarVal->getType()));
6928 } else {
6929 assert(Val.isComplex() && "Must be a scalar or complex.");
6930 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
6931 QualType DestElementType =
6932 DestType->castAs<ComplexType>()->getElementType();
6933 ComplexVal.first = CGF.EmitScalarConversion(
6934 Src: Val.getComplexVal().first, SrcTy: SrcElementType, DstTy: DestElementType, Loc);
6935 ComplexVal.second = CGF.EmitScalarConversion(
6936 Src: Val.getComplexVal().second, SrcTy: SrcElementType, DstTy: DestElementType, Loc);
6937 }
6938 return ComplexVal;
6939}
6940
6941static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6942 LValue LVal, RValue RVal) {
6943 if (LVal.isGlobalReg())
6944 CGF.EmitStoreThroughGlobalRegLValue(Src: RVal, Dst: LVal);
6945 else
6946 CGF.EmitAtomicStore(rvalue: RVal, lvalue: LVal, AO, IsVolatile: LVal.isVolatile(), /*isInit=*/false);
6947}
6948
6949static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
6950 llvm::AtomicOrdering AO, LValue LVal,
6951 SourceLocation Loc) {
6952 if (LVal.isGlobalReg())
6953 return CGF.EmitLoadOfLValue(V: LVal, Loc);
6954 return CGF.EmitAtomicLoad(
6955 lvalue: LVal, loc: Loc, AO: llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO),
6956 IsVolatile: LVal.isVolatile());
6957}
6958
6959void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
6960 QualType RValTy, SourceLocation Loc) {
6961 switch (getEvaluationKind(T: LVal.getType())) {
6962 case TEK_Scalar:
6963 EmitStoreThroughLValue(Src: RValue::get(V: convertToScalarValue(
6964 CGF&: *this, Val: RVal, SrcType: RValTy, DestType: LVal.getType(), Loc)),
6965 Dst: LVal);
6966 break;
6967 case TEK_Complex:
6968 EmitStoreOfComplex(
6969 V: convertToComplexValue(CGF&: *this, Val: RVal, SrcType: RValTy, DestType: LVal.getType(), Loc), dest: LVal,
6970 /*isInit=*/false);
6971 break;
6972 case TEK_Aggregate:
6973 llvm_unreachable("Must be a scalar or complex.");
6974 }
6975}
6976
6977static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6978 const Expr *X, const Expr *V,
6979 SourceLocation Loc) {
6980 // v = x;
6981 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6982 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6983 LValue XLValue = CGF.EmitLValue(E: X);
6984 LValue VLValue = CGF.EmitLValue(E: V);
6985 RValue Res = emitSimpleAtomicLoad(CGF, AO, LVal: XLValue, Loc);
6986 // OpenMP, 2.17.7, atomic Construct
6987 // If the read or capture clause is specified and the acquire, acq_rel, or
6988 // seq_cst clause is specified then the strong flush on exit from the atomic
6989 // operation is also an acquire flush.
6990 switch (AO) {
6991 case llvm::AtomicOrdering::Acquire:
6992 case llvm::AtomicOrdering::AcquireRelease:
6993 case llvm::AtomicOrdering::SequentiallyConsistent:
6994 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
6995 AO: llvm::AtomicOrdering::Acquire);
6996 break;
6997 case llvm::AtomicOrdering::Monotonic:
6998 case llvm::AtomicOrdering::Release:
6999 break;
7000 case llvm::AtomicOrdering::NotAtomic:
7001 case llvm::AtomicOrdering::Unordered:
7002 llvm_unreachable("Unexpected ordering.");
7003 }
7004 CGF.emitOMPSimpleStore(LVal: VLValue, RVal: Res, RValTy: X->getType().getNonReferenceType(), Loc);
7005 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: V);
7006}
7007
7008static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,
7009 llvm::AtomicOrdering AO, const Expr *X,
7010 const Expr *E, SourceLocation Loc) {
7011 // x = expr;
7012 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
7013 emitSimpleAtomicStore(CGF, AO, LVal: CGF.EmitLValue(E: X), RVal: CGF.EmitAnyExpr(E));
7014 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7015 // OpenMP, 2.17.7, atomic Construct
7016 // If the write, update, or capture clause is specified and the release,
7017 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7018 // the atomic operation is also a release flush.
7019 switch (AO) {
7020 case llvm::AtomicOrdering::Release:
7021 case llvm::AtomicOrdering::AcquireRelease:
7022 case llvm::AtomicOrdering::SequentiallyConsistent:
7023 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7024 AO: llvm::AtomicOrdering::Release);
7025 break;
7026 case llvm::AtomicOrdering::Acquire:
7027 case llvm::AtomicOrdering::Monotonic:
7028 break;
7029 case llvm::AtomicOrdering::NotAtomic:
7030 case llvm::AtomicOrdering::Unordered:
7031 llvm_unreachable("Unexpected ordering.");
7032 }
7033}
7034
7035static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
7036 RValue Update,
7037 BinaryOperatorKind BO,
7038 llvm::AtomicOrdering AO,
7039 bool IsXLHSInRHSPart) {
7040 ASTContext &Context = CGF.getContext();
7041 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
7042 // expression is simple and atomic is allowed for the given type for the
7043 // target platform.
7044 if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||
7045 (!isa<llvm::ConstantInt>(Val: Update.getScalarVal()) &&
7046 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||
7047 !Context.getTargetInfo().hasBuiltinAtomic(
7048 AtomicSizeInBits: Context.getTypeSize(T: X.getType()), AlignmentInBits: Context.toBits(CharSize: X.getAlignment())))
7049 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7050
7051 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
7052 if (T->isIntegerTy())
7053 return true;
7054
7055 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
7056 return llvm::isPowerOf2_64(Value: CGF.CGM.getDataLayout().getTypeStoreSize(Ty: T));
7057
7058 return false;
7059 };
7060
7061 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
7062 !CheckAtomicSupport(X.getAddress().getElementType(), BO))
7063 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7064
7065 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();
7066 llvm::AtomicRMWInst::BinOp RMWOp;
7067 switch (BO) {
7068 case BO_Add:
7069 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
7070 break;
7071 case BO_Sub:
7072 if (!IsXLHSInRHSPart)
7073 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7074 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
7075 break;
7076 case BO_And:
7077 RMWOp = llvm::AtomicRMWInst::And;
7078 break;
7079 case BO_Or:
7080 RMWOp = llvm::AtomicRMWInst::Or;
7081 break;
7082 case BO_Xor:
7083 RMWOp = llvm::AtomicRMWInst::Xor;
7084 break;
7085 case BO_LT:
7086 if (IsInteger)
7087 RMWOp = X.getType()->hasSignedIntegerRepresentation()
7088 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
7089 : llvm::AtomicRMWInst::Max)
7090 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
7091 : llvm::AtomicRMWInst::UMax);
7092 else
7093 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
7094 : llvm::AtomicRMWInst::FMax;
7095 break;
7096 case BO_GT:
7097 if (IsInteger)
7098 RMWOp = X.getType()->hasSignedIntegerRepresentation()
7099 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
7100 : llvm::AtomicRMWInst::Min)
7101 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
7102 : llvm::AtomicRMWInst::UMin);
7103 else
7104 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
7105 : llvm::AtomicRMWInst::FMin;
7106 break;
7107 case BO_Assign:
7108 RMWOp = llvm::AtomicRMWInst::Xchg;
7109 break;
7110 case BO_Mul:
7111 case BO_Div:
7112 case BO_Rem:
7113 case BO_Shl:
7114 case BO_Shr:
7115 case BO_LAnd:
7116 case BO_LOr:
7117 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7118 case BO_PtrMemD:
7119 case BO_PtrMemI:
7120 case BO_LE:
7121 case BO_GE:
7122 case BO_EQ:
7123 case BO_NE:
7124 case BO_Cmp:
7125 case BO_AddAssign:
7126 case BO_SubAssign:
7127 case BO_AndAssign:
7128 case BO_OrAssign:
7129 case BO_XorAssign:
7130 case BO_MulAssign:
7131 case BO_DivAssign:
7132 case BO_RemAssign:
7133 case BO_ShlAssign:
7134 case BO_ShrAssign:
7135 case BO_Comma:
7136 llvm_unreachable("Unsupported atomic update operation");
7137 }
7138 llvm::Value *UpdateVal = Update.getScalarVal();
7139 if (auto *IC = dyn_cast<llvm::ConstantInt>(Val: UpdateVal)) {
7140 if (IsInteger)
7141 UpdateVal = CGF.Builder.CreateIntCast(
7142 V: IC, DestTy: X.getAddress().getElementType(),
7143 isSigned: X.getType()->hasSignedIntegerRepresentation());
7144 else
7145 UpdateVal = CGF.Builder.CreateCast(Op: llvm::Instruction::CastOps::UIToFP, V: IC,
7146 DestTy: X.getAddress().getElementType());
7147 }
7148 llvm::AtomicRMWInst *Res =
7149 CGF.emitAtomicRMWInst(Op: RMWOp, Addr: X.getAddress(), Val: UpdateVal, Order: AO);
7150 return std::make_pair(x: true, y: RValue::get(V: Res));
7151}
7152
7153std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
7154 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
7155 llvm::AtomicOrdering AO, SourceLocation Loc,
7156 const llvm::function_ref<RValue(RValue)> CommonGen) {
7157 // Update expressions are allowed to have the following forms:
7158 // x binop= expr; -> xrval + expr;
7159 // x++, ++x -> xrval + 1;
7160 // x--, --x -> xrval - 1;
7161 // x = x binop expr; -> xrval binop expr
7162 // x = expr Op x; - > expr binop xrval;
7163 auto Res = emitOMPAtomicRMW(CGF&: *this, X, Update: E, BO, AO, IsXLHSInRHSPart);
7164 if (!Res.first) {
7165 if (X.isGlobalReg()) {
7166 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
7167 // 'xrval'.
7168 EmitStoreThroughLValue(Src: CommonGen(EmitLoadOfLValue(V: X, Loc)), Dst: X);
7169 } else {
7170 // Perform compare-and-swap procedure.
7171 EmitAtomicUpdate(LVal: X, AO, UpdateOp: CommonGen, IsVolatile: X.getType().isVolatileQualified());
7172 }
7173 }
7174 return Res;
7175}
7176
7177static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,
7178 llvm::AtomicOrdering AO, const Expr *X,
7179 const Expr *E, const Expr *UE,
7180 bool IsXLHSInRHSPart, SourceLocation Loc) {
7181 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
7182 "Update expr in 'atomic update' must be a binary operator.");
7183 const auto *BOUE = cast<BinaryOperator>(Val: UE->IgnoreImpCasts());
7184 // Update expressions are allowed to have the following forms:
7185 // x binop= expr; -> xrval + expr;
7186 // x++, ++x -> xrval + 1;
7187 // x--, --x -> xrval - 1;
7188 // x = x binop expr; -> xrval binop expr
7189 // x = expr Op x; - > expr binop xrval;
7190 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
7191 LValue XLValue = CGF.EmitLValue(E: X);
7192 RValue ExprRValue = CGF.EmitAnyExpr(E);
7193 const auto *LHS = cast<OpaqueValueExpr>(Val: BOUE->getLHS()->IgnoreImpCasts());
7194 const auto *RHS = cast<OpaqueValueExpr>(Val: BOUE->getRHS()->IgnoreImpCasts());
7195 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
7196 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
7197 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
7198 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7199 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
7200 return CGF.EmitAnyExpr(E: UE);
7201 };
7202 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
7203 X: XLValue, E: ExprRValue, BO: BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, CommonGen: Gen);
7204 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7205 // OpenMP, 2.17.7, atomic Construct
7206 // If the write, update, or capture clause is specified and the release,
7207 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7208 // the atomic operation is also a release flush.
7209 switch (AO) {
7210 case llvm::AtomicOrdering::Release:
7211 case llvm::AtomicOrdering::AcquireRelease:
7212 case llvm::AtomicOrdering::SequentiallyConsistent:
7213 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7214 AO: llvm::AtomicOrdering::Release);
7215 break;
7216 case llvm::AtomicOrdering::Acquire:
7217 case llvm::AtomicOrdering::Monotonic:
7218 break;
7219 case llvm::AtomicOrdering::NotAtomic:
7220 case llvm::AtomicOrdering::Unordered:
7221 llvm_unreachable("Unexpected ordering.");
7222 }
7223}
7224
7225static RValue convertToType(CodeGenFunction &CGF, RValue Value,
7226 QualType SourceType, QualType ResType,
7227 SourceLocation Loc) {
7228 switch (CGF.getEvaluationKind(T: ResType)) {
7229 case TEK_Scalar:
7230 return RValue::get(
7231 V: convertToScalarValue(CGF, Val: Value, SrcType: SourceType, DestType: ResType, Loc));
7232 case TEK_Complex: {
7233 auto Res = convertToComplexValue(CGF, Val: Value, SrcType: SourceType, DestType: ResType, Loc);
7234 return RValue::getComplex(V1: Res.first, V2: Res.second);
7235 }
7236 case TEK_Aggregate:
7237 break;
7238 }
7239 llvm_unreachable("Must be a scalar or complex.");
7240}
7241
7242static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
7243 llvm::AtomicOrdering AO,
7244 bool IsPostfixUpdate, const Expr *V,
7245 const Expr *X, const Expr *E,
7246 const Expr *UE, bool IsXLHSInRHSPart,
7247 SourceLocation Loc) {
7248 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
7249 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
7250 RValue NewVVal;
7251 LValue VLValue = CGF.EmitLValue(E: V);
7252 LValue XLValue = CGF.EmitLValue(E: X);
7253 RValue ExprRValue = CGF.EmitAnyExpr(E);
7254 QualType NewVValType;
7255 if (UE) {
7256 // 'x' is updated with some additional value.
7257 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
7258 "Update expr in 'atomic capture' must be a binary operator.");
7259 const auto *BOUE = cast<BinaryOperator>(Val: UE->IgnoreImpCasts());
7260 // Update expressions are allowed to have the following forms:
7261 // x binop= expr; -> xrval + expr;
7262 // x++, ++x -> xrval + 1;
7263 // x--, --x -> xrval - 1;
7264 // x = x binop expr; -> xrval binop expr
7265 // x = expr Op x; - > expr binop xrval;
7266 const auto *LHS = cast<OpaqueValueExpr>(Val: BOUE->getLHS()->IgnoreImpCasts());
7267 const auto *RHS = cast<OpaqueValueExpr>(Val: BOUE->getRHS()->IgnoreImpCasts());
7268 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
7269 NewVValType = XRValExpr->getType();
7270 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
7271 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
7272 IsPostfixUpdate](RValue XRValue) {
7273 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7274 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
7275 RValue Res = CGF.EmitAnyExpr(E: UE);
7276 NewVVal = IsPostfixUpdate ? XRValue : Res;
7277 return Res;
7278 };
7279 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
7280 X: XLValue, E: ExprRValue, BO: BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, CommonGen: Gen);
7281 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7282 if (Res.first) {
7283 // 'atomicrmw' instruction was generated.
7284 if (IsPostfixUpdate) {
7285 // Use old value from 'atomicrmw'.
7286 NewVVal = Res.second;
7287 } else {
7288 // 'atomicrmw' does not provide new value, so evaluate it using old
7289 // value of 'x'.
7290 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7291 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
7292 NewVVal = CGF.EmitAnyExpr(E: UE);
7293 }
7294 }
7295 } else {
7296 // 'x' is simply rewritten with some 'expr'.
7297 NewVValType = X->getType().getNonReferenceType();
7298 ExprRValue = convertToType(CGF, Value: ExprRValue, SourceType: E->getType(),
7299 ResType: X->getType().getNonReferenceType(), Loc);
7300 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
7301 NewVVal = XRValue;
7302 return ExprRValue;
7303 };
7304 // Try to perform atomicrmw xchg, otherwise simple exchange.
7305 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
7306 X: XLValue, E: ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
7307 Loc, CommonGen: Gen);
7308 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7309 if (Res.first) {
7310 // 'atomicrmw' instruction was generated.
7311 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
7312 }
7313 }
7314 // Emit post-update store to 'v' of old/new 'x' value.
7315 CGF.emitOMPSimpleStore(LVal: VLValue, RVal: NewVVal, RValTy: NewVValType, Loc);
7316 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: V);
7317 // OpenMP 5.1 removes the required flush for capture clause.
7318 if (CGF.CGM.getLangOpts().OpenMP < 51) {
7319 // OpenMP, 2.17.7, atomic Construct
7320 // If the write, update, or capture clause is specified and the release,
7321 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7322 // the atomic operation is also a release flush.
7323 // If the read or capture clause is specified and the acquire, acq_rel, or
7324 // seq_cst clause is specified then the strong flush on exit from the atomic
7325 // operation is also an acquire flush.
7326 switch (AO) {
7327 case llvm::AtomicOrdering::Release:
7328 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7329 AO: llvm::AtomicOrdering::Release);
7330 break;
7331 case llvm::AtomicOrdering::Acquire:
7332 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7333 AO: llvm::AtomicOrdering::Acquire);
7334 break;
7335 case llvm::AtomicOrdering::AcquireRelease:
7336 case llvm::AtomicOrdering::SequentiallyConsistent:
7337 CGF.CGM.getOpenMPRuntime().emitFlush(
7338 CGF, Vars: {}, Loc, AO: llvm::AtomicOrdering::AcquireRelease);
7339 break;
7340 case llvm::AtomicOrdering::Monotonic:
7341 break;
7342 case llvm::AtomicOrdering::NotAtomic:
7343 case llvm::AtomicOrdering::Unordered:
7344 llvm_unreachable("Unexpected ordering.");
7345 }
7346 }
7347}
7348
7349static void emitOMPAtomicCompareExpr(
7350 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
7351 const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,
7352 const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,
7353 SourceLocation Loc) {
7354 llvm::OpenMPIRBuilder &OMPBuilder =
7355 CGF.CGM.getOpenMPRuntime().getOMPBuilder();
7356
7357 OMPAtomicCompareOp Op;
7358 assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");
7359 switch (cast<BinaryOperator>(Val: CE)->getOpcode()) {
7360 case BO_EQ:
7361 Op = OMPAtomicCompareOp::EQ;
7362 break;
7363 case BO_LT:
7364 Op = OMPAtomicCompareOp::MIN;
7365 break;
7366 case BO_GT:
7367 Op = OMPAtomicCompareOp::MAX;
7368 break;
7369 default:
7370 llvm_unreachable("unsupported atomic compare binary operator");
7371 }
7372
7373 LValue XLVal = CGF.EmitLValue(E: X);
7374 Address XAddr = XLVal.getAddress();
7375
7376 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
7377 if (X->getType() == E->getType())
7378 return CGF.EmitScalarExpr(E);
7379 const Expr *NewE = E->IgnoreImplicitAsWritten();
7380 llvm::Value *V = CGF.EmitScalarExpr(E: NewE);
7381 if (NewE->getType() == X->getType())
7382 return V;
7383 return CGF.EmitScalarConversion(Src: V, SrcTy: NewE->getType(), DstTy: X->getType(), Loc);
7384 };
7385
7386 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
7387 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
7388 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: EVal))
7389 EVal = CGF.Builder.CreateIntCast(
7390 V: CI, DestTy: XLVal.getAddress().getElementType(),
7391 isSigned: E->getType()->hasSignedIntegerRepresentation());
7392 if (DVal)
7393 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: DVal))
7394 DVal = CGF.Builder.CreateIntCast(
7395 V: CI, DestTy: XLVal.getAddress().getElementType(),
7396 isSigned: D->getType()->hasSignedIntegerRepresentation());
7397
7398 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
7399 .Var: XAddr.emitRawPointer(CGF), .ElemTy: XAddr.getElementType(),
7400 .IsSigned: X->getType()->hasSignedIntegerRepresentation(),
7401 .IsVolatile: X->getType().isVolatileQualified()};
7402 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
7403 if (V) {
7404 LValue LV = CGF.EmitLValue(E: V);
7405 Address Addr = LV.getAddress();
7406 VOpVal = {.Var: Addr.emitRawPointer(CGF), .ElemTy: Addr.getElementType(),
7407 .IsSigned: V->getType()->hasSignedIntegerRepresentation(),
7408 .IsVolatile: V->getType().isVolatileQualified()};
7409 }
7410 if (R) {
7411 LValue LV = CGF.EmitLValue(E: R);
7412 Address Addr = LV.getAddress();
7413 ROpVal = {.Var: Addr.emitRawPointer(CGF), .ElemTy: Addr.getElementType(),
7414 .IsSigned: R->getType()->hasSignedIntegerRepresentation(),
7415 .IsVolatile: R->getType().isVolatileQualified()};
7416 }
7417
7418 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
7419 // fail clause was not mentioned on the
7420 // "#pragma omp atomic compare" construct.
7421 CGF.Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
7422 Loc: CGF.Builder, X&: XOpVal, V&: VOpVal, R&: ROpVal, E: EVal, D: DVal, AO, Op, IsXBinopExpr,
7423 IsPostfixUpdate, IsFailOnly));
7424 } else
7425 CGF.Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
7426 Loc: CGF.Builder, X&: XOpVal, V&: VOpVal, R&: ROpVal, E: EVal, D: DVal, AO, Op, IsXBinopExpr,
7427 IsPostfixUpdate, IsFailOnly, Failure: FailAO));
7428}
7429
7430static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
7431 llvm::AtomicOrdering AO,
7432 llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,
7433 const Expr *X, const Expr *V, const Expr *R,
7434 const Expr *E, const Expr *UE, const Expr *D,
7435 const Expr *CE, bool IsXLHSInRHSPart,
7436 bool IsFailOnly, SourceLocation Loc) {
7437 switch (Kind) {
7438 case OMPC_read:
7439 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
7440 break;
7441 case OMPC_write:
7442 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
7443 break;
7444 case OMPC_unknown:
7445 case OMPC_update:
7446 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
7447 break;
7448 case OMPC_capture:
7449 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
7450 IsXLHSInRHSPart, Loc);
7451 break;
7452 case OMPC_compare: {
7453 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
7454 IsXBinopExpr: IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
7455 break;
7456 }
7457 default:
7458 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
7459 }
7460}
7461
7462void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
7463 llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7464 // Fail Memory Clause Ordering.
7465 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
7466 bool MemOrderingSpecified = false;
7467 if (S.getSingleClause<OMPSeqCstClause>()) {
7468 AO = llvm::AtomicOrdering::SequentiallyConsistent;
7469 MemOrderingSpecified = true;
7470 } else if (S.getSingleClause<OMPAcqRelClause>()) {
7471 AO = llvm::AtomicOrdering::AcquireRelease;
7472 MemOrderingSpecified = true;
7473 } else if (S.getSingleClause<OMPAcquireClause>()) {
7474 AO = llvm::AtomicOrdering::Acquire;
7475 MemOrderingSpecified = true;
7476 } else if (S.getSingleClause<OMPReleaseClause>()) {
7477 AO = llvm::AtomicOrdering::Release;
7478 MemOrderingSpecified = true;
7479 } else if (S.getSingleClause<OMPRelaxedClause>()) {
7480 AO = llvm::AtomicOrdering::Monotonic;
7481 MemOrderingSpecified = true;
7482 }
7483 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
7484 OpenMPClauseKind Kind = OMPC_unknown;
7485 for (const OMPClause *C : S.clauses()) {
7486 // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
7487 // if it is first).
7488 OpenMPClauseKind K = C->getClauseKind();
7489 // TBD
7490 if (K == OMPC_weak)
7491 return;
7492 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
7493 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
7494 continue;
7495 Kind = K;
7496 KindsEncountered.insert(V: K);
7497 }
7498 // We just need to correct Kind here. No need to set a bool saying it is
7499 // actually compare capture because we can tell from whether V and R are
7500 // nullptr.
7501 if (KindsEncountered.contains(V: OMPC_compare) &&
7502 KindsEncountered.contains(V: OMPC_capture))
7503 Kind = OMPC_compare;
7504 if (!MemOrderingSpecified) {
7505 llvm::AtomicOrdering DefaultOrder =
7506 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7507 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
7508 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
7509 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
7510 Kind == OMPC_capture)) {
7511 AO = DefaultOrder;
7512 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
7513 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
7514 AO = llvm::AtomicOrdering::Release;
7515 } else if (Kind == OMPC_read) {
7516 assert(Kind == OMPC_read && "Unexpected atomic kind.");
7517 AO = llvm::AtomicOrdering::Acquire;
7518 }
7519 }
7520 }
7521
7522 if (KindsEncountered.contains(V: OMPC_compare) &&
7523 KindsEncountered.contains(V: OMPC_fail)) {
7524 Kind = OMPC_compare;
7525 const auto *FailClause = S.getSingleClause<OMPFailClause>();
7526 if (FailClause) {
7527 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
7528 if (FailParameter == llvm::omp::OMPC_relaxed)
7529 FailAO = llvm::AtomicOrdering::Monotonic;
7530 else if (FailParameter == llvm::omp::OMPC_acquire)
7531 FailAO = llvm::AtomicOrdering::Acquire;
7532 else if (FailParameter == llvm::omp::OMPC_seq_cst)
7533 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
7534 }
7535 }
7536
7537 // The fp atomics hints override -f[no-]atomic-ignore-denormal-mode for this
7538 // construct. The safe hint takes precedence if both are given.
7539 AtomicOptions AtomicOpts = CGM.getAtomicOpts();
7540 if (const auto *HintClause = S.getSingleClause<OMPHintClause>()) {
7541 uint64_t Hint = HintClause->getHint()
7542 ->EvaluateKnownConstInt(Ctx: getContext())
7543 .getZExtValue();
7544 if (Hint & llvm::to_underlying(E: OMPSyncHint::OMPXSafeFPAtomics))
7545 AtomicOpts.ignore_denormal_mode = false;
7546 else if (Hint & llvm::to_underlying(E: OMPSyncHint::OMPXFastFPAtomics))
7547 AtomicOpts.ignore_denormal_mode = true;
7548 }
7549 CGAtomicOptionsRAII AtomicOptsRAII(CGM, AtomicOpts);
7550
7551 LexicalScope Scope(*this, S.getSourceRange());
7552 EmitStopPoint(S: S.getAssociatedStmt());
7553 emitOMPAtomicExpr(CGF&: *this, Kind, AO, FailAO, IsPostfixUpdate: S.isPostfixUpdate(), X: S.getX(),
7554 V: S.getV(), R: S.getR(), E: S.getExpr(), UE: S.getUpdateExpr(),
7555 D: S.getD(), CE: S.getCondExpr(), IsXLHSInRHSPart: S.isXLHSInRHSPart(),
7556 IsFailOnly: S.isFailOnly(), Loc: S.getBeginLoc());
7557}
7558
7559static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
7560 const OMPExecutableDirective &S,
7561 const RegionCodeGenTy &CodeGen) {
7562 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
7563 CodeGenModule &CGM = CGF.CGM;
7564
7565 // On device emit this construct as inlined code.
7566 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
7567 OMPLexicalScope Scope(CGF, S, OMPD_target);
7568 CGM.getOpenMPRuntime().emitInlinedDirective(
7569 CGF, InnermostKind: OMPD_target, CodeGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7570 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
7571 });
7572 return;
7573 }
7574
7575 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
7576 llvm::Function *Fn = nullptr;
7577 llvm::Constant *FnID = nullptr;
7578
7579 const Expr *IfCond = nullptr;
7580 // Check for the at most one if clause associated with the target region.
7581 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7582 if (C->getNameModifier() == OMPD_unknown ||
7583 C->getNameModifier() == OMPD_target) {
7584 IfCond = C->getCondition();
7585 break;
7586 }
7587 }
7588
7589 // Check if we have any device clause associated with the directive.
7590 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
7591 nullptr, OMPC_DEVICE_unknown);
7592 if (auto *C = S.getSingleClause<OMPDeviceClause>())
7593 Device.setPointerAndInt(PtrVal: C->getDevice(), IntVal: C->getModifier());
7594
7595 // Check if we have an if clause whose conditional always evaluates to false
7596 // or if we do not have any targets specified. If so the target region is not
7597 // an offload entry point.
7598 bool IsOffloadEntry = true;
7599 if (IfCond) {
7600 bool Val;
7601 if (CGF.ConstantFoldsToSimpleInteger(Cond: IfCond, Result&: Val) && !Val)
7602 IsOffloadEntry = false;
7603 }
7604 if (CGM.getLangOpts().OMPTargetTriples.empty())
7605 IsOffloadEntry = false;
7606
7607 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
7608 CGM.getDiags().Report(DiagID: diag::err_missing_mandatory_offloading);
7609 }
7610
7611 // Lambdas and blocks at namespace scope have no parent function. Regions in
7612 // constructs outlined from such a lambda are named after its call operator,
7613 // as the device names them.
7614 const Decl *CodeDecl = CGF.CurFuncDecl ? CGF.CurFuncDecl : CGF.CurCodeDecl;
7615 while (const auto *CD = dyn_cast_or_null<CapturedDecl>(Val: CodeDecl))
7616 CodeDecl = Decl::castFromDeclContext(CD->getParent());
7617 const auto *ParentFD = dyn_cast_or_null<FunctionDecl>(Val: CodeDecl);
7618 StringRef ParentName =
7619 ParentFD ? CGM.getOpenMPRuntime().getTargetRegionParentName(FD: ParentFD)
7620 : CGF.CurFn->getName();
7621
7622 // Emit target region as a standalone region.
7623 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: FnID,
7624 IsOffloadEntry, CodeGen);
7625 OMPLexicalScope Scope(CGF, S, OMPD_task);
7626 auto &&SizeEmitter =
7627 [IsOffloadEntry](CodeGenFunction &CGF,
7628 const OMPLoopDirective &D) -> llvm::Value * {
7629 if (IsOffloadEntry) {
7630 OMPLoopScope PreInitScope(CGF, D);
7631 // Emit calculation of the iterations count.
7632 llvm::Value *NumIterations = CGF.EmitScalarExpr(E: D.getNumIterations());
7633 NumIterations = CGF.Builder.CreateIntCast(V: NumIterations, DestTy: CGF.Int64Ty,
7634 /*isSigned=*/false);
7635 return NumIterations;
7636 }
7637 return nullptr;
7638 };
7639 CGM.getOpenMPRuntime().emitTargetCall(CGF, D: S, OutlinedFn: Fn, OutlinedFnID: FnID, IfCond, Device,
7640 SizeEmitter);
7641}
7642
7643static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
7644 PrePostActionTy &Action) {
7645 Action.Enter(CGF);
7646 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7647 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7648 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7649 (void)PrivateScope.Privatize();
7650 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
7651 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
7652
7653 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_target)->getCapturedStmt());
7654 CGF.EnsureInsertPoint();
7655}
7656
7657void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
7658 StringRef ParentName,
7659 const OMPTargetDirective &S) {
7660 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7661 emitTargetRegion(CGF, S, Action);
7662 };
7663 llvm::Function *Fn;
7664 llvm::Constant *Addr;
7665 // Emit target region as a standalone region.
7666 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7667 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7668 assert(Fn && Addr && "Target device function emission failed.");
7669}
7670
7671void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
7672 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7673 emitTargetRegion(CGF, S, Action);
7674 };
7675 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7676}
7677
7678static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
7679 const OMPExecutableDirective &S,
7680 OpenMPDirectiveKind InnermostKind,
7681 const RegionCodeGenTy &CodeGen) {
7682 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_teams);
7683 llvm::Function *OutlinedFn =
7684 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
7685 CGF, D: S, ThreadIDVar: *CS->getCapturedDecl()->param_begin(), InnermostKind,
7686 CodeGen);
7687
7688 OMPTeamsScope Scope(CGF, S);
7689 auto ParallelLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7690 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
7691 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7692 if (NT || TL) {
7693 const Expr *NumTeams = NT ? NT->getNumTeams().front() : nullptr;
7694 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7695
7696 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
7697 Loc: S.getBeginLoc());
7698 }
7699 };
7700
7701 const Expr *IfCond = nullptr;
7702 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7703 if (C->getNameModifier() == OMPD_unknown ||
7704 C->getNameModifier() == OMPD_teams) {
7705 IfCond = C->getCondition();
7706 break;
7707 }
7708 }
7709 if (IfCond && CGF.CGM.getLangOpts().OpenMP >= 52) {
7710 auto SerialLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7711 // OpenMP 5.2, 10.2, teams Construct
7712 // When an if clause is present on a teams construct and the if clause
7713 // expression evaluates to false, the number of created teams is one.
7714 const llvm::APInt One(32, 1);
7715 IntegerLiteral NumTeams(
7716 CGF.getContext(), One,
7717 CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
7718 SourceLocation());
7719 // The thread_limit clause is unaffected by the if clause.
7720 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7721 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7722 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams: &NumTeams, ThreadLimit,
7723 Loc: S.getBeginLoc());
7724 };
7725 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, Cond: IfCond, ThenGen: ParallelLeague,
7726 ElseGen: SerialLeague);
7727 } else {
7728 const RegionCodeGenTy ThenRCG(ParallelLeague);
7729 ThenRCG(CGF);
7730 }
7731
7732 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
7733 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
7734 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, D: S, Loc: S.getBeginLoc(), OutlinedFn,
7735 CapturedVars);
7736}
7737
7738void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
7739 // Emit teams region as a standalone region.
7740 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7741 Action.Enter(CGF);
7742 OMPPrivateScope PrivateScope(CGF);
7743 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7744 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7745 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7746 (void)PrivateScope.Privatize();
7747 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_teams)->getCapturedStmt());
7748 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7749 };
7750 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
7751 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7752 CondGen: [](CodeGenFunction &) { return nullptr; });
7753}
7754
7755static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
7756 const OMPTargetTeamsDirective &S) {
7757 auto *CS = S.getCapturedStmt(RegionKind: OMPD_teams);
7758 Action.Enter(CGF);
7759 // Emit teams region as a standalone region.
7760 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7761 Action.Enter(CGF);
7762 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7763 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7764 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7765 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7766 (void)PrivateScope.Privatize();
7767 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
7768 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
7769 CGF.EmitStmt(S: CS->getCapturedStmt());
7770 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7771 };
7772 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_teams, CodeGen);
7773 emitPostUpdateForReductionClause(CGF, D: S,
7774 CondGen: [](CodeGenFunction &) { return nullptr; });
7775}
7776
7777void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
7778 CodeGenModule &CGM, StringRef ParentName,
7779 const OMPTargetTeamsDirective &S) {
7780 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7781 emitTargetTeamsRegion(CGF, Action, S);
7782 };
7783 llvm::Function *Fn;
7784 llvm::Constant *Addr;
7785 // Emit target region as a standalone region.
7786 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7787 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7788 assert(Fn && Addr && "Target device function emission failed.");
7789}
7790
7791void CodeGenFunction::EmitOMPTargetTeamsDirective(
7792 const OMPTargetTeamsDirective &S) {
7793 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7794 emitTargetTeamsRegion(CGF, Action, S);
7795 };
7796 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7797}
7798
7799static void
7800emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
7801 const OMPTargetTeamsDistributeDirective &S) {
7802 Action.Enter(CGF);
7803 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7804 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7805 };
7806
7807 // Emit teams region as a standalone region.
7808 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7809 PrePostActionTy &Action) {
7810 Action.Enter(CGF);
7811 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7812 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7813 (void)PrivateScope.Privatize();
7814 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7815 CodeGen: CodeGenDistribute);
7816 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7817 };
7818 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute, CodeGen);
7819 emitPostUpdateForReductionClause(CGF, D: S,
7820 CondGen: [](CodeGenFunction &) { return nullptr; });
7821}
7822
7823void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
7824 CodeGenModule &CGM, StringRef ParentName,
7825 const OMPTargetTeamsDistributeDirective &S) {
7826 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7827 emitTargetTeamsDistributeRegion(CGF, Action, S);
7828 };
7829 llvm::Function *Fn;
7830 llvm::Constant *Addr;
7831 // Emit target region as a standalone region.
7832 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7833 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7834 assert(Fn && Addr && "Target device function emission failed.");
7835}
7836
7837void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
7838 const OMPTargetTeamsDistributeDirective &S) {
7839 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7840 emitTargetTeamsDistributeRegion(CGF, Action, S);
7841 };
7842 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7843}
7844
7845static void emitTargetTeamsDistributeSimdRegion(
7846 CodeGenFunction &CGF, PrePostActionTy &Action,
7847 const OMPTargetTeamsDistributeSimdDirective &S) {
7848 Action.Enter(CGF);
7849 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7850 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7851 };
7852
7853 // Emit teams region as a standalone region.
7854 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7855 PrePostActionTy &Action) {
7856 Action.Enter(CGF);
7857 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7858 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7859 (void)PrivateScope.Privatize();
7860 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7861 CodeGen: CodeGenDistribute);
7862 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7863 };
7864 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_simd, CodeGen);
7865 emitPostUpdateForReductionClause(CGF, D: S,
7866 CondGen: [](CodeGenFunction &) { return nullptr; });
7867}
7868
7869void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
7870 CodeGenModule &CGM, StringRef ParentName,
7871 const OMPTargetTeamsDistributeSimdDirective &S) {
7872 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7873 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7874 };
7875 llvm::Function *Fn;
7876 llvm::Constant *Addr;
7877 // Emit target region as a standalone region.
7878 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7879 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7880 assert(Fn && Addr && "Target device function emission failed.");
7881}
7882
7883void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
7884 const OMPTargetTeamsDistributeSimdDirective &S) {
7885 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7886 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7887 };
7888 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7889}
7890
7891void CodeGenFunction::EmitOMPTeamsDistributeDirective(
7892 const OMPTeamsDistributeDirective &S) {
7893
7894 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7895 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7896 };
7897
7898 // Emit teams region as a standalone region.
7899 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7900 PrePostActionTy &Action) {
7901 Action.Enter(CGF);
7902 OMPPrivateScope PrivateScope(CGF);
7903 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7904 (void)PrivateScope.Privatize();
7905 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7906 CodeGen: CodeGenDistribute);
7907 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7908 };
7909 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
7910 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7911 CondGen: [](CodeGenFunction &) { return nullptr; });
7912}
7913
7914void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
7915 const OMPTeamsDistributeSimdDirective &S) {
7916 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7917 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7918 };
7919
7920 // Emit teams region as a standalone region.
7921 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7922 PrePostActionTy &Action) {
7923 Action.Enter(CGF);
7924 OMPPrivateScope PrivateScope(CGF);
7925 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7926 (void)PrivateScope.Privatize();
7927 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd,
7928 CodeGen: CodeGenDistribute);
7929 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7930 };
7931 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_simd, CodeGen);
7932 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7933 CondGen: [](CodeGenFunction &) { return nullptr; });
7934}
7935
7936void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
7937 const OMPTeamsDistributeParallelForDirective &S) {
7938 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7939 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7940 IncExpr: S.getDistInc());
7941 };
7942
7943 // Emit teams region as a standalone region.
7944 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7945 PrePostActionTy &Action) {
7946 Action.Enter(CGF);
7947 OMPPrivateScope PrivateScope(CGF);
7948 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7949 (void)PrivateScope.Privatize();
7950 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7951 CodeGen: CodeGenDistribute);
7952 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7953 };
7954 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_parallel_for, CodeGen);
7955 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7956 CondGen: [](CodeGenFunction &) { return nullptr; });
7957}
7958
7959void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
7960 const OMPTeamsDistributeParallelForSimdDirective &S) {
7961 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7962 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7963 IncExpr: S.getDistInc());
7964 };
7965
7966 // Emit teams region as a standalone region.
7967 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7968 PrePostActionTy &Action) {
7969 Action.Enter(CGF);
7970 OMPPrivateScope PrivateScope(CGF);
7971 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7972 (void)PrivateScope.Privatize();
7973 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7974 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
7975 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7976 };
7977 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_parallel_for_simd,
7978 CodeGen);
7979 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7980 CondGen: [](CodeGenFunction &) { return nullptr; });
7981}
7982
7983void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) {
7984 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7985 llvm::Value *Device = nullptr;
7986 llvm::Value *NumDependences = nullptr;
7987 llvm::Value *DependenceList = nullptr;
7988
7989 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7990 Device = EmitScalarExpr(E: C->getDevice());
7991
7992 // Build list and emit dependences
7993 OMPTaskDataTy Data;
7994 buildDependences(S, Data);
7995 if (!Data.Dependences.empty()) {
7996 Address DependenciesArray = Address::invalid();
7997 std::tie(args&: NumDependences, args&: DependenciesArray) =
7998 CGM.getOpenMPRuntime().emitDependClause(CGF&: *this, Dependencies: Data.Dependences,
7999 Loc: S.getBeginLoc());
8000 DependenceList = DependenciesArray.emitRawPointer(CGF&: *this);
8001 }
8002 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
8003
8004 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||
8005 S.getSingleClause<OMPDestroyClause>() ||
8006 S.getSingleClause<OMPUseClause>())) &&
8007 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
8008
8009 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();
8010 if (!ItOMPInitClause.empty()) {
8011 // Look at the multiple init clauses
8012 for (const OMPInitClause *C : ItOMPInitClause) {
8013 llvm::Value *InteropvarPtr =
8014 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
8015 llvm::omp::OMPInteropType InteropType =
8016 llvm::omp::OMPInteropType::Unknown;
8017 if (C->getIsTarget()) {
8018 InteropType = llvm::omp::OMPInteropType::Target;
8019 } else {
8020 assert(C->getIsTargetSync() &&
8021 "Expected interop-type target/targetsync");
8022 InteropType = llvm::omp::OMPInteropType::TargetSync;
8023 }
8024 OMPBuilder.createOMPInteropInit(Loc: Builder, InteropVar: InteropvarPtr, InteropType,
8025 Device, NumDependences, DependenceAddress: DependenceList,
8026 HaveNowaitClause: Data.HasNowaitClause);
8027 }
8028 }
8029 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();
8030 if (!ItOMPDestroyClause.empty()) {
8031 // Look at the multiple destroy clauses
8032 for (const OMPDestroyClause *C : ItOMPDestroyClause) {
8033 llvm::Value *InteropvarPtr =
8034 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
8035 OMPBuilder.createOMPInteropDestroy(Loc: Builder, InteropVar: InteropvarPtr, Device,
8036 NumDependences, DependenceAddress: DependenceList,
8037 HaveNowaitClause: Data.HasNowaitClause);
8038 }
8039 }
8040 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();
8041 if (!ItOMPUseClause.empty()) {
8042 // Look at the multiple use clauses
8043 for (const OMPUseClause *C : ItOMPUseClause) {
8044 llvm::Value *InteropvarPtr =
8045 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
8046 OMPBuilder.createOMPInteropUse(Loc: Builder, InteropVar: InteropvarPtr, Device,
8047 NumDependences, DependenceAddress: DependenceList,
8048 HaveNowaitClause: Data.HasNowaitClause);
8049 }
8050 }
8051}
8052
8053static void emitTargetTeamsDistributeParallelForRegion(
8054 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
8055 PrePostActionTy &Action) {
8056 Action.Enter(CGF);
8057 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8058 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
8059 IncExpr: S.getDistInc());
8060 };
8061
8062 // Emit teams region as a standalone region.
8063 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8064 PrePostActionTy &Action) {
8065 Action.Enter(CGF);
8066 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8067 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8068 (void)PrivateScope.Privatize();
8069 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8070 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
8071 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8072 };
8073
8074 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for,
8075 CodeGen: CodeGenTeams);
8076 emitPostUpdateForReductionClause(CGF, D: S,
8077 CondGen: [](CodeGenFunction &) { return nullptr; });
8078}
8079
8080void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
8081 CodeGenModule &CGM, StringRef ParentName,
8082 const OMPTargetTeamsDistributeParallelForDirective &S) {
8083 // Emit SPMD target teams distribute parallel for region as a standalone
8084 // region.
8085 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8086 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
8087 };
8088 llvm::Function *Fn;
8089 llvm::Constant *Addr;
8090 // Emit target region as a standalone region.
8091 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8092 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8093 assert(Fn && Addr && "Target device function emission failed.");
8094}
8095
8096void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
8097 const OMPTargetTeamsDistributeParallelForDirective &S) {
8098 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8099 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
8100 };
8101 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8102}
8103
8104static void emitTargetTeamsDistributeParallelForSimdRegion(
8105 CodeGenFunction &CGF,
8106 const OMPTargetTeamsDistributeParallelForSimdDirective &S,
8107 PrePostActionTy &Action) {
8108 Action.Enter(CGF);
8109 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8110 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
8111 IncExpr: S.getDistInc());
8112 };
8113
8114 // Emit teams region as a standalone region.
8115 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8116 PrePostActionTy &Action) {
8117 Action.Enter(CGF);
8118 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8119 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8120 (void)PrivateScope.Privatize();
8121 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8122 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
8123 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8124 };
8125
8126 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for_simd,
8127 CodeGen: CodeGenTeams);
8128 emitPostUpdateForReductionClause(CGF, D: S,
8129 CondGen: [](CodeGenFunction &) { return nullptr; });
8130}
8131
8132void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
8133 CodeGenModule &CGM, StringRef ParentName,
8134 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
8135 // Emit SPMD target teams distribute parallel for simd region as a standalone
8136 // region.
8137 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8138 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
8139 };
8140 llvm::Function *Fn;
8141 llvm::Constant *Addr;
8142 // Emit target region as a standalone region.
8143 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8144 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8145 assert(Fn && Addr && "Target device function emission failed.");
8146}
8147
8148void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
8149 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
8150 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8151 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
8152 };
8153 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8154}
8155
8156void CodeGenFunction::EmitOMPCancellationPointDirective(
8157 const OMPCancellationPointDirective &S) {
8158 CGM.getOpenMPRuntime().emitCancellationPointCall(CGF&: *this, Loc: S.getBeginLoc(),
8159 CancelRegion: S.getCancelRegion());
8160}
8161
8162void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
8163 const Expr *IfCond = nullptr;
8164 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
8165 if (C->getNameModifier() == OMPD_unknown ||
8166 C->getNameModifier() == OMPD_cancel) {
8167 IfCond = C->getCondition();
8168 break;
8169 }
8170 }
8171 if (CGM.getLangOpts().OpenMPIRBuilder) {
8172 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
8173 // TODO: This check is necessary as we only generate `omp parallel` through
8174 // the OpenMPIRBuilder for now.
8175 if (S.getCancelRegion() == OMPD_parallel ||
8176 S.getCancelRegion() == OMPD_sections ||
8177 S.getCancelRegion() == OMPD_section) {
8178 llvm::Value *IfCondition = nullptr;
8179 if (IfCond)
8180 IfCondition = EmitScalarExpr(E: IfCond,
8181 /*IgnoreResultAssign=*/true);
8182 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
8183 ValOrErr: OMPBuilder.createCancel(Loc: Builder, IfCondition, CanceledDirective: S.getCancelRegion()));
8184 return Builder.restoreIP(IP: AfterIP);
8185 }
8186 }
8187
8188 CGM.getOpenMPRuntime().emitCancelCall(CGF&: *this, Loc: S.getBeginLoc(), IfCond,
8189 CancelRegion: S.getCancelRegion());
8190}
8191
8192CodeGenFunction::JumpDest
8193CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
8194 if (Kind == OMPD_parallel || Kind == OMPD_target_parallel ||
8195 isOpenMPTaskingDirective(Kind))
8196 return ReturnBlock;
8197 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
8198 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
8199 Kind == OMPD_distribute_parallel_for ||
8200 Kind == OMPD_target_parallel_for ||
8201 Kind == OMPD_teams_distribute_parallel_for ||
8202 Kind == OMPD_target_teams_distribute_parallel_for);
8203 return OMPCancelStack.getExitBlock();
8204}
8205
8206void CodeGenFunction::EmitOMPUseDevicePtrClause(
8207 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
8208 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8209 CaptureDeviceAddrMap) {
8210 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8211 for (const Expr *OrigVarIt : C.varlist()) {
8212 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: OrigVarIt)->getDecl());
8213 if (!Processed.insert(V: OrigVD).second)
8214 continue;
8215
8216 // In order to identify the right initializer we need to match the
8217 // declaration used by the mapping logic. In some cases we may get
8218 // OMPCapturedExprDecl that refers to the original declaration.
8219 const ValueDecl *MatchingVD = OrigVD;
8220 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: MatchingVD)) {
8221 // OMPCapturedExprDecl are used to privative fields of the current
8222 // structure.
8223 const auto *ME = cast<MemberExpr>(Val: OED->getInit());
8224 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
8225 "Base should be the current struct!");
8226 MatchingVD = ME->getMemberDecl();
8227 }
8228
8229 // If we don't have information about the current list item, move on to
8230 // the next one.
8231 auto InitAddrIt = CaptureDeviceAddrMap.find(Val: MatchingVD);
8232 if (InitAddrIt == CaptureDeviceAddrMap.end())
8233 continue;
8234
8235 llvm::Type *Ty = ConvertTypeForMem(T: OrigVD->getType().getNonReferenceType());
8236
8237 // Return the address of the private variable.
8238 bool IsRegistered = PrivateScope.addPrivate(
8239 LocalVD: OrigVD,
8240 Addr: Address(InitAddrIt->second, Ty,
8241 getContext().getTypeAlignInChars(T: getContext().VoidPtrTy)));
8242 assert(IsRegistered && "firstprivate var already registered as private");
8243 // Silence the warning about unused variable.
8244 (void)IsRegistered;
8245 }
8246}
8247
8248static const VarDecl *getBaseDecl(const Expr *Ref) {
8249 const Expr *Base = Ref->IgnoreParenImpCasts();
8250 while (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Base))
8251 Base = OASE->getBase()->IgnoreParenImpCasts();
8252 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
8253 Base = ASE->getBase()->IgnoreParenImpCasts();
8254 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Base)->getDecl());
8255}
8256
8257void CodeGenFunction::EmitOMPUseDeviceAddrClause(
8258 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
8259 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8260 CaptureDeviceAddrMap) {
8261 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8262 for (const Expr *Ref : C.varlist()) {
8263 const VarDecl *OrigVD = getBaseDecl(Ref);
8264 if (!Processed.insert(V: OrigVD).second)
8265 continue;
8266 // In order to identify the right initializer we need to match the
8267 // declaration used by the mapping logic. In some cases we may get
8268 // OMPCapturedExprDecl that refers to the original declaration.
8269 const ValueDecl *MatchingVD = OrigVD;
8270 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: MatchingVD)) {
8271 // OMPCapturedExprDecl are used to privative fields of the current
8272 // structure.
8273 const auto *ME = cast<MemberExpr>(Val: OED->getInit());
8274 assert(isa<CXXThisExpr>(ME->getBase()) &&
8275 "Base should be the current struct!");
8276 MatchingVD = ME->getMemberDecl();
8277 }
8278
8279 // If we don't have information about the current list item, move on to
8280 // the next one.
8281 auto InitAddrIt = CaptureDeviceAddrMap.find(Val: MatchingVD);
8282 if (InitAddrIt == CaptureDeviceAddrMap.end())
8283 continue;
8284
8285 llvm::Type *Ty = ConvertTypeForMem(T: OrigVD->getType().getNonReferenceType());
8286
8287 Address PrivAddr =
8288 Address(InitAddrIt->second, Ty,
8289 getContext().getTypeAlignInChars(T: getContext().VoidPtrTy));
8290 // For declrefs and variable length array need to load the pointer for
8291 // correct mapping, since the pointer to the data was passed to the runtime.
8292 if (isa<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts()) ||
8293 MatchingVD->getType()->isArrayType()) {
8294 QualType PtrTy = getContext().getPointerType(
8295 T: OrigVD->getType().getNonReferenceType());
8296 PrivAddr =
8297 EmitLoadOfPointer(Ptr: PrivAddr.withElementType(ElemTy: ConvertTypeForMem(T: PtrTy)),
8298 PtrTy: PtrTy->castAs<PointerType>());
8299 }
8300
8301 (void)PrivateScope.addPrivate(LocalVD: OrigVD, Addr: PrivAddr);
8302 }
8303}
8304
8305// Generate the instructions for '#pragma omp target data' directive.
8306void CodeGenFunction::EmitOMPTargetDataDirective(
8307 const OMPTargetDataDirective &S) {
8308 // Emit vtable only from host for target data directive.
8309 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
8310 CGM.getOpenMPRuntime().registerVTable(D: S);
8311
8312 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,
8313 /*SeparateBeginEndCalls=*/true);
8314
8315 // Create a pre/post action to signal the privatization of the device pointer.
8316 // This action can be replaced by the OpenMP runtime code generation to
8317 // deactivate privatization.
8318 bool PrivatizeDevicePointers = false;
8319 class DevicePointerPrivActionTy : public PrePostActionTy {
8320 bool &PrivatizeDevicePointers;
8321
8322 public:
8323 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
8324 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
8325 void Enter(CodeGenFunction &CGF) override {
8326 PrivatizeDevicePointers = true;
8327 }
8328 };
8329 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
8330
8331 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8332 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8333 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
8334 };
8335
8336 // Codegen that selects whether to generate the privatization code or not.
8337 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8338 RegionCodeGenTy RCG(InnermostCodeGen);
8339 PrivatizeDevicePointers = false;
8340
8341 // Call the pre-action to change the status of PrivatizeDevicePointers if
8342 // needed.
8343 Action.Enter(CGF);
8344
8345 if (PrivatizeDevicePointers) {
8346 OMPPrivateScope PrivateScope(CGF);
8347 // Emit all instances of the use_device_ptr clause.
8348 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8349 CGF.EmitOMPUseDevicePtrClause(C: *C, PrivateScope,
8350 CaptureDeviceAddrMap: Info.CaptureDeviceAddrMap);
8351 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8352 CGF.EmitOMPUseDeviceAddrClause(C: *C, PrivateScope,
8353 CaptureDeviceAddrMap: Info.CaptureDeviceAddrMap);
8354 (void)PrivateScope.Privatize();
8355 RCG(CGF);
8356 } else {
8357 // If we don't have target devices, don't bother emitting the data
8358 // mapping code.
8359 std::optional<OpenMPDirectiveKind> CaptureRegion;
8360 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8361 // Emit helper decls of the use_device_ptr/use_device_addr clauses.
8362 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8363 for (const Expr *E : C->varlist()) {
8364 const Decl *D = cast<DeclRefExpr>(Val: E)->getDecl();
8365 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
8366 CGF.EmitVarDecl(D: *OED);
8367 }
8368 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8369 for (const Expr *E : C->varlist()) {
8370 const Decl *D = getBaseDecl(Ref: E);
8371 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
8372 CGF.EmitVarDecl(D: *OED);
8373 }
8374 } else {
8375 CaptureRegion = OMPD_unknown;
8376 }
8377
8378 OMPLexicalScope Scope(CGF, S, CaptureRegion);
8379 RCG(CGF);
8380 }
8381 };
8382
8383 // Forward the provided action to the privatization codegen.
8384 RegionCodeGenTy PrivRCG(PrivCodeGen);
8385 PrivRCG.setAction(Action);
8386
8387 // Notwithstanding the body of the region is emitted as inlined directive,
8388 // we don't use an inline scope as changes in the references inside the
8389 // region are expected to be visible outside, so we do not privative them.
8390 OMPLexicalScope Scope(CGF, S);
8391 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_target_data,
8392 CodeGen: PrivRCG);
8393 };
8394
8395 RegionCodeGenTy RCG(CodeGen);
8396
8397 // If we don't have target devices, don't bother emitting the data mapping
8398 // code.
8399 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8400 RCG(*this);
8401 return;
8402 }
8403
8404 // Check if we have any if clause associated with the directive.
8405 const Expr *IfCond = nullptr;
8406 if (const auto *C = S.getSingleClause<OMPIfClause>())
8407 IfCond = C->getCondition();
8408
8409 // Check if we have any device clause associated with the directive.
8410 const Expr *Device = nullptr;
8411 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8412 Device = C->getDevice();
8413
8414 // Set the action to signal privatization of device pointers.
8415 RCG.setAction(PrivAction);
8416
8417 // Emit region code.
8418 CGM.getOpenMPRuntime().emitTargetDataCalls(CGF&: *this, D: S, IfCond, Device, CodeGen: RCG,
8419 Info);
8420}
8421
8422void CodeGenFunction::EmitOMPTargetEnterDataDirective(
8423 const OMPTargetEnterDataDirective &S) {
8424 // If we don't have target devices, don't bother emitting the data mapping
8425 // code.
8426 if (CGM.getLangOpts().OMPTargetTriples.empty())
8427 return;
8428
8429 // Check if we have any if clause associated with the directive.
8430 const Expr *IfCond = nullptr;
8431 if (const auto *C = S.getSingleClause<OMPIfClause>())
8432 IfCond = C->getCondition();
8433
8434 // Check if we have any device clause associated with the directive.
8435 const Expr *Device = nullptr;
8436 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8437 Device = C->getDevice();
8438
8439 OMPLexicalScope Scope(*this, S, OMPD_task);
8440 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8441}
8442
8443void CodeGenFunction::EmitOMPTargetExitDataDirective(
8444 const OMPTargetExitDataDirective &S) {
8445 // If we don't have target devices, don't bother emitting the data mapping
8446 // code.
8447 if (CGM.getLangOpts().OMPTargetTriples.empty())
8448 return;
8449
8450 // Check if we have any if clause associated with the directive.
8451 const Expr *IfCond = nullptr;
8452 if (const auto *C = S.getSingleClause<OMPIfClause>())
8453 IfCond = C->getCondition();
8454
8455 // Check if we have any device clause associated with the directive.
8456 const Expr *Device = nullptr;
8457 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8458 Device = C->getDevice();
8459
8460 OMPLexicalScope Scope(*this, S, OMPD_task);
8461 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8462}
8463
8464static void emitTargetParallelRegion(CodeGenFunction &CGF,
8465 const OMPTargetParallelDirective &S,
8466 PrePostActionTy &Action) {
8467 // Get the captured statement associated with the 'parallel' region.
8468 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
8469 Action.Enter(CGF);
8470 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
8471 Action.Enter(CGF);
8472 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8473 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
8474 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
8475 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8476 (void)PrivateScope.Privatize();
8477 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
8478 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
8479 // TODO: Add support for clauses.
8480 CGF.EmitStmt(S: CS->getCapturedStmt());
8481 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
8482 };
8483 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_parallel, CodeGen,
8484 CodeGenBoundParameters: emitEmptyBoundParameters);
8485 emitPostUpdateForReductionClause(CGF, D: S,
8486 CondGen: [](CodeGenFunction &) { return nullptr; });
8487}
8488
8489void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
8490 CodeGenModule &CGM, StringRef ParentName,
8491 const OMPTargetParallelDirective &S) {
8492 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8493 emitTargetParallelRegion(CGF, S, Action);
8494 };
8495 llvm::Function *Fn;
8496 llvm::Constant *Addr;
8497 // Emit target region as a standalone region.
8498 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8499 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8500 assert(Fn && Addr && "Target device function emission failed.");
8501}
8502
8503void CodeGenFunction::EmitOMPTargetParallelDirective(
8504 const OMPTargetParallelDirective &S) {
8505 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8506 emitTargetParallelRegion(CGF, S, Action);
8507 };
8508 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8509}
8510
8511static void emitTargetParallelForRegion(CodeGenFunction &CGF,
8512 const OMPTargetParallelForDirective &S,
8513 PrePostActionTy &Action) {
8514 Action.Enter(CGF);
8515 // Emit directive as a combined directive that consists of two implicit
8516 // directives: 'parallel' with 'for' directive.
8517 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8518 Action.Enter(CGF);
8519 CodeGenFunction::OMPCancelStackRAII CancelRegion(
8520 CGF, OMPD_target_parallel_for, S.hasCancel());
8521 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
8522 CGDispatchBounds: emitDispatchForLoopBounds);
8523 };
8524 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_for, CodeGen,
8525 CodeGenBoundParameters: emitEmptyBoundParameters);
8526}
8527
8528void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
8529 CodeGenModule &CGM, StringRef ParentName,
8530 const OMPTargetParallelForDirective &S) {
8531 // Emit SPMD target parallel for region as a standalone region.
8532 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8533 emitTargetParallelForRegion(CGF, S, Action);
8534 };
8535 llvm::Function *Fn;
8536 llvm::Constant *Addr;
8537 // Emit target region as a standalone region.
8538 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8539 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8540 assert(Fn && Addr && "Target device function emission failed.");
8541}
8542
8543void CodeGenFunction::EmitOMPTargetParallelForDirective(
8544 const OMPTargetParallelForDirective &S) {
8545 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8546 emitTargetParallelForRegion(CGF, S, Action);
8547 };
8548 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8549}
8550
8551static void
8552emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
8553 const OMPTargetParallelForSimdDirective &S,
8554 PrePostActionTy &Action) {
8555 Action.Enter(CGF);
8556 // Emit directive as a combined directive that consists of two implicit
8557 // directives: 'parallel' with 'for' directive.
8558 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8559 Action.Enter(CGF);
8560 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
8561 CGDispatchBounds: emitDispatchForLoopBounds);
8562 };
8563 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_simd, CodeGen,
8564 CodeGenBoundParameters: emitEmptyBoundParameters);
8565}
8566
8567void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
8568 CodeGenModule &CGM, StringRef ParentName,
8569 const OMPTargetParallelForSimdDirective &S) {
8570 // Emit SPMD target parallel for region as a standalone region.
8571 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8572 emitTargetParallelForSimdRegion(CGF, S, Action);
8573 };
8574 llvm::Function *Fn;
8575 llvm::Constant *Addr;
8576 // Emit target region as a standalone region.
8577 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8578 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8579 assert(Fn && Addr && "Target device function emission failed.");
8580}
8581
8582void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
8583 const OMPTargetParallelForSimdDirective &S) {
8584 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8585 emitTargetParallelForSimdRegion(CGF, S, Action);
8586 };
8587 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8588}
8589
8590/// Emit a helper variable and return corresponding lvalue.
8591static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
8592 const ImplicitParamDecl *PVD,
8593 CodeGenFunction::OMPPrivateScope &Privates) {
8594 const auto *VDecl = cast<VarDecl>(Val: Helper->getDecl());
8595 Privates.addPrivate(LocalVD: VDecl, Addr: CGF.GetAddrOfLocalVar(VD: PVD));
8596}
8597
8598void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
8599 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
8600 // Emit outlined function for task construct.
8601 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_taskloop);
8602 Address CapturedStruct = Address::invalid();
8603 {
8604 OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8605 CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
8606 }
8607 CanQualType SharedsTy =
8608 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
8609 const Expr *IfCond = nullptr;
8610 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
8611 if (C->getNameModifier() == OMPD_unknown ||
8612 C->getNameModifier() == OMPD_taskloop) {
8613 IfCond = C->getCondition();
8614 break;
8615 }
8616 }
8617
8618 OMPTaskDataTy Data;
8619 // Check if taskloop must be emitted without taskgroup.
8620 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
8621 // TODO: Check if we should emit tied or untied task.
8622 Data.Tied = true;
8623 // Set scheduling for taskloop
8624 if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
8625 // grainsize clause
8626 Data.Schedule.setInt(/*IntVal=*/false);
8627 Data.Schedule.setPointer(EmitScalarExpr(E: Clause->getGrainsize()));
8628 Data.HasModifier =
8629 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;
8630 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
8631 // num_tasks clause
8632 Data.Schedule.setInt(/*IntVal=*/true);
8633 Data.Schedule.setPointer(EmitScalarExpr(E: Clause->getNumTasks()));
8634 Data.HasModifier =
8635 (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;
8636 }
8637
8638 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
8639 // if (PreCond) {
8640 // for (IV in 0..LastIteration) BODY;
8641 // <Final counter/linear vars updates>;
8642 // }
8643 //
8644
8645 // Emit: if (PreCond) - begin.
8646 // If the condition constant folds and can be elided, avoid emitting the
8647 // whole loop.
8648 bool CondConstant;
8649 llvm::BasicBlock *ContBlock = nullptr;
8650 OMPLoopScope PreInitScope(CGF, S);
8651 if (CGF.ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
8652 if (!CondConstant)
8653 return;
8654 } else {
8655 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "taskloop.if.then");
8656 ContBlock = CGF.createBasicBlock(name: "taskloop.if.end");
8657 emitPreCond(CGF, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
8658 TrueCount: CGF.getProfileCount(S: &S));
8659 CGF.EmitBlock(BB: ThenBlock);
8660 CGF.incrementProfileCounter(S: &S);
8661 }
8662
8663 (void)CGF.EmitOMPLinearClauseInit(D: S);
8664
8665 OMPPrivateScope LoopScope(CGF);
8666 // Emit helper vars inits.
8667 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8668 auto *I = CS->getCapturedDecl()->param_begin();
8669 auto *LBP = std::next(x: I, n: LowerBound);
8670 auto *UBP = std::next(x: I, n: UpperBound);
8671 auto *STP = std::next(x: I, n: Stride);
8672 auto *LIP = std::next(x: I, n: LastIter);
8673 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getLowerBoundVariable()), PVD: *LBP,
8674 Privates&: LoopScope);
8675 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getUpperBoundVariable()), PVD: *UBP,
8676 Privates&: LoopScope);
8677 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()), PVD: *STP, Privates&: LoopScope);
8678 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()), PVD: *LIP,
8679 Privates&: LoopScope);
8680 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8681 CGF.EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
8682 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
8683 (void)LoopScope.Privatize();
8684 // Emit the loop iteration variable.
8685 const Expr *IVExpr = S.getIterationVariable();
8686 const auto *IVDecl = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IVExpr)->getDecl());
8687 CGF.EmitVarDecl(D: *IVDecl);
8688 CGF.EmitIgnoredExpr(E: S.getInit());
8689
8690 // Emit the iterations count variable.
8691 // If it is not a variable, Sema decided to calculate iterations count on
8692 // each iteration (e.g., it is foldable into a constant).
8693 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
8694 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
8695 // Emit calculation of the iterations count.
8696 CGF.EmitIgnoredExpr(E: S.getCalcLastIteration());
8697 }
8698
8699 {
8700 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8701 emitCommonSimdLoop(
8702 CGF, S,
8703 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8704 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()))
8705 CGF.EmitOMPSimdInit(D: S);
8706 },
8707 BodyCodeGen: [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
8708 CGF.EmitOMPInnerLoop(
8709 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: S.getCond(), IncExpr: S.getInc(),
8710 BodyGen: [&S](CodeGenFunction &CGF) {
8711 emitOMPLoopBodyWithStopPoint(CGF, S,
8712 LoopExit: CodeGenFunction::JumpDest());
8713 },
8714 PostIncGen: [](CodeGenFunction &) {});
8715 });
8716 }
8717 // Emit: if (PreCond) - end.
8718 if (ContBlock) {
8719 CGF.EmitBranch(Block: ContBlock);
8720 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
8721 }
8722 // Emit final copy of the lastprivate variables if IsLastIter != 0.
8723 if (HasLastprivateClause) {
8724 CGF.EmitOMPLastprivateClauseFinal(
8725 D: S, NoFinals: isOpenMPSimdDirective(DKind: S.getDirectiveKind()),
8726 IsLastIterCond: CGF.Builder.CreateIsNotNull(Arg: CGF.EmitLoadOfScalar(
8727 Addr: CGF.GetAddrOfLocalVar(VD: *LIP), /*Volatile=*/false,
8728 Ty: (*LIP)->getType(), Loc: S.getBeginLoc())));
8729 }
8730 LoopScope.restoreMap();
8731 CGF.EmitOMPLinearClauseFinal(D: S, CondGen: [LIP, &S](CodeGenFunction &CGF) {
8732 return CGF.Builder.CreateIsNotNull(
8733 Arg: CGF.EmitLoadOfScalar(Addr: CGF.GetAddrOfLocalVar(VD: *LIP), /*Volatile=*/false,
8734 Ty: (*LIP)->getType(), Loc: S.getBeginLoc()));
8735 });
8736 };
8737 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8738 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8739 const OMPTaskDataTy &Data) {
8740 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8741 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
8742 OMPLoopScope PreInitScope(CGF, S);
8743 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, Loc: S.getBeginLoc(), D: S,
8744 TaskFunction: OutlinedFn, SharedsTy,
8745 Shareds: CapturedStruct, IfCond, Data);
8746 };
8747 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_taskloop,
8748 CodeGen);
8749 };
8750 if (Data.Nogroup) {
8751 EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_taskloop, BodyGen, TaskGen, Data);
8752 } else {
8753 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8754 CGF&: *this,
8755 TaskgroupOpGen: [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
8756 PrePostActionTy &Action) {
8757 Action.Enter(CGF);
8758 CGF.EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_taskloop, BodyGen, TaskGen,
8759 Data);
8760 },
8761 Loc: S.getBeginLoc());
8762 }
8763}
8764
8765void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
8766 auto LPCRegion =
8767 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8768 EmitOMPTaskLoopBasedDirective(S);
8769}
8770
8771void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
8772 const OMPTaskLoopSimdDirective &S) {
8773 auto LPCRegion =
8774 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8775 OMPLexicalScope Scope(*this, S);
8776 EmitOMPTaskLoopBasedDirective(S);
8777}
8778
8779void CodeGenFunction::EmitOMPMasterTaskLoopDirective(
8780 const OMPMasterTaskLoopDirective &S) {
8781 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8782 Action.Enter(CGF);
8783 EmitOMPTaskLoopBasedDirective(S);
8784 };
8785 auto LPCRegion =
8786 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8787 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8788 CGM.getOpenMPRuntime().emitMasterRegion(CGF&: *this, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
8789}
8790
8791void CodeGenFunction::EmitOMPMaskedTaskLoopDirective(
8792 const OMPMaskedTaskLoopDirective &S) {
8793 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8794 Action.Enter(CGF);
8795 EmitOMPTaskLoopBasedDirective(S);
8796 };
8797 auto LPCRegion =
8798 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8799 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8800 CGM.getOpenMPRuntime().emitMaskedRegion(CGF&: *this, MaskedOpGen: CodeGen, Loc: S.getBeginLoc());
8801}
8802
8803void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(
8804 const OMPMasterTaskLoopSimdDirective &S) {
8805 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8806 Action.Enter(CGF);
8807 EmitOMPTaskLoopBasedDirective(S);
8808 };
8809 auto LPCRegion =
8810 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8811 OMPLexicalScope Scope(*this, S);
8812 CGM.getOpenMPRuntime().emitMasterRegion(CGF&: *this, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
8813}
8814
8815void CodeGenFunction::EmitOMPMaskedTaskLoopSimdDirective(
8816 const OMPMaskedTaskLoopSimdDirective &S) {
8817 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8818 Action.Enter(CGF);
8819 EmitOMPTaskLoopBasedDirective(S);
8820 };
8821 auto LPCRegion =
8822 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8823 OMPLexicalScope Scope(*this, S);
8824 CGM.getOpenMPRuntime().emitMaskedRegion(CGF&: *this, MaskedOpGen: CodeGen, Loc: S.getBeginLoc());
8825}
8826
8827void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(
8828 const OMPParallelMasterTaskLoopDirective &S) {
8829 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8830 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8831 PrePostActionTy &Action) {
8832 Action.Enter(CGF);
8833 CGF.EmitOMPTaskLoopBasedDirective(S);
8834 };
8835 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8836 CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: TaskLoopCodeGen,
8837 Loc: S.getBeginLoc());
8838 };
8839 auto LPCRegion =
8840 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8841 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master_taskloop, CodeGen,
8842 CodeGenBoundParameters: emitEmptyBoundParameters);
8843}
8844
8845void CodeGenFunction::EmitOMPParallelMaskedTaskLoopDirective(
8846 const OMPParallelMaskedTaskLoopDirective &S) {
8847 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8848 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8849 PrePostActionTy &Action) {
8850 Action.Enter(CGF);
8851 CGF.EmitOMPTaskLoopBasedDirective(S);
8852 };
8853 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8854 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: TaskLoopCodeGen,
8855 Loc: S.getBeginLoc());
8856 };
8857 auto LPCRegion =
8858 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8859 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked_taskloop, CodeGen,
8860 CodeGenBoundParameters: emitEmptyBoundParameters);
8861}
8862
8863void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(
8864 const OMPParallelMasterTaskLoopSimdDirective &S) {
8865 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8866 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8867 PrePostActionTy &Action) {
8868 Action.Enter(CGF);
8869 CGF.EmitOMPTaskLoopBasedDirective(S);
8870 };
8871 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8872 CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: TaskLoopCodeGen,
8873 Loc: S.getBeginLoc());
8874 };
8875 auto LPCRegion =
8876 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8877 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master_taskloop_simd, CodeGen,
8878 CodeGenBoundParameters: emitEmptyBoundParameters);
8879}
8880
8881void CodeGenFunction::EmitOMPParallelMaskedTaskLoopSimdDirective(
8882 const OMPParallelMaskedTaskLoopSimdDirective &S) {
8883 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8884 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8885 PrePostActionTy &Action) {
8886 Action.Enter(CGF);
8887 CGF.EmitOMPTaskLoopBasedDirective(S);
8888 };
8889 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8890 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: TaskLoopCodeGen,
8891 Loc: S.getBeginLoc());
8892 };
8893 auto LPCRegion =
8894 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8895 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked_taskloop_simd, CodeGen,
8896 CodeGenBoundParameters: emitEmptyBoundParameters);
8897}
8898
8899// Generate the instructions for '#pragma omp target update' directive.
8900void CodeGenFunction::EmitOMPTargetUpdateDirective(
8901 const OMPTargetUpdateDirective &S) {
8902 // If we don't have target devices, don't bother emitting the data mapping
8903 // code.
8904 if (CGM.getLangOpts().OMPTargetTriples.empty())
8905 return;
8906
8907 // Check if we have any if clause associated with the directive.
8908 const Expr *IfCond = nullptr;
8909 if (const auto *C = S.getSingleClause<OMPIfClause>())
8910 IfCond = C->getCondition();
8911
8912 // Check if we have any device clause associated with the directive.
8913 const Expr *Device = nullptr;
8914 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8915 Device = C->getDevice();
8916
8917 OMPLexicalScope Scope(*this, S, OMPD_task);
8918 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8919}
8920
8921void CodeGenFunction::EmitOMPGenericLoopDirective(
8922 const OMPGenericLoopDirective &S) {
8923 // Always expect a bind clause on the loop directive. It it wasn't
8924 // in the source, it should have been added in sema.
8925
8926 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;
8927 if (const auto *C = S.getSingleClause<OMPBindClause>())
8928 BindKind = C->getBindKind();
8929
8930 switch (BindKind) {
8931 case OMPC_BIND_parallel: // for
8932 return emitOMPForDirective(S, CGF&: *this, CGM, /*HasCancel=*/false);
8933 case OMPC_BIND_teams: // distribute
8934 return emitOMPDistributeDirective(S, CGF&: *this, CGM);
8935 case OMPC_BIND_thread: // simd
8936 return emitOMPSimdDirective(S, CGF&: *this, CGM);
8937 case OMPC_BIND_unknown:
8938 break;
8939 }
8940
8941 // Unimplemented, just inline the underlying statement for now.
8942 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8943 // Emit the loop iteration variable.
8944 const Stmt *CS =
8945 cast<CapturedStmt>(Val: S.getAssociatedStmt())->getCapturedStmt();
8946 const auto *ForS = dyn_cast<ForStmt>(Val: CS);
8947 if (ForS && !isa<DeclStmt>(Val: ForS->getInit())) {
8948 OMPPrivateScope LoopScope(CGF);
8949 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8950 (void)LoopScope.Privatize();
8951 CGF.EmitStmt(S: CS);
8952 LoopScope.restoreMap();
8953 } else {
8954 CGF.EmitStmt(S: CS);
8955 }
8956 };
8957 OMPLexicalScope Scope(*this, S, OMPD_unknown);
8958 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_loop, CodeGen);
8959}
8960
8961void CodeGenFunction::EmitOMPParallelGenericLoopDirective(
8962 const OMPLoopDirective &S) {
8963 // Emit combined directive as if its constituent constructs are 'parallel'
8964 // and 'for'.
8965 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8966 Action.Enter(CGF);
8967 emitOMPCopyinClause(CGF, S);
8968 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
8969 };
8970 {
8971 auto LPCRegion =
8972 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8973 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for, CodeGen,
8974 CodeGenBoundParameters: emitEmptyBoundParameters);
8975 }
8976 // Check for outer lastprivate conditional update.
8977 checkForLastprivateConditionalUpdate(CGF&: *this, S);
8978}
8979
8980void CodeGenFunction::EmitOMPTeamsGenericLoopDirective(
8981 const OMPTeamsGenericLoopDirective &S) {
8982 // To be consistent with current behavior of 'target teams loop', emit
8983 // 'teams loop' as if its constituent constructs are 'teams' and 'distribute'.
8984 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8985 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
8986 };
8987
8988 // Emit teams region as a standalone region.
8989 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8990 PrePostActionTy &Action) {
8991 Action.Enter(CGF);
8992 OMPPrivateScope PrivateScope(CGF);
8993 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8994 (void)PrivateScope.Privatize();
8995 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
8996 CodeGen: CodeGenDistribute);
8997 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8998 };
8999 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
9000 emitPostUpdateForReductionClause(CGF&: *this, D: S,
9001 CondGen: [](CodeGenFunction &) { return nullptr; });
9002}
9003
9004#ifndef NDEBUG
9005static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF,
9006 std::string StatusMsg,
9007 const OMPExecutableDirective &D) {
9008 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
9009 if (IsDevice)
9010 StatusMsg += ": DEVICE";
9011 else
9012 StatusMsg += ": HOST";
9013 SourceLocation L = D.getBeginLoc();
9014 auto &SM = CGF.getContext().getSourceManager();
9015 PresumedLoc PLoc = SM.getPresumedLoc(L);
9016 const char *FileName = PLoc.isValid() ? PLoc.getFilename() : nullptr;
9017 unsigned LineNo =
9018 PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L);
9019 llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n";
9020}
9021#endif
9022
9023static void emitTargetTeamsGenericLoopRegionAsParallel(
9024 CodeGenFunction &CGF, PrePostActionTy &Action,
9025 const OMPTargetTeamsGenericLoopDirective &S) {
9026 Action.Enter(CGF);
9027 // Emit 'teams loop' as if its constituent constructs are 'distribute,
9028 // 'parallel, and 'for'.
9029 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
9030 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
9031 IncExpr: S.getDistInc());
9032 };
9033
9034 // Emit teams region as a standalone region.
9035 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
9036 PrePostActionTy &Action) {
9037 Action.Enter(CGF);
9038 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
9039 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
9040 (void)PrivateScope.Privatize();
9041 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
9042 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
9043 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
9044 };
9045 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
9046 emitTargetTeamsLoopCodegenStatus(
9047 CGF, TTL_CODEGEN_TYPE " as parallel for", S));
9048 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for,
9049 CodeGen: CodeGenTeams);
9050 emitPostUpdateForReductionClause(CGF, D: S,
9051 CondGen: [](CodeGenFunction &) { return nullptr; });
9052}
9053
9054static void emitTargetTeamsGenericLoopRegionAsDistribute(
9055 CodeGenFunction &CGF, PrePostActionTy &Action,
9056 const OMPTargetTeamsGenericLoopDirective &S) {
9057 Action.Enter(CGF);
9058 // Emit 'teams loop' as if its constituent construct is 'distribute'.
9059 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
9060 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
9061 };
9062
9063 // Emit teams region as a standalone region.
9064 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
9065 PrePostActionTy &Action) {
9066 Action.Enter(CGF);
9067 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
9068 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
9069 (void)PrivateScope.Privatize();
9070 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
9071 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
9072 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
9073 };
9074 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
9075 emitTargetTeamsLoopCodegenStatus(
9076 CGF, TTL_CODEGEN_TYPE " as distribute", S));
9077 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute, CodeGen);
9078 emitPostUpdateForReductionClause(CGF, D: S,
9079 CondGen: [](CodeGenFunction &) { return nullptr; });
9080}
9081
9082void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDirective(
9083 const OMPTargetTeamsGenericLoopDirective &S) {
9084 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9085 if (S.canBeParallelFor())
9086 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
9087 else
9088 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
9089 };
9090 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
9091}
9092
9093void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
9094 CodeGenModule &CGM, StringRef ParentName,
9095 const OMPTargetTeamsGenericLoopDirective &S) {
9096 // Emit SPMD target parallel loop region as a standalone region.
9097 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9098 if (S.canBeParallelFor())
9099 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
9100 else
9101 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
9102 };
9103 llvm::Function *Fn;
9104 llvm::Constant *Addr;
9105 // Emit target region as a standalone region.
9106 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9107 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
9108 assert(Fn && Addr &&
9109 "Target device function emission failed for 'target teams loop'.");
9110}
9111
9112static void emitTargetParallelGenericLoopRegion(
9113 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S,
9114 PrePostActionTy &Action) {
9115 Action.Enter(CGF);
9116 // Emit as 'parallel for'.
9117 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9118 Action.Enter(CGF);
9119 CodeGenFunction::OMPCancelStackRAII CancelRegion(
9120 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
9121 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
9122 CGDispatchBounds: emitDispatchForLoopBounds);
9123 };
9124 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_for, CodeGen,
9125 CodeGenBoundParameters: emitEmptyBoundParameters);
9126}
9127
9128void CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
9129 CodeGenModule &CGM, StringRef ParentName,
9130 const OMPTargetParallelGenericLoopDirective &S) {
9131 // Emit target parallel loop region as a standalone region.
9132 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9133 emitTargetParallelGenericLoopRegion(CGF, S, Action);
9134 };
9135 llvm::Function *Fn;
9136 llvm::Constant *Addr;
9137 // Emit target region as a standalone region.
9138 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9139 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
9140 assert(Fn && Addr && "Target device function emission failed.");
9141}
9142
9143/// Emit combined directive 'target parallel loop' as if its constituent
9144/// constructs are 'target', 'parallel', and 'for'.
9145void CodeGenFunction::EmitOMPTargetParallelGenericLoopDirective(
9146 const OMPTargetParallelGenericLoopDirective &S) {
9147 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9148 emitTargetParallelGenericLoopRegion(CGF, S, Action);
9149 };
9150 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
9151}
9152
9153void CodeGenFunction::EmitSimpleOMPExecutableDirective(
9154 const OMPExecutableDirective &D) {
9155 if (const auto *SD = dyn_cast<OMPScanDirective>(Val: &D)) {
9156 EmitOMPScanDirective(S: *SD);
9157 return;
9158 }
9159 // A metadirective has no associated statement; emit its variants.
9160 if (const auto *MD = dyn_cast<OMPMetaDirective>(Val: &D)) {
9161 EmitOMPMetaDirective(S: *MD);
9162 return;
9163 }
9164 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
9165 return;
9166 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
9167 OMPPrivateScope GlobalsScope(CGF);
9168 if (isOpenMPTaskingDirective(Kind: D.getDirectiveKind())) {
9169 // Capture global firstprivates to avoid crash.
9170 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
9171 for (const Expr *Ref : C->varlist()) {
9172 const auto *DRE = cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
9173 if (!DRE)
9174 continue;
9175 const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl());
9176 if (!VD || VD->hasLocalStorage())
9177 continue;
9178 if (!CGF.LocalDeclMap.count(Val: VD)) {
9179 LValue GlobLVal = CGF.EmitLValue(E: Ref);
9180 GlobalsScope.addPrivate(LocalVD: VD, Addr: GlobLVal.getAddress());
9181 }
9182 }
9183 }
9184 }
9185 if (isOpenMPSimdDirective(DKind: D.getDirectiveKind())) {
9186 (void)GlobalsScope.Privatize();
9187 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
9188 emitOMPSimdRegion(CGF, S: cast<OMPLoopDirective>(Val: D), Action);
9189 } else {
9190 if (const auto *LD = dyn_cast<OMPLoopDirective>(Val: &D)) {
9191 for (const Expr *E : LD->counters()) {
9192 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
9193 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(Val: VD)) {
9194 LValue GlobLVal = CGF.EmitLValue(E);
9195 GlobalsScope.addPrivate(LocalVD: VD, Addr: GlobLVal.getAddress());
9196 }
9197 if (isa<OMPCapturedExprDecl>(Val: VD)) {
9198 // Emit only those that were not explicitly referenced in clauses.
9199 if (!CGF.LocalDeclMap.count(Val: VD))
9200 CGF.EmitVarDecl(D: *VD);
9201 }
9202 }
9203 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
9204 if (!C->getNumForLoops())
9205 continue;
9206 for (unsigned I = LD->getLoopsNumber(),
9207 E = C->getLoopNumIterations().size();
9208 I < E; ++I) {
9209 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
9210 Val: cast<DeclRefExpr>(Val: C->getLoopCounter(NumLoop: I))->getDecl())) {
9211 // Emit only those that were not explicitly referenced in clauses.
9212 if (!CGF.LocalDeclMap.count(Val: VD))
9213 CGF.EmitVarDecl(D: *VD);
9214 }
9215 }
9216 }
9217 }
9218 (void)GlobalsScope.Privatize();
9219 CGF.EmitStmt(S: D.getInnermostCapturedStmt()->getCapturedStmt());
9220 }
9221 };
9222 if (D.getDirectiveKind() == OMPD_atomic ||
9223 D.getDirectiveKind() == OMPD_critical ||
9224 D.getDirectiveKind() == OMPD_section ||
9225 D.getDirectiveKind() == OMPD_master ||
9226 D.getDirectiveKind() == OMPD_masked ||
9227 D.getDirectiveKind() == OMPD_assume ||
9228 isOpenMPLoopTransformationDirective(DKind: D.getDirectiveKind())) {
9229 EmitStmt(S: D.getAssociatedStmt());
9230 } else {
9231 auto LPCRegion =
9232 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S: D);
9233 OMPSimdLexicalScope Scope(*this, D);
9234 CGM.getOpenMPRuntime().emitInlinedDirective(
9235 CGF&: *this,
9236 InnermostKind: isOpenMPSimdDirective(DKind: D.getDirectiveKind()) ? OMPD_simd
9237 : D.getDirectiveKind(),
9238 CodeGen);
9239 }
9240 // Check for outer lastprivate conditional update.
9241 checkForLastprivateConditionalUpdate(CGF&: *this, S: D);
9242}
9243
9244void CodeGenFunction::EmitOMPAssumeDirective(const OMPAssumeDirective &S) {
9245 for (const auto *C : S.getClausesOfKind<OMPHoldsClause>()) {
9246 const Expr *E = C->getExpr();
9247 assert(E && "holds clause requires an expression");
9248 if (!E->HasSideEffects(Ctx: getContext()))
9249 Builder.CreateAssumption(Cond: EvaluateExprAsBool(E));
9250 }
9251 EmitStmt(S: S.getAssociatedStmt());
9252}
9253