1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGDebugInfo.h"
15#include "CGOpenMPRuntime.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
18#include "CodeGenPGO.h"
19#include "TargetInfo.h"
20#include "clang/AST/ASTContext.h"
21#include "clang/AST/Attr.h"
22#include "clang/AST/DeclOpenMP.h"
23#include "clang/AST/OpenMPClause.h"
24#include "clang/AST/Stmt.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/PrettyStackTrace.h"
30#include "clang/Basic/SourceManager.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/BinaryFormat/Dwarf.h"
33#include "llvm/Frontend/OpenMP/OMPConstants.h"
34#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugInfoMetadata.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/AtomicOrdering.h"
40#include "llvm/Support/Debug.h"
41#include <optional>
42using namespace clang;
43using namespace CodeGen;
44using namespace llvm::omp;
45
46#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
47
48static const VarDecl *getBaseDecl(const Expr *Ref);
49static OpenMPDirectiveKind
50getEffectiveDirectiveKind(const OMPExecutableDirective &S);
51
52/// Whether a combined `distribute parallel for` may use the fused
53/// distr_static_chunk + static_chunkone schedule (enum 93): one
54/// for_static_init, no surrounding distribute_static_init.
55static bool canEmitGPUFusedDistSchedule(const CodeGenModule &CGM,
56 const OMPLoopDirective &S,
57 OpenMPDirectiveKind DKind) {
58 // 'teams loop' is always emitted as 'distribute', and 'target teams loop'
59 // only becomes 'distribute parallel for' if canBeParallelFor() holds.
60 // Without the inner worksharing loop, there is nothing that would schedule
61 // the iteration space if the outer distribute loop is omitted.
62 if (DKind == OMPD_teams_loop)
63 return false;
64 if (const auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(Val: &S);
65 TTLD && !TTLD->canBeParallelFor())
66 return false;
67 // Reduction-only for now. Non-reduction cases might follow in the future, but
68 // need more analysis for maximum profit.
69 return CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
70 isOpenMPLoopBoundSharingDirective(Kind: DKind) &&
71 S.hasClausesOfKind<OMPReductionClause>() &&
72 !S.getSingleClause<OMPDistScheduleClause>() &&
73 !S.getSingleClause<OMPScheduleClause>() &&
74 !S.getSingleClause<OMPOrderedClause>();
75}
76
77namespace {
78/// Lexical scope for OpenMP executable constructs, that handles correct codegen
79/// for captured expressions.
80class OMPLexicalScope : public CodeGenFunction::LexicalScope {
81 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
82 for (const auto *C : S.clauses()) {
83 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
84 if (const auto *PreInit =
85 cast_or_null<DeclStmt>(Val: CPI->getPreInitStmt())) {
86 for (const auto *I : PreInit->decls()) {
87 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
88 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
89 } else {
90 CodeGenFunction::AutoVarEmission Emission =
91 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
92 CGF.EmitAutoVarCleanups(emission: Emission);
93 }
94 }
95 }
96 }
97 }
98 }
99 CodeGenFunction::OMPPrivateScope InlinedShareds;
100
101 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
102 return CGF.LambdaCaptureFields.lookup(Val: VD) ||
103 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
104 (isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl) &&
105 cast<BlockDecl>(Val: CGF.CurCodeDecl)->capturesVariable(var: VD));
106 }
107
108public:
109 OMPLexicalScope(
110 CodeGenFunction &CGF, const OMPExecutableDirective &S,
111 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
112 const bool EmitPreInitStmt = true)
113 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
114 InlinedShareds(CGF) {
115 if (EmitPreInitStmt)
116 emitPreInitStmt(CGF, S);
117 if (!CapturedRegion)
118 return;
119 assert(S.hasAssociatedStmt() &&
120 "Expected associated statement for inlined directive.");
121 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: *CapturedRegion);
122 for (const auto &C : CS->captures()) {
123 if (C.capturesVariable() || C.capturesVariableByCopy()) {
124 auto *VD = C.getCapturedVar();
125 assert(VD == VD->getCanonicalDecl() &&
126 "Canonical decl must be captured.");
127 DeclRefExpr DRE(
128 CGF.getContext(), const_cast<VarDecl *>(VD),
129 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
130 InlinedShareds.isGlobalVarCaptured(VD)),
131 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
132 InlinedShareds.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
133 }
134 }
135 (void)InlinedShareds.Privatize();
136 }
137};
138
139/// Lexical scope for OpenMP parallel construct, that handles correct codegen
140/// for captured expressions.
141class OMPParallelScope final : public OMPLexicalScope {
142 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
143 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
144 return !(isOpenMPTargetExecutionDirective(DKind: EKind) ||
145 isOpenMPLoopBoundSharingDirective(Kind: EKind)) &&
146 isOpenMPParallelDirective(DKind: EKind);
147 }
148
149public:
150 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
151 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
152 EmitPreInitStmt(S)) {}
153};
154
155/// Lexical scope for OpenMP teams construct, that handles correct codegen
156/// for captured expressions.
157class OMPTeamsScope final : public OMPLexicalScope {
158 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
159 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
160 return !isOpenMPTargetExecutionDirective(DKind: EKind) &&
161 isOpenMPTeamsDirective(DKind: EKind);
162 }
163
164public:
165 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
166 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
167 EmitPreInitStmt(S)) {}
168};
169
170/// Private scope for OpenMP loop-based directives, that supports capturing
171/// of used expression from loop statement.
172class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
173 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
174 const Stmt *PreInits;
175 CodeGenFunction::OMPMapVars PreCondVars;
176 if (auto *LD = dyn_cast<OMPLoopDirective>(Val: &S)) {
177 // Emit init, __range, __begin and __end variables for C++ range loops.
178 (void)OMPLoopBasedDirective::doForAllLoops(
179 CurStmt: LD->getInnermostCapturedStmt()->getCapturedStmt(),
180 /*TryImperfectlyNestedLoops=*/true, NumLoops: LD->getLoopsNumber(),
181 Callback: [&CGF](unsigned Cnt, const Stmt *CurStmt) {
182 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(Val: CurStmt)) {
183 if (const Stmt *Init = CXXFor->getInit())
184 CGF.EmitStmt(S: Init);
185 CGF.EmitStmt(S: CXXFor->getRangeStmt());
186 CGF.EmitStmt(S: CXXFor->getBeginStmt());
187 CGF.EmitStmt(S: CXXFor->getEndStmt());
188 }
189 return false;
190 });
191 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
192 for (const auto *E : LD->counters()) {
193 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
194 EmittedAsPrivate.insert(V: VD->getCanonicalDecl());
195 (void)PreCondVars.setVarAddr(
196 CGF, LocalVD: VD, TempAddr: CGF.CreateMemTemp(T: VD->getType().getNonReferenceType()));
197 }
198 // Mark private vars as undefs.
199 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
200 for (const Expr *IRef : C->varlist()) {
201 const auto *OrigDecl = cast<DeclRefExpr>(Val: IRef)->getDecl();
202 const auto *OrigVD = dyn_cast<VarDecl>(Val: OrigDecl);
203 if (!OrigVD)
204 continue;
205 if (EmittedAsPrivate.insert(V: OrigVD->getCanonicalDecl()).second) {
206 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
207 (void)PreCondVars.setVarAddr(
208 CGF, LocalVD: OrigVD,
209 TempAddr: Address(llvm::UndefValue::get(T: CGF.ConvertTypeForMem(
210 T: CGF.getContext().getPointerType(T: OrigVDTy))),
211 CGF.ConvertTypeForMem(T: OrigVDTy),
212 CGF.getContext().getDeclAlign(D: OrigVD)));
213 }
214 }
215 }
216 (void)PreCondVars.apply(CGF);
217 PreInits = LD->getPreInits();
218 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(Val: &S)) {
219 PreInits = Tile->getPreInits();
220 } else if (const auto *Stripe = dyn_cast<OMPStripeDirective>(Val: &S)) {
221 PreInits = Stripe->getPreInits();
222 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(Val: &S)) {
223 PreInits = Unroll->getPreInits();
224 } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(Val: &S)) {
225 PreInits = Reverse->getPreInits();
226 } else if (const auto *Split = dyn_cast<OMPSplitDirective>(Val: &S)) {
227 PreInits = Split->getPreInits();
228 } else if (const auto *Interchange =
229 dyn_cast<OMPInterchangeDirective>(Val: &S)) {
230 PreInits = Interchange->getPreInits();
231 } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(Val: &S)) {
232 PreInits = Flatten->getPreInits();
233 } else {
234 llvm_unreachable("Unknown loop-based directive kind.");
235 }
236 doEmitPreinits(PreInits);
237 PreCondVars.restore(CGF);
238 }
239
240 void
241 emitPreInitStmt(CodeGenFunction &CGF,
242 const OMPCanonicalLoopSequenceTransformationDirective &S) {
243 const Stmt *PreInits;
244 if (const auto *Fuse = dyn_cast<OMPFuseDirective>(Val: &S)) {
245 PreInits = Fuse->getPreInits();
246 } else {
247 llvm_unreachable(
248 "Unknown canonical loop sequence transform directive kind.");
249 }
250 doEmitPreinits(PreInits);
251 }
252
253 void doEmitPreinits(const Stmt *PreInits) {
254 if (PreInits) {
255 // CompoundStmts and DeclStmts are used as lists of PreInit statements and
256 // declarations. Since declarations must be visible in the the following
257 // that they initialize, unpack the CompoundStmt they are nested in.
258 SmallVector<const Stmt *> PreInitStmts;
259 if (auto *PreInitCompound = dyn_cast<CompoundStmt>(Val: PreInits))
260 llvm::append_range(C&: PreInitStmts, R: PreInitCompound->body());
261 else
262 PreInitStmts.push_back(Elt: PreInits);
263
264 for (const Stmt *S : PreInitStmts) {
265 // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted
266 // here.
267 if (auto *PreInitDecl = dyn_cast<DeclStmt>(Val: S)) {
268 for (Decl *I : PreInitDecl->decls())
269 CGF.EmitVarDecl(D: cast<VarDecl>(Val&: *I));
270 continue;
271 }
272 CGF.EmitStmt(S);
273 }
274 }
275 }
276
277public:
278 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
279 : CodeGenFunction::RunCleanupsScope(CGF) {
280 emitPreInitStmt(CGF, S);
281 }
282 OMPLoopScope(CodeGenFunction &CGF,
283 const OMPCanonicalLoopSequenceTransformationDirective &S)
284 : CodeGenFunction::RunCleanupsScope(CGF) {
285 emitPreInitStmt(CGF, S);
286 }
287};
288
289class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
290 CodeGenFunction::OMPPrivateScope InlinedShareds;
291
292 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
293 return CGF.LambdaCaptureFields.lookup(Val: VD) ||
294 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
295 (isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl) &&
296 cast<BlockDecl>(Val: CGF.CurCodeDecl)->capturesVariable(var: VD));
297 }
298
299public:
300 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
301 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
302 InlinedShareds(CGF) {
303 for (const auto *C : S.clauses()) {
304 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
305 if (const auto *PreInit =
306 cast_or_null<DeclStmt>(Val: CPI->getPreInitStmt())) {
307 for (const auto *I : PreInit->decls()) {
308 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
309 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
310 } else {
311 CodeGenFunction::AutoVarEmission Emission =
312 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
313 CGF.EmitAutoVarCleanups(emission: Emission);
314 }
315 }
316 }
317 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(Val: C)) {
318 for (const Expr *E : UDP->varlist()) {
319 const Decl *D = cast<DeclRefExpr>(Val: E)->getDecl();
320 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
321 CGF.EmitVarDecl(D: *OED);
322 }
323 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(Val: C)) {
324 for (const Expr *E : UDP->varlist()) {
325 const Decl *D = getBaseDecl(Ref: E);
326 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
327 CGF.EmitVarDecl(D: *OED);
328 }
329 }
330 }
331 if (!isOpenMPSimdDirective(DKind: getEffectiveDirectiveKind(S)))
332 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: InlinedShareds);
333 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(Val: &S)) {
334 if (const Expr *E = TG->getReductionRef())
335 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl()));
336 }
337 // Temp copy arrays for inscan reductions should not be emitted as they are
338 // not used in simd only mode.
339 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
340 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
341 if (C->getModifier() != OMPC_REDUCTION_inscan)
342 continue;
343 for (const Expr *E : C->copy_array_temps())
344 CopyArrayTemps.insert(V: cast<DeclRefExpr>(Val: E)->getDecl());
345 }
346 const auto *CS = cast_or_null<CapturedStmt>(Val: S.getAssociatedStmt());
347 while (CS) {
348 for (auto &C : CS->captures()) {
349 if (C.capturesVariable() || C.capturesVariableByCopy()) {
350 auto *VD = C.getCapturedVar();
351 if (CopyArrayTemps.contains(V: VD))
352 continue;
353 assert(VD == VD->getCanonicalDecl() &&
354 "Canonical decl must be captured.");
355 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
356 isCapturedVar(CGF, VD) ||
357 (CGF.CapturedStmtInfo &&
358 InlinedShareds.isGlobalVarCaptured(VD)),
359 VD->getType().getNonReferenceType(), VK_LValue,
360 C.getLocation());
361 InlinedShareds.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
362 }
363 }
364 CS = dyn_cast<CapturedStmt>(Val: CS->getCapturedStmt());
365 }
366 (void)InlinedShareds.Privatize();
367 }
368};
369
370} // namespace
371
372// Map `loop` with a bind clause, `teams loop`, and `target teams loop` to the
373// construct CodeGen emits.
374static OpenMPDirectiveKind
375getEffectiveDirectiveKind(const OMPExecutableDirective &S) {
376 OpenMPDirectiveKind Kind = S.getDirectiveKind();
377
378 if (Kind == OMPD_teams_loop)
379 return OMPD_teams_distribute;
380 if (Kind == OMPD_target_teams_loop)
381 return cast<OMPTargetTeamsGenericLoopDirective>(Val: S).canBeParallelFor()
382 ? OMPD_target_teams_distribute_parallel_for
383 : OMPD_target_teams_distribute;
384
385 if (Kind != OMPD_loop)
386 return Kind;
387
388 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;
389 if (const auto *C = S.getSingleClause<OMPBindClause>())
390 BindKind = C->getBindKind();
391
392 switch (BindKind) {
393 case OMPC_BIND_parallel:
394 return OMPD_for;
395 case OMPC_BIND_teams:
396 return OMPD_distribute;
397 case OMPC_BIND_thread:
398 return OMPD_simd;
399 default:
400 return OMPD_loop;
401 }
402}
403
404static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
405 const OMPExecutableDirective &S,
406 const RegionCodeGenTy &CodeGen);
407
408Address CodeGenFunction::EmitOMPBindingOriginalAddr(const BindingDecl *BD,
409 SourceLocation Loc) {
410 if (CapturedStmtInfo &&
411 CapturedStmtInfo->getKind() == CapturedRegionKind::CR_OpenMP) {
412 if (const auto *DD = dyn_cast<VarDecl>(Val: BD->getDecomposedDecl())) {
413 if (CapturedStmtInfo->lookup(VD: DD))
414 return EmitOMPCapturedBindingLValue(BD).getAddress();
415 }
416 }
417 DeclRefExpr DRE(getContext(), const_cast<BindingDecl *>(BD),
418 /*RefersToEnclosingVariableOrCapture=*/false, BD->getType(),
419 VK_LValue, Loc);
420 return EmitLValue(E: &DRE).getAddress();
421}
422
423LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
424 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(Val: E)) {
425 if (const auto *OrigVD = dyn_cast<VarDecl>(Val: OrigDRE->getDecl())) {
426 OrigVD = OrigVD->getCanonicalDecl();
427 bool IsCaptured =
428 LambdaCaptureFields.lookup(Val: OrigVD) ||
429 (CapturedStmtInfo && CapturedStmtInfo->lookup(VD: OrigVD)) ||
430 (isa_and_nonnull<BlockDecl>(Val: CurCodeDecl));
431 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
432 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
433 return EmitLValue(E: &DRE);
434 }
435 if (const auto *OrigBD = dyn_cast<BindingDecl>(Val: OrigDRE->getDecl())) {
436 OrigBD = cast<BindingDecl>(Val: OrigBD->getCanonicalDecl());
437 const auto *DD = cast<VarDecl>(Val: OrigBD->getDecomposedDecl());
438 bool IsCaptured = LambdaCaptureFields.lookup(Val: OrigBD) ||
439 (CapturedStmtInfo && CapturedStmtInfo->lookup(VD: DD)) ||
440 isa_and_nonnull<BlockDecl>(Val: CurCodeDecl);
441 DeclRefExpr DRE(getContext(), const_cast<BindingDecl *>(OrigBD),
442 IsCaptured, OrigDRE->getType(), VK_LValue,
443 OrigDRE->getExprLoc());
444 return EmitLValue(E: &DRE);
445 }
446 }
447 return EmitLValue(E);
448}
449
450llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
451 ASTContext &C = getContext();
452 llvm::Value *Size = nullptr;
453 auto SizeInChars = C.getTypeSizeInChars(T: Ty);
454 if (SizeInChars.isZero()) {
455 // getTypeSizeInChars() returns 0 for a VLA.
456 while (const VariableArrayType *VAT = C.getAsVariableArrayType(T: Ty)) {
457 VlaSizePair VlaSize = getVLASize(vla: VAT);
458 Ty = VlaSize.Type;
459 Size =
460 Size ? Builder.CreateNUWMul(LHS: Size, RHS: VlaSize.NumElts) : VlaSize.NumElts;
461 }
462 SizeInChars = C.getTypeSizeInChars(T: Ty);
463 if (SizeInChars.isZero())
464 return llvm::ConstantInt::get(Ty: SizeTy, /*V=*/0);
465 return Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: SizeInChars));
466 }
467 return CGM.getSize(numChars: SizeInChars);
468}
469
470void CodeGenFunction::GenerateOpenMPCapturedVars(
471 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
472 const RecordDecl *RD = S.getCapturedRecordDecl();
473 auto CurField = RD->field_begin();
474 auto CurCap = S.captures().begin();
475 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
476 E = S.capture_init_end();
477 I != E; ++I, ++CurField, ++CurCap) {
478 if (CurField->hasCapturedVLAType()) {
479 const VariableArrayType *VAT = CurField->getCapturedVLAType();
480 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
481 CapturedVars.push_back(Elt: Val);
482 } else if (CurCap->capturesThis()) {
483 CapturedVars.push_back(Elt: CXXThisValue);
484 } else if (CurCap->capturesVariableByCopy()) {
485 llvm::Value *CV = EmitLoadOfScalar(lvalue: EmitLValue(E: *I), Loc: CurCap->getLocation());
486
487 // If the field is not a pointer, we need to save the actual value
488 // and load it as a void pointer.
489 if (!CurField->getType()->isAnyPointerType()) {
490 ASTContext &Ctx = getContext();
491 Address DstAddr = CreateMemTempWithoutCast(
492 T: Ctx.getUIntPtrType(),
493 Name: Twine(CurCap->getCapturedVar()->getName(), ".casted"));
494 LValue DstLV = MakeAddrLValue(Addr: DstAddr, T: Ctx.getUIntPtrType());
495
496 llvm::Value *SrcAddrVal = EmitScalarConversion(
497 Src: DstAddr.emitRawPointer(CGF&: *this),
498 SrcTy: Ctx.getPointerType(T: Ctx.getUIntPtrType()),
499 DstTy: Ctx.getPointerType(T: CurField->getType()), Loc: CurCap->getLocation());
500 LValue SrcLV =
501 MakeNaturalAlignAddrLValue(V: SrcAddrVal, T: CurField->getType());
502
503 // Store the value using the source type pointer.
504 EmitStoreThroughLValue(Src: RValue::get(V: CV), Dst: SrcLV);
505
506 // Load the value using the destination type pointer.
507 CV = EmitLoadOfScalar(lvalue: DstLV, Loc: CurCap->getLocation());
508 }
509 CapturedVars.push_back(Elt: CV);
510 } else {
511 assert(CurCap->capturesVariable() && "Expected capture by reference.");
512 llvm::Value *Addr = EmitLValue(E: *I).getAddress().emitRawPointer(CGF&: *this);
513 // Sema strips the address space from the type of the captured field.
514 llvm::Type *ArgTy = ConvertType(T: CurField->getType());
515 if (Addr->getType() != ArgTy)
516 Addr = performAddrSpaceCast(Src: Addr, DestTy: ArgTy);
517 CapturedVars.push_back(Elt: Addr);
518 }
519 }
520}
521
522static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
523 QualType DstType, StringRef Name,
524 LValue AddrLV) {
525 ASTContext &Ctx = CGF.getContext();
526
527 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
528 Src: AddrLV.getAddress().emitRawPointer(CGF), SrcTy: Ctx.getUIntPtrType(),
529 DstTy: Ctx.getPointerType(T: DstType), Loc);
530 // FIXME: should the pointee type (DstType) be passed?
531 Address TmpAddr =
532 CGF.MakeNaturalAlignAddrLValue(V: CastedPtr, T: DstType).getAddress();
533 return TmpAddr;
534}
535
536static QualType getCanonicalParamType(ASTContext &C, QualType T) {
537 if (T->isLValueReferenceType())
538 return C.getLValueReferenceType(
539 T: getCanonicalParamType(C, T: T.getNonReferenceType()),
540 /*SpelledAsLValue=*/false);
541 if (T->isPointerType())
542 return C.getPointerType(T: getCanonicalParamType(C, T: T->getPointeeType()));
543 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
544 if (const auto *VLA = dyn_cast<VariableArrayType>(Val: A))
545 return getCanonicalParamType(C, T: VLA->getElementType());
546 if (!A->isVariablyModifiedType())
547 return C.getCanonicalType(T);
548 }
549 return C.getCanonicalParamType(T);
550}
551
552namespace {
553/// Contains required data for proper outlined function codegen.
554struct FunctionOptions {
555 /// Captured statement for which the function is generated.
556 const CapturedStmt *S = nullptr;
557 /// true if cast to/from UIntPtr is required for variables captured by
558 /// value.
559 const bool UIntPtrCastRequired = true;
560 /// true if only casted arguments must be registered as local args or VLA
561 /// sizes.
562 const bool RegisterCastedArgsOnly = false;
563 /// Name of the generated function.
564 const StringRef FunctionName;
565 /// Location of the non-debug version of the outlined function.
566 SourceLocation Loc;
567 const bool IsDeviceKernel = false;
568 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
569 bool RegisterCastedArgsOnly, StringRef FunctionName,
570 SourceLocation Loc, bool IsDeviceKernel)
571 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
572 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
573 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
574};
575} // namespace
576
577static llvm::Function *emitOutlinedFunctionPrologue(
578 CodeGenFunction &CGF, FunctionArgList &Args,
579 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
580 &LocalAddrs,
581 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
582 &VLASizes,
583 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
584 const CapturedDecl *CD = FO.S->getCapturedDecl();
585 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
586 assert(CD->hasBody() && "missing CapturedDecl body");
587
588 CXXThisValue = nullptr;
589 // Build the argument list.
590 CodeGenModule &CGM = CGF.CGM;
591 ASTContext &Ctx = CGM.getContext();
592 FunctionArgList TargetArgs;
593 Args.append(in_start: CD->param_begin(),
594 in_end: std::next(x: CD->param_begin(), n: CD->getContextParamPosition()));
595 TargetArgs.append(
596 in_start: CD->param_begin(),
597 in_end: std::next(x: CD->param_begin(), n: CD->getContextParamPosition()));
598 auto I = FO.S->captures().begin();
599 FunctionDecl *DebugFunctionDecl = nullptr;
600 if (!FO.UIntPtrCastRequired) {
601 FunctionProtoType::ExtProtoInfo EPI;
602 QualType FunctionTy = Ctx.getFunctionType(ResultTy: Ctx.VoidTy, Args: {}, EPI);
603 DebugFunctionDecl = FunctionDecl::Create(
604 C&: Ctx, DC: Ctx.getTranslationUnitDecl(), StartLoc: FO.S->getBeginLoc(),
605 NLoc: SourceLocation(), N: DeclarationName(), T: FunctionTy,
606 TInfo: Ctx.getTrivialTypeSourceInfo(T: FunctionTy), SC: SC_Static,
607 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
608 /*hasWrittenPrototype=*/false);
609 }
610 for (const FieldDecl *FD : RD->fields()) {
611 QualType ArgType = FD->getType();
612 IdentifierInfo *II = nullptr;
613 VarDecl *CapVar = nullptr;
614
615 // If this is a capture by copy and the type is not a pointer, the outlined
616 // function argument type should be uintptr and the value properly casted to
617 // uintptr. This is necessary given that the runtime library is only able to
618 // deal with pointers. We can pass in the same way the VLA type sizes to the
619 // outlined function.
620 if (FO.UIntPtrCastRequired &&
621 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
622 I->capturesVariableArrayType()))
623 ArgType = Ctx.getUIntPtrType();
624
625 if (I->capturesVariable() || I->capturesVariableByCopy()) {
626 CapVar = I->getCapturedVar();
627 II = CapVar->getIdentifier();
628 } else if (I->capturesThis()) {
629 II = &Ctx.Idents.get(Name: "this");
630 } else {
631 assert(I->capturesVariableArrayType());
632 II = &Ctx.Idents.get(Name: "vla");
633 }
634 if (ArgType->isVariablyModifiedType())
635 ArgType = getCanonicalParamType(C&: Ctx, T: ArgType);
636 VarDecl *Arg;
637 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
638 Arg = ImplicitParamDecl::Create(C&: Ctx, /*DC=*/nullptr, IdLoc: FD->getLocation(),
639 Id: II, T: ArgType,
640 ParamKind: ImplicitParamKind::ThreadPrivateVar);
641 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
642 Arg = ParmVarDecl::Create(
643 C&: Ctx, DC: DebugFunctionDecl,
644 StartLoc: CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
645 IdLoc: CapVar ? CapVar->getLocation() : FD->getLocation(), Id: II, T: ArgType,
646 /*TInfo=*/nullptr, S: SC_None, /*DefArg=*/nullptr);
647 } else {
648 Arg = ImplicitParamDecl::Create(C&: Ctx, /*DC=*/nullptr, IdLoc: FD->getLocation(),
649 Id: II, T: ArgType, ParamKind: ImplicitParamKind::Other);
650 }
651 Args.emplace_back(Args&: Arg);
652 // Do not cast arguments if we emit function with non-original types.
653 TargetArgs.emplace_back(
654 Args: FO.UIntPtrCastRequired
655 ? Arg
656 : CGM.getOpenMPRuntime().translateParameter(FD, NativeParam: Arg));
657 ++I;
658 }
659 Args.append(in_start: std::next(x: CD->param_begin(), n: CD->getContextParamPosition() + 1),
660 in_end: CD->param_end());
661 TargetArgs.append(
662 in_start: std::next(x: CD->param_begin(), n: CD->getContextParamPosition() + 1),
663 in_end: CD->param_end());
664
665 // Create the function declaration.
666 const CGFunctionInfo &FuncInfo =
667 FO.IsDeviceKernel
668 ? CGM.getTypes().arrangeDeviceKernelCallerDeclaration(resultType: Ctx.VoidTy,
669 args: TargetArgs)
670 : CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy,
671 args: TargetArgs);
672 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
673
674 auto *F =
675 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
676 N: FO.FunctionName, M: &CGM.getModule());
677 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
678
679 // Adjust the calling convention for SPIR-V targets to avoid mismatches
680 // between callee and caller.
681 if (CGM.getTriple().isSPIRV() && !FO.IsDeviceKernel)
682 F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
683
684 if (CD->isNothrow())
685 F->setDoesNotThrow();
686
687 // Always inline the outlined function if optimizations are enabled.
688 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
689 F->removeFnAttr(Kind: llvm::Attribute::NoInline);
690 F->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
691 }
692 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
693 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
694
695 // Generate the function.
696 CGF.StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args: TargetArgs,
697 Loc: FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
698 StartLoc: FO.UIntPtrCastRequired ? FO.Loc
699 : CD->getBody()->getBeginLoc());
700 unsigned Cnt = CD->getContextParamPosition();
701 I = FO.S->captures().begin();
702 for (const FieldDecl *FD : RD->fields()) {
703 // Do not map arguments if we emit function with non-original types.
704 Address LocalAddr(Address::invalid());
705 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
706 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, NativeParam: Args[Cnt],
707 TargetParam: TargetArgs[Cnt]);
708 } else {
709 LocalAddr = CGF.GetAddrOfLocalVar(VD: Args[Cnt]);
710 }
711 // If we are capturing a pointer by copy we don't need to do anything, just
712 // use the value that we get from the arguments.
713 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
714 const VarDecl *CurVD = I->getCapturedVar();
715 if (!FO.RegisterCastedArgsOnly)
716 LocalAddrs.insert(KV: {Args[Cnt], {CurVD, LocalAddr}});
717 ++Cnt;
718 ++I;
719 continue;
720 }
721
722 LValue ArgLVal = CGF.MakeAddrLValue(Addr: LocalAddr, T: Args[Cnt]->getType(),
723 Source: AlignmentSource::Decl);
724 if (FD->hasCapturedVLAType()) {
725 if (FO.UIntPtrCastRequired) {
726 ArgLVal = CGF.MakeAddrLValue(
727 Addr: castValueFromUintptr(CGF, Loc: I->getLocation(), DstType: FD->getType(),
728 Name: Args[Cnt]->getName(), AddrLV: ArgLVal),
729 T: FD->getType(), Source: AlignmentSource::Decl);
730 }
731 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: I->getLocation());
732 const VariableArrayType *VAT = FD->getCapturedVLAType();
733 VLASizes.try_emplace(Key: Args[Cnt], Args: VAT->getSizeExpr(), Args&: ExprArg);
734 } else if (I->capturesVariable()) {
735 const VarDecl *Var = I->getCapturedVar();
736 QualType VarTy = Var->getType();
737 Address ArgAddr = ArgLVal.getAddress();
738 if (ArgLVal.getType()->isLValueReferenceType()) {
739 ArgAddr = CGF.EmitLoadOfReference(RefLVal: ArgLVal);
740 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
741 assert(ArgLVal.getType()->isPointerType());
742 ArgAddr = CGF.EmitLoadOfPointer(
743 Ptr: ArgAddr, PtrTy: ArgLVal.getType()->castAs<PointerType>());
744 }
745 if (!FO.RegisterCastedArgsOnly) {
746 LocalAddrs.insert(
747 KV: {Args[Cnt], {Var, ArgAddr.withAlignment(NewAlignment: Ctx.getDeclAlign(D: Var))}});
748 }
749 } else if (I->capturesVariableByCopy()) {
750 assert(!FD->getType()->isAnyPointerType() &&
751 "Not expecting a captured pointer.");
752 const VarDecl *Var = I->getCapturedVar();
753 LocalAddrs.insert(KV: {Args[Cnt],
754 {Var, FO.UIntPtrCastRequired
755 ? castValueFromUintptr(
756 CGF, Loc: I->getLocation(), DstType: FD->getType(),
757 Name: Args[Cnt]->getName(), AddrLV: ArgLVal)
758 : ArgLVal.getAddress()}});
759 } else {
760 // If 'this' is captured, load it into CXXThisValue.
761 assert(I->capturesThis());
762 CXXThisValue = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: I->getLocation());
763 LocalAddrs.insert(KV: {Args[Cnt], {nullptr, ArgLVal.getAddress()}});
764 }
765 ++Cnt;
766 ++I;
767 }
768
769 return F;
770}
771
772static llvm::Function *emitOutlinedFunctionPrologueAggregate(
773 CodeGenFunction &CGF, FunctionArgList &Args,
774 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
775 &LocalAddrs,
776 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
777 &VLASizes,
778 llvm::Value *&CXXThisValue, llvm::Value *&ContextV, const CapturedStmt &CS,
779 SourceLocation Loc, StringRef FunctionName) {
780 const CapturedDecl *CD = CS.getCapturedDecl();
781 const RecordDecl *RD = CS.getCapturedRecordDecl();
782
783 CXXThisValue = nullptr;
784 CodeGenModule &CGM = CGF.CGM;
785 ASTContext &Ctx = CGM.getContext();
786 Args.push_back(Elt: CD->getContextParam());
787
788 const CGFunctionInfo &FuncInfo =
789 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
790 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
791
792 auto *F =
793 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
794 N: FunctionName, M: &CGM.getModule());
795 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
796 if (CD->isNothrow())
797 F->setDoesNotThrow();
798
799 CGF.StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc, StartLoc: Loc);
800 Address ContextAddr = CGF.GetAddrOfLocalVar(VD: CD->getContextParam());
801 ContextV = CGF.Builder.CreateLoad(Addr: ContextAddr);
802
803 // The runtime passes arguments as an array of pointers.
804 llvm::Type *PtrTy = CGF.Builder.getPtrTy();
805 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
806 CharUnits SlotAlign = CharUnits::fromQuantity(Quantity: PtrAlign.value());
807
808 for (auto [FD, C, FieldIdx] :
809 llvm::zip(t: RD->fields(), u: CS.captures(),
810 args: llvm::seq<unsigned>(Size: RD->getNumFields()))) {
811 llvm::Value *SlotPtr =
812 CGF.Builder.CreateConstInBoundsGEP1_32(Ty: PtrTy, Ptr: ContextV, Idx0: FieldIdx);
813 llvm::Value *Slot = CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Ptr: SlotPtr, Align: PtrAlign);
814
815 // Generate the appropriate load from the per-argument storage. This
816 // includes all of the user arguments as well as the implicit kernel
817 // argument pointer.
818 if (C.capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
819 const VarDecl *CurVD = C.getCapturedVar();
820 Slot->setName(CurVD->getName());
821 Address SlotAddr(Slot, PtrTy, SlotAlign);
822 LocalAddrs.insert(KV: {FD, {CurVD, SlotAddr}});
823 } else if (FD->hasCapturedVLAType()) {
824 // VLA size is stored as intptr_t directly in the slot.
825 Address SlotAddr(Slot, CGF.ConvertTypeForMem(T: FD->getType()), SlotAlign);
826 LValue ArgLVal =
827 CGF.MakeAddrLValue(Addr: SlotAddr, T: FD->getType(), Source: AlignmentSource::Decl);
828 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: C.getLocation());
829 const VariableArrayType *VAT = FD->getCapturedVLAType();
830 VLASizes.try_emplace(Key: FD, Args: VAT->getSizeExpr(), Args&: ExprArg);
831 } else if (C.capturesVariable()) {
832 const VarDecl *Var = C.getCapturedVar();
833 QualType VarTy = Var->getType();
834
835 if (VarTy->isVariablyModifiedType() && VarTy->isPointerType()) {
836 Slot->setName(Var->getName() + ".addr");
837 Address SlotAddr(Slot, PtrTy, SlotAlign);
838 LocalAddrs.insert(KV: {FD, {Var, SlotAddr}});
839 } else {
840 llvm::Value *VarAddr = CGF.Builder.CreateAlignedLoad(
841 Ty: PtrTy, Ptr: Slot, Align: PtrAlign, Name: Var->getName());
842 LocalAddrs.insert(KV: {FD,
843 {Var, Address(VarAddr, CGF.ConvertTypeForMem(T: VarTy),
844 Ctx.getDeclAlign(D: Var))}});
845 }
846 } else if (C.capturesVariableByCopy()) {
847 assert(!FD->getType()->isAnyPointerType() &&
848 "Not expecting a captured pointer.");
849 const VarDecl *Var = C.getCapturedVar();
850 QualType FieldTy = FD->getType();
851
852 // Scalar values are promoted and stored directly in the slot.
853 Address SlotAddr(Slot, CGF.ConvertTypeForMem(T: FieldTy), SlotAlign);
854 Address CopyAddr =
855 CGF.CreateMemTemp(T: FieldTy, Align: Ctx.getDeclAlign(D: FD), Name: Var->getName());
856 LValue SrcLVal =
857 CGF.MakeAddrLValue(Addr: SlotAddr, T: FieldTy, Source: AlignmentSource::Decl);
858 LValue CopyLVal =
859 CGF.MakeAddrLValue(Addr: CopyAddr, T: FieldTy, Source: AlignmentSource::Decl);
860
861 RValue ArgRVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc: C.getLocation());
862 CGF.EmitStoreThroughLValue(Src: ArgRVal, Dst: CopyLVal);
863
864 LocalAddrs.insert(KV: {FD, {Var, CopyAddr}});
865 } else {
866 assert(C.capturesThis() && "Default case expected to be CXX 'this'");
867 CXXThisValue =
868 CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Ptr: Slot, Align: PtrAlign, Name: "this");
869 Address SlotAddr(Slot, PtrTy, SlotAlign);
870 LocalAddrs.insert(KV: {FD, {nullptr, SlotAddr}});
871 }
872 }
873
874 return F;
875}
876
877llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunction(
878 const CapturedStmt &S, const OMPExecutableDirective &D) {
879 SourceLocation Loc = D.getBeginLoc();
880 assert(
881 CapturedStmtInfo &&
882 "CapturedStmtInfo should be set when generating the captured function");
883 const CapturedDecl *CD = S.getCapturedDecl();
884 // Build the argument list.
885 bool NeedWrapperFunction =
886 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
887 FunctionArgList Args, WrapperArgs;
888 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
889 WrapperLocalAddrs;
890 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
891 WrapperVLASizes;
892 SmallString<256> Buffer;
893 llvm::raw_svector_ostream Out(Buffer);
894 Out << CapturedStmtInfo->getHelperName();
895 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
896 bool IsDeviceKernel = CGM.getOpenMPRuntime().isGPU() &&
897 isOpenMPTargetExecutionDirective(DKind: EKind) &&
898 D.getCapturedStmt(RegionKind: OMPD_target) == &S;
899 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
900 llvm::Function *WrapperF = nullptr;
901 if (NeedWrapperFunction) {
902 // Emit the final kernel early to allow attributes to be added by the
903 // OpenMPI-IR-Builder.
904 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
905 /*RegisterCastedArgsOnly=*/true,
906 CapturedStmtInfo->getHelperName(), Loc,
907 IsDeviceKernel);
908 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
909 WrapperF =
910 emitOutlinedFunctionPrologue(CGF&: WrapperCGF, Args, LocalAddrs, VLASizes,
911 CXXThisValue&: WrapperCGF.CXXThisValue, FO: WrapperFO);
912 Out << "_debug__";
913 }
914 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
915 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
916 llvm::Function *F = emitOutlinedFunctionPrologue(
917 CGF&: *this, Args&: WrapperArgs, LocalAddrs&: WrapperLocalAddrs, VLASizes&: WrapperVLASizes, CXXThisValue, FO);
918 CodeGenFunction::OMPPrivateScope LocalScope(*this);
919 for (const auto &LocalAddrPair : WrapperLocalAddrs) {
920 if (LocalAddrPair.second.first) {
921 LocalScope.addPrivate(LocalVD: LocalAddrPair.second.first,
922 Addr: LocalAddrPair.second.second);
923 }
924 }
925 (void)LocalScope.Privatize();
926 for (const auto &VLASizePair : WrapperVLASizes)
927 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
928 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
929 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
930 LocalScope.ForceCleanup();
931 FinishFunction(EndLoc: CD->getBodyRBrace());
932 if (!NeedWrapperFunction)
933 return F;
934
935 // Reverse the order.
936 WrapperF->removeFromParent();
937 F->getParent()->getFunctionList().insertAfter(where: F->getIterator(), New: WrapperF);
938
939 llvm::SmallVector<llvm::Value *, 4> CallArgs;
940 auto *PI = F->arg_begin();
941 for (const auto *Arg : Args) {
942 llvm::Value *CallArg;
943 auto I = LocalAddrs.find(Key: Arg);
944 if (I != LocalAddrs.end()) {
945 LValue LV = WrapperCGF.MakeAddrLValue(
946 Addr: I->second.second,
947 T: I->second.first ? I->second.first->getType() : Arg->getType(),
948 Source: AlignmentSource::Decl);
949 if (LV.getType()->isAnyComplexType())
950 LV.setAddress(LV.getAddress().withElementType(ElemTy: PI->getType()));
951 CallArg = WrapperCGF.EmitLoadOfScalar(lvalue: LV, Loc: S.getBeginLoc());
952 } else {
953 auto EI = VLASizes.find(Val: Arg);
954 if (EI != VLASizes.end()) {
955 CallArg = EI->second.second;
956 } else {
957 LValue LV =
958 WrapperCGF.MakeAddrLValue(Addr: WrapperCGF.GetAddrOfLocalVar(VD: Arg),
959 T: Arg->getType(), Source: AlignmentSource::Decl);
960 CallArg = WrapperCGF.EmitLoadOfScalar(lvalue: LV, Loc: S.getBeginLoc());
961 }
962 }
963 CallArgs.emplace_back(Args: WrapperCGF.EmitFromMemory(Value: CallArg, Ty: Arg->getType()));
964 ++PI;
965 }
966 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF&: WrapperCGF, Loc, OutlinedFn: F, Args: CallArgs);
967 WrapperCGF.FinishFunction();
968 return WrapperF;
969}
970
971llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunctionAggregate(
972 const CapturedStmt &S, const OMPExecutableDirective &D) {
973 SourceLocation Loc = D.getBeginLoc();
974 assert(
975 CapturedStmtInfo &&
976 "CapturedStmtInfo should be set when generating the captured function");
977 const CapturedDecl *CD = S.getCapturedDecl();
978 const RecordDecl *RD = S.getCapturedRecordDecl();
979 StringRef FunctionName = CapturedStmtInfo->getHelperName();
980 bool NeedWrapperFunction =
981 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
982
983 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
984 llvm::Function *WrapperF = nullptr;
985 llvm::Value *WrapperContextV = nullptr;
986 if (NeedWrapperFunction) {
987 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
988 FunctionArgList WrapperArgs;
989 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
990 WrapperLocalAddrs;
991 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
992 WrapperVLASizes;
993 WrapperF = emitOutlinedFunctionPrologueAggregate(
994 CGF&: WrapperCGF, Args&: WrapperArgs, LocalAddrs&: WrapperLocalAddrs, VLASizes&: WrapperVLASizes,
995 CXXThisValue&: WrapperCGF.CXXThisValue, ContextV&: WrapperContextV, CS: S, Loc, FunctionName);
996 }
997
998 FunctionArgList Args;
999 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
1000 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
1001 llvm::Function *F;
1002
1003 if (NeedWrapperFunction) {
1004 SmallString<256> Buffer;
1005 llvm::raw_svector_ostream Out(Buffer);
1006 Out << FunctionName << "_debug__";
1007
1008 FunctionOptions FO(&S, /*UIntPtrCastRequired=*/false,
1009 /*RegisterCastedArgsOnly=*/false, Out.str(), Loc,
1010 /*IsDeviceKernel=*/false);
1011 F = emitOutlinedFunctionPrologue(CGF&: *this, Args, LocalAddrs, VLASizes,
1012 CXXThisValue, FO);
1013 } else {
1014 llvm::Value *ContextV = nullptr;
1015 F = emitOutlinedFunctionPrologueAggregate(CGF&: *this, Args, LocalAddrs, VLASizes,
1016 CXXThisValue, ContextV, CS: S, Loc,
1017 FunctionName);
1018
1019 const RecordDecl *RD = S.getCapturedRecordDecl();
1020 unsigned FieldIdx = RD->getNumFields();
1021 for (unsigned I = 0; I < CD->getNumParams(); ++I) {
1022 const ImplicitParamDecl *Param = CD->getParam(i: I);
1023 if (Param == CD->getContextParam())
1024 continue;
1025 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
1026 llvm::Value *SlotPtr = Builder.CreateConstInBoundsGEP1_32(
1027 Ty: Builder.getPtrTy(), Ptr: ContextV, Idx0: FieldIdx,
1028 Name: Twine(Param->getName()) + ".addr");
1029 llvm::Value *ParamAddr =
1030 Builder.CreateAlignedLoad(Ty: Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1031 llvm::Value *ParamVal = Builder.CreateAlignedLoad(
1032 Ty: Builder.getPtrTy(), Ptr: ParamAddr, Align: PtrAlign, Name: Param->getName());
1033 Address ParamLocalAddr =
1034 CreateMemTemp(T: Param->getType(), Name: Param->getName());
1035 Builder.CreateStore(Val: ParamVal, Addr: ParamLocalAddr);
1036 LocalAddrs.insert(KV: {Param, {Param, ParamLocalAddr}});
1037 ++FieldIdx;
1038 }
1039 }
1040
1041 CodeGenFunction::OMPPrivateScope LocalScope(*this);
1042 for (const auto &LocalAddrPair : LocalAddrs) {
1043 if (LocalAddrPair.second.first)
1044 LocalScope.addPrivate(LocalVD: LocalAddrPair.second.first,
1045 Addr: LocalAddrPair.second.second);
1046 }
1047 (void)LocalScope.Privatize();
1048 for (const auto &VLASizePair : VLASizes)
1049 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
1050 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
1051 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
1052 (void)LocalScope.ForceCleanup();
1053 FinishFunction(EndLoc: CD->getBodyRBrace());
1054
1055 if (!NeedWrapperFunction)
1056 return F;
1057
1058 // Reverse the order.
1059 WrapperF->removeFromParent();
1060 F->getParent()->getFunctionList().insertAfter(where: F->getIterator(), New: WrapperF);
1061
1062 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
1063 llvm::SmallVector<llvm::Value *, 16> CallArgs;
1064 assert(CD->getContextParamPosition() == 0 &&
1065 "Expected context param at position 0 for target regions");
1066 assert(RD->getNumFields() + 1 == F->getNumOperands() &&
1067 "Argument count mismatch");
1068
1069 for (auto [FD, InnerParam, SlotIdx] : llvm::zip(
1070 t: RD->fields(), u: F->args(), args: llvm::seq<unsigned>(Size: RD->getNumFields()))) {
1071 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1072 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: WrapperContextV, Idx0: SlotIdx);
1073 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1074 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1075 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1076 Ty: InnerParam.getType(), Ptr: Slot, Align: PtrAlign, Name: InnerParam.getName());
1077 CallArgs.push_back(Elt: Val);
1078 }
1079
1080 // Handle the load from the implicit dyn_ptr at the end of the __context.
1081 unsigned SlotIdx = RD->getNumFields();
1082 auto InnerParam = F->arg_begin() + SlotIdx;
1083 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1084 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: WrapperContextV, Idx0: SlotIdx);
1085 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1086 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1087 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1088 Ty: InnerParam->getType(), Ptr: Slot, Align: PtrAlign, Name: InnerParam->getName());
1089 CallArgs.push_back(Elt: Val);
1090
1091 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF&: WrapperCGF, Loc, OutlinedFn: F, Args: CallArgs);
1092 WrapperCGF.FinishFunction();
1093 return WrapperF;
1094}
1095
1096//===----------------------------------------------------------------------===//
1097// OpenMP Directive Emission
1098//===----------------------------------------------------------------------===//
1099void CodeGenFunction::EmitOMPAggregateAssign(
1100 Address DestAddr, Address SrcAddr, QualType OriginalType,
1101 const llvm::function_ref<void(Address, Address)> CopyGen) {
1102 // Perform element-by-element initialization.
1103 QualType ElementTy;
1104
1105 // Drill down to the base element type on both arrays.
1106 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
1107 llvm::Value *NumElements = emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
1108 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
1109
1110 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(CGF&: *this);
1111 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF&: *this);
1112 // Cast from pointer to array type to pointer to single element.
1113 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(Ty: DestAddr.getElementType(),
1114 Ptr: DestBegin, IdxList: NumElements);
1115
1116 // The basic structure here is a while-do loop.
1117 llvm::BasicBlock *BodyBB = createBasicBlock(name: "omp.arraycpy.body");
1118 llvm::BasicBlock *DoneBB = createBasicBlock(name: "omp.arraycpy.done");
1119 llvm::Value *IsEmpty =
1120 Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arraycpy.isempty");
1121 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
1122
1123 // Enter the loop body, making that address the current address.
1124 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
1125 EmitBlock(BB: BodyBB);
1126
1127 CharUnits ElementSize = getContext().getTypeSizeInChars(T: ElementTy);
1128
1129 llvm::PHINode *SrcElementPHI =
1130 Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
1131 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
1132 Address SrcElementCurrent =
1133 Address(SrcElementPHI, SrcAddr.getElementType(),
1134 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
1135
1136 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
1137 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
1138 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
1139 Address DestElementCurrent =
1140 Address(DestElementPHI, DestAddr.getElementType(),
1141 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
1142
1143 // Emit copy.
1144 CopyGen(DestElementCurrent, SrcElementCurrent);
1145
1146 // Shift the address forward by one element.
1147 llvm::Value *DestElementNext =
1148 Builder.CreateConstGEP1_32(Ty: DestAddr.getElementType(), Ptr: DestElementPHI,
1149 /*Idx0=*/1, Name: "omp.arraycpy.dest.element");
1150 llvm::Value *SrcElementNext =
1151 Builder.CreateConstGEP1_32(Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI,
1152 /*Idx0=*/1, Name: "omp.arraycpy.src.element");
1153 // Check whether we've reached the end.
1154 llvm::Value *Done =
1155 Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
1156 Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
1157 DestElementPHI->addIncoming(V: DestElementNext, BB: Builder.GetInsertBlock());
1158 SrcElementPHI->addIncoming(V: SrcElementNext, BB: Builder.GetInsertBlock());
1159
1160 // Done.
1161 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
1162}
1163
1164void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
1165 Address SrcAddr, const VarDecl *DestVD,
1166 const VarDecl *SrcVD, const Expr *Copy) {
1167 if (OriginalType->isArrayType()) {
1168 const auto *BO = dyn_cast<BinaryOperator>(Val: Copy);
1169 if (BO && BO->getOpcode() == BO_Assign) {
1170 // Perform simple memcpy for simple copying.
1171 LValue Dest = MakeAddrLValue(Addr: DestAddr, T: OriginalType);
1172 LValue Src = MakeAddrLValue(Addr: SrcAddr, T: OriginalType);
1173 EmitAggregateAssign(Dest, Src, EltTy: OriginalType);
1174 } else {
1175 // For arrays with complex element types perform element by element
1176 // copying.
1177 EmitOMPAggregateAssign(
1178 DestAddr, SrcAddr, OriginalType,
1179 CopyGen: [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
1180 // Working with the single array element, so have to remap
1181 // destination and source variables to corresponding array
1182 // elements.
1183 CodeGenFunction::OMPPrivateScope Remap(*this);
1184 Remap.addPrivate(LocalVD: DestVD, Addr: DestElement);
1185 Remap.addPrivate(LocalVD: SrcVD, Addr: SrcElement);
1186 (void)Remap.Privatize();
1187 EmitIgnoredExpr(E: Copy);
1188 });
1189 }
1190 } else {
1191 // Remap pseudo source variable to private copy.
1192 CodeGenFunction::OMPPrivateScope Remap(*this);
1193 Remap.addPrivate(LocalVD: SrcVD, Addr: SrcAddr);
1194 Remap.addPrivate(LocalVD: DestVD, Addr: DestAddr);
1195 (void)Remap.Privatize();
1196 // Emit copying of the whole variable.
1197 EmitIgnoredExpr(E: Copy);
1198 }
1199}
1200
1201bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
1202 OMPPrivateScope &PrivateScope) {
1203 if (!HaveInsertPoint())
1204 return false;
1205 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1206 bool DeviceConstTarget = getLangOpts().OpenMPIsTargetDevice &&
1207 isOpenMPTargetExecutionDirective(DKind: EKind);
1208 bool FirstprivateIsLastprivate = false;
1209 llvm::SmallDenseMap<const Decl *, OpenMPLastprivateModifier> Lastprivates;
1210 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1211 for (const auto *D : C->varlist()) {
1212 const auto *VD = cast<DeclRefExpr>(Val: D)->getDecl();
1213 Lastprivates.try_emplace(Key: VD->getCanonicalDecl(), Args: C->getKind());
1214 }
1215 }
1216 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
1217 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
1218 getOpenMPCaptureRegions(CaptureRegions, DKind: EKind);
1219 // Force emission of the firstprivate copy if the directive does not emit
1220 // outlined function, like omp for, omp simd, omp distribute etc.
1221 bool MustEmitFirstprivateCopy =
1222 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
1223 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
1224 const auto *IRef = C->varlist_begin();
1225 const auto *InitsRef = C->inits().begin();
1226 for (const Expr *IInit : C->private_copies()) {
1227 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1228 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1229
1230 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
1231 // Check if this binding is also lastprivate.
1232 bool ThisFirstprivateIsLastprivate =
1233 Lastprivates.count(Val: BD->getCanonicalDecl()) > 0;
1234 const auto *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
1235 // If the decomposition is captured by copy, the captured field is
1236 // already a private copy; map the binding to its member directly.
1237 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate) {
1238 if (const FieldDecl *FD = CapturedStmtInfo->lookup(VD: DD)) {
1239 if (!FD->getType()->isReferenceType()) {
1240 bool IsRegistered = PrivateScope.addPrivate(
1241 LocalVD: BD, Addr: EmitOMPCapturedBindingLValue(BD).getAddress());
1242 assert(IsRegistered &&
1243 "firstprivate var already registered as firstprivate");
1244 (void)IsRegistered;
1245 ++IRef;
1246 ++InitsRef;
1247 continue;
1248 }
1249 }
1250 }
1251 const auto *VDInit =
1252 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *InitsRef)->getDecl());
1253 Address OriginalAddr =
1254 EmitOMPBindingOriginalAddr(BD, Loc: (*IRef)->getExprLoc());
1255
1256 QualType Type = VD->getType();
1257 bool IsRegistered;
1258 if (Type->isArrayType()) {
1259 // For array bindings, use array copy logic
1260 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
1261 const Expr *Init = VD->getInit();
1262 LValue OriginalLVal = MakeAddrLValue(Addr: OriginalAddr, T: Type);
1263 if (!Init || !isa<CXXConstructExpr>(Val: Init) ||
1264 isTrivialInitializer(Init)) {
1265 // Perform simple memcpy.
1266 LValue Dest = MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: Type);
1267 EmitAggregateAssign(Dest, Src: OriginalLVal, EltTy: Type);
1268 } else {
1269 EmitOMPAggregateAssign(
1270 DestAddr: Emission.getAllocatedAddress(), SrcAddr: OriginalAddr, OriginalType: Type,
1271 CopyGen: [this, VDInit, Init](Address DestElement, Address SrcElement) {
1272 // Clean up any temporaries needed by the initialization.
1273 RunCleanupsScope InitScope(*this);
1274 // Emit initialization for single element.
1275 setAddrOfLocalVar(VD: VDInit, Addr: SrcElement);
1276 EmitAnyExprToMem(E: Init, Location: DestElement,
1277 Quals: Init->getType().getQualifiers(),
1278 /*IsInitializer*/ false);
1279 LocalDeclMap.erase(Val: VDInit);
1280 });
1281 }
1282 EmitAutoVarCleanups(emission: Emission);
1283 IsRegistered =
1284 PrivateScope.addPrivate(LocalVD: BD, Addr: Emission.getAllocatedAddress());
1285 } else {
1286 // VD now has the binding's type (e.g., int), not the struct type.
1287 // Emit VD initialized from the binding's field address.
1288 setAddrOfLocalVar(VD: VDInit, Addr: OriginalAddr);
1289 EmitDecl(D: *VD);
1290 LocalDeclMap.erase(Val: VDInit);
1291 Address VDAddr = GetAddrOfLocalVar(VD);
1292 // VD is the private copy of the binding, map BD to VDAddr directly
1293 IsRegistered = PrivateScope.addPrivate(LocalVD: BD, Addr: VDAddr);
1294 }
1295
1296 assert(IsRegistered &&
1297 "firstprivate var already registered as firstprivate");
1298 (void)IsRegistered;
1299 FirstprivateIsLastprivate =
1300 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1301 ++IRef;
1302 ++InitsRef;
1303 continue;
1304 }
1305
1306 // Original VarDecl logic.
1307 const VarDecl *OrigVD = dyn_cast<VarDecl>(Val: OrigDecl);
1308 assert(OrigVD && "Expected VarDecl for non-BindingDecl firstprivate");
1309 bool ThisFirstprivateIsLastprivate =
1310 Lastprivates.count(Val: OrigVD->getCanonicalDecl()) > 0;
1311 const FieldDecl *FD = CapturedStmtInfo->lookup(VD: OrigVD);
1312 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
1313 !FD->getType()->isReferenceType() &&
1314 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1315 EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl());
1316 ++IRef;
1317 ++InitsRef;
1318 continue;
1319 }
1320 // Do not emit copy for firstprivate constant variables in target regions,
1321 // captured by reference.
1322 if (DeviceConstTarget && OrigVD->getType().isConstant(Ctx: getContext()) &&
1323 FD && FD->getType()->isReferenceType() &&
1324 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1325 EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl());
1326 ++IRef;
1327 ++InitsRef;
1328 continue;
1329 }
1330 FirstprivateIsLastprivate =
1331 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1332 if (EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl()).second) {
1333 const auto *VDInit =
1334 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *InitsRef)->getDecl());
1335 bool IsRegistered;
1336 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1337 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
1338 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1339 LValue OriginalLVal;
1340 if (!FD) {
1341 // Check if the firstprivate variable is just a constant value.
1342 ConstantEmission CE = tryEmitAsConstant(RefExpr: &DRE);
1343 if (CE && !CE.isReference()) {
1344 // Constant value, no need to create a copy.
1345 ++IRef;
1346 ++InitsRef;
1347 continue;
1348 }
1349 if (CE && CE.isReference()) {
1350 OriginalLVal = CE.getReferenceLValue(CGF&: *this, RefExpr: &DRE);
1351 } else {
1352 assert(!CE && "Expected non-constant firstprivate.");
1353 OriginalLVal = EmitLValue(E: &DRE);
1354 }
1355 } else {
1356 OriginalLVal = EmitLValue(E: &DRE);
1357 }
1358 QualType Type = VD->getType();
1359 if (Type->isArrayType()) {
1360 // Emit VarDecl with copy init for arrays.
1361 // Get the address of the original variable captured in current
1362 // captured region.
1363 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
1364 const Expr *Init = VD->getInit();
1365 if (!isa<CXXConstructExpr>(Val: Init) || isTrivialInitializer(Init)) {
1366 // Perform simple memcpy.
1367 LValue Dest = MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: Type);
1368 EmitAggregateAssign(Dest, Src: OriginalLVal, EltTy: Type);
1369 } else {
1370 EmitOMPAggregateAssign(
1371 DestAddr: Emission.getAllocatedAddress(), SrcAddr: OriginalLVal.getAddress(), OriginalType: Type,
1372 CopyGen: [this, VDInit, Init](Address DestElement, Address SrcElement) {
1373 // Clean up any temporaries needed by the
1374 // initialization.
1375 RunCleanupsScope InitScope(*this);
1376 // Emit initialization for single element.
1377 setAddrOfLocalVar(VD: VDInit, Addr: SrcElement);
1378 EmitAnyExprToMem(E: Init, Location: DestElement,
1379 Quals: Init->getType().getQualifiers(),
1380 /*IsInitializer*/ false);
1381 LocalDeclMap.erase(Val: VDInit);
1382 });
1383 }
1384 EmitAutoVarCleanups(emission: Emission);
1385 IsRegistered =
1386 PrivateScope.addPrivate(LocalVD: OrigVD, Addr: Emission.getAllocatedAddress());
1387 } else {
1388 Address OriginalAddr = OriginalLVal.getAddress();
1389 // Emit private VarDecl with copy init.
1390 // Remap temp VDInit variable to the address of the original
1391 // variable (for proper handling of captured global variables).
1392 setAddrOfLocalVar(VD: VDInit, Addr: OriginalAddr);
1393 EmitDecl(D: *VD);
1394 LocalDeclMap.erase(Val: VDInit);
1395 Address VDAddr = GetAddrOfLocalVar(VD);
1396 if (ThisFirstprivateIsLastprivate &&
1397 Lastprivates[OrigVD->getCanonicalDecl()] ==
1398 OMPC_LASTPRIVATE_conditional) {
1399 // Create/init special variable for lastprivate conditionals.
1400 llvm::Value *V =
1401 EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: VDAddr, T: (*IRef)->getType(),
1402 Source: AlignmentSource::Decl),
1403 Loc: (*IRef)->getExprLoc());
1404 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1405 CGF&: *this, VD: OrigVD);
1406 EmitStoreOfScalar(value: V, lvalue: MakeAddrLValue(Addr: VDAddr, T: (*IRef)->getType(),
1407 Source: AlignmentSource::Decl));
1408 LocalDeclMap.erase(Val: VD);
1409 setAddrOfLocalVar(VD, Addr: VDAddr);
1410 }
1411 IsRegistered = PrivateScope.addPrivate(LocalVD: OrigVD, Addr: VDAddr);
1412 }
1413 assert(IsRegistered &&
1414 "firstprivate var already registered as private");
1415 // Silence the warning about unused variable.
1416 (void)IsRegistered;
1417 }
1418 ++IRef;
1419 ++InitsRef;
1420 }
1421 }
1422 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1423}
1424
1425void CodeGenFunction::EmitOMPPrivateClause(
1426 const OMPExecutableDirective &D,
1427 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1428 if (!HaveInsertPoint())
1429 return;
1430 llvm::SmallDenseSet<const ValueDecl *> EmittedAsPrivate;
1431 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
1432 auto IRef = C->varlist_begin();
1433 for (const Expr *IInit : C->private_copies()) {
1434 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1435 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1436 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
1437 .second) {
1438 EmitDecl(D: *VD);
1439 bool IsRegistered =
1440 PrivateScope.addPrivate(LocalVD: OrigDecl, Addr: GetAddrOfLocalVar(VD));
1441 assert(IsRegistered && "private var already registered as private");
1442 (void)IsRegistered;
1443 }
1444 ++IRef;
1445 }
1446 }
1447}
1448
1449bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
1450 if (!HaveInsertPoint())
1451 return false;
1452 // threadprivate_var1 = master_threadprivate_var1;
1453 // operator=(threadprivate_var2, master_threadprivate_var2);
1454 // ...
1455 // __kmpc_barrier(&loc, global_tid);
1456 llvm::DenseSet<const VarDecl *> CopiedVars;
1457 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
1458 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
1459 auto IRef = C->varlist_begin();
1460 auto ISrcRef = C->source_exprs().begin();
1461 auto IDestRef = C->destination_exprs().begin();
1462 for (const Expr *AssignOp : C->assignment_ops()) {
1463 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
1464 QualType Type = VD->getType();
1465 if (CopiedVars.insert(V: VD->getCanonicalDecl()).second) {
1466 // Get the address of the master variable. If we are emitting code with
1467 // TLS support, the address is passed from the master as field in the
1468 // captured declaration.
1469 Address MasterAddr = Address::invalid();
1470 if (getLangOpts().OpenMPUseTLS &&
1471 getContext().getTargetInfo().isTLSSupported()) {
1472 assert(CapturedStmtInfo->lookup(VD) &&
1473 "Copyin threadprivates should have been captured!");
1474 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
1475 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1476 MasterAddr = EmitLValue(E: &DRE).getAddress();
1477 LocalDeclMap.erase(Val: VD);
1478 } else {
1479 MasterAddr =
1480 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(D: VD)
1481 : CGM.GetAddrOfGlobal(GD: VD),
1482 CGM.getTypes().ConvertTypeForMem(T: VD->getType()),
1483 getContext().getDeclAlign(D: VD));
1484 }
1485 // Get the address of the threadprivate variable.
1486 Address PrivateAddr = EmitLValue(E: *IRef).getAddress();
1487 if (CopiedVars.size() == 1) {
1488 // At first check if current thread is a master thread. If it is, no
1489 // need to copy data.
1490 CopyBegin = createBasicBlock(name: "copyin.not.master");
1491 CopyEnd = createBasicBlock(name: "copyin.not.master.end");
1492 // TODO: Avoid ptrtoint conversion.
1493 auto *MasterAddrInt = Builder.CreatePtrToInt(
1494 V: MasterAddr.emitRawPointer(CGF&: *this), DestTy: CGM.IntPtrTy);
1495 auto *PrivateAddrInt = Builder.CreatePtrToInt(
1496 V: PrivateAddr.emitRawPointer(CGF&: *this), DestTy: CGM.IntPtrTy);
1497 Builder.CreateCondBr(
1498 Cond: Builder.CreateICmpNE(LHS: MasterAddrInt, RHS: PrivateAddrInt), True: CopyBegin,
1499 False: CopyEnd);
1500 EmitBlock(BB: CopyBegin);
1501 }
1502 const auto *SrcVD =
1503 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1504 const auto *DestVD =
1505 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1506 EmitOMPCopy(OriginalType: Type, DestAddr: PrivateAddr, SrcAddr: MasterAddr, DestVD, SrcVD, Copy: AssignOp);
1507 }
1508 ++IRef;
1509 ++ISrcRef;
1510 ++IDestRef;
1511 }
1512 }
1513 if (CopyEnd) {
1514 // Exit out of copying procedure for non-master thread.
1515 EmitBlock(BB: CopyEnd, /*IsFinished=*/true);
1516 return true;
1517 }
1518 return false;
1519}
1520
1521bool CodeGenFunction::EmitOMPLastprivateClauseInit(
1522 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1523 if (!HaveInsertPoint())
1524 return false;
1525 bool HasAtLeastOneLastprivate = false;
1526 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1527 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1528 if (isOpenMPSimdDirective(DKind: EKind)) {
1529 const auto *LoopDirective = cast<OMPLoopDirective>(Val: &D);
1530 for (const Expr *C : LoopDirective->counters()) {
1531 SIMDLCVs.insert(
1532 V: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: C)->getDecl())->getCanonicalDecl());
1533 }
1534 }
1535 llvm::SmallDenseSet<const ValueDecl *> AlreadyEmittedVars;
1536 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1537 HasAtLeastOneLastprivate = true;
1538 if (isOpenMPTaskLoopDirective(DKind: EKind) && !getLangOpts().OpenMPSimd)
1539 break;
1540 const auto *IRef = C->varlist_begin();
1541 const auto *IDestRef = C->destination_exprs().begin();
1542 for (const Expr *IInit : C->private_copies()) {
1543 // Keep the address of the original variable for future update at the end
1544 // of the loop.
1545 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1546 // Handle BindingDecls with the same level of support as VarDecls.
1547 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
1548 if (AlreadyEmittedVars.insert(V: cast<ValueDecl>(Val: BD->getCanonicalDecl()))
1549 .second) {
1550 const auto *DestVD =
1551 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1552
1553 // Get the original binding address.
1554 Address OrigAddr =
1555 EmitOMPBindingOriginalAddr(BD, Loc: (*IRef)->getExprLoc());
1556 PrivateScope.addPrivate(LocalVD: DestVD, Addr: OrigAddr);
1557 if (IInit) {
1558 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1559 // Emit private VarDecl with copy init.
1560 EmitDecl(D: *VD);
1561 Address VDAddr = GetAddrOfLocalVar(VD);
1562 bool IsRegistered = PrivateScope.addPrivate(LocalVD: BD, Addr: VDAddr);
1563 assert(IsRegistered &&
1564 "lastprivate binding already registered as private");
1565 (void)IsRegistered;
1566 }
1567 }
1568 ++IRef;
1569 ++IDestRef;
1570 continue;
1571 }
1572 const auto *OrigVD = cast<VarDecl>(Val: OrigDecl);
1573 // Taskloops do not require additional initialization, it is done in
1574 // runtime support library.
1575 if (AlreadyEmittedVars.insert(V: OrigVD->getCanonicalDecl()).second) {
1576 const auto *DestVD =
1577 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1578 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1579 /*RefersToEnclosingVariableOrCapture=*/
1580 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
1581 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1582 PrivateScope.addPrivate(LocalVD: DestVD, Addr: EmitLValue(E: &DRE).getAddress());
1583 // Check if the variable is also a firstprivate: in this case IInit is
1584 // not generated. Initialization of this variable will happen in codegen
1585 // for 'firstprivate' clause.
1586 if (IInit && !SIMDLCVs.count(V: OrigVD->getCanonicalDecl())) {
1587 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1588 Address VDAddr = Address::invalid();
1589 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1590 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1591 CGF&: *this, VD: OrigVD);
1592 setAddrOfLocalVar(VD, Addr: VDAddr);
1593 } else {
1594 // Emit private VarDecl with copy init.
1595 EmitDecl(D: *VD);
1596 VDAddr = GetAddrOfLocalVar(VD);
1597 }
1598 bool IsRegistered = PrivateScope.addPrivate(LocalVD: OrigVD, Addr: VDAddr);
1599 assert(IsRegistered &&
1600 "lastprivate var already registered as private");
1601 (void)IsRegistered;
1602 }
1603 }
1604 ++IRef;
1605 ++IDestRef;
1606 }
1607 }
1608 return HasAtLeastOneLastprivate;
1609}
1610
1611void CodeGenFunction::EmitOMPLastprivateClauseFinal(
1612 const OMPExecutableDirective &D, bool NoFinals,
1613 llvm::Value *IsLastIterCond) {
1614 if (!HaveInsertPoint())
1615 return;
1616 // Emit following code:
1617 // if (<IsLastIterCond>) {
1618 // orig_var1 = private_orig_var1;
1619 // ...
1620 // orig_varn = private_orig_varn;
1621 // }
1622 llvm::BasicBlock *ThenBB = nullptr;
1623 llvm::BasicBlock *DoneBB = nullptr;
1624 if (IsLastIterCond) {
1625 // Emit implicit barrier if at least one lastprivate conditional is found
1626 // and this is not a simd mode.
1627 if (!getLangOpts().OpenMPSimd &&
1628 llvm::any_of(Range: D.getClausesOfKind<OMPLastprivateClause>(),
1629 P: [](const OMPLastprivateClause *C) {
1630 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1631 })) {
1632 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: D.getBeginLoc(),
1633 Kind: OMPD_unknown,
1634 /*EmitChecks=*/false,
1635 /*ForceSimpleCall=*/true);
1636 }
1637 ThenBB = createBasicBlock(name: ".omp.lastprivate.then");
1638 DoneBB = createBasicBlock(name: ".omp.lastprivate.done");
1639 Builder.CreateCondBr(Cond: IsLastIterCond, True: ThenBB, False: DoneBB);
1640 EmitBlock(BB: ThenBB);
1641 }
1642 llvm::DenseSet<const ValueDecl *> AlreadyEmittedVars;
1643 llvm::SmallDenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1644 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(Val: &D)) {
1645 auto IC = LoopDirective->counters().begin();
1646 for (const Expr *F : LoopDirective->finals()) {
1647 const auto *D =
1648 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IC)->getDecl())->getCanonicalDecl();
1649 if (NoFinals)
1650 AlreadyEmittedVars.insert(V: D);
1651 else
1652 LoopCountersAndUpdates[D] = F;
1653 ++IC;
1654 }
1655 }
1656 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1657 auto IRef = C->varlist_begin();
1658 auto ISrcRef = C->source_exprs().begin();
1659 auto IDestRef = C->destination_exprs().begin();
1660 for (const Expr *AssignOp : C->assignment_ops()) {
1661 const auto *PrivateDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
1662
1663 // For BindingDecls, check if we should use .lastprivate.src or the BD
1664 // itself.
1665 const VarDecl *PrivateVD = nullptr;
1666 const BindingDecl *BD = nullptr;
1667 if ((BD = dyn_cast<BindingDecl>(Val: PrivateDecl))) {
1668 // Check if .lastprivate.src is available (taskloop case).
1669 const auto *SrcVD =
1670 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1671
1672 if (LocalDeclMap.count(Val: SrcVD)) {
1673 // Taskloop case: use .lastprivate.src.
1674 PrivateVD = SrcVD;
1675 } else if (OMPPrivatizedBindings.count(Val: BD)) {
1676 // Parallel for case: BindingDecl is directly privatized.
1677 // Leave PrivateVD as nullptr to handle specially below.
1678 }
1679 } else {
1680 PrivateVD = cast<VarDecl>(Val: PrivateDecl);
1681 }
1682
1683 QualType Type = PrivateVD ? PrivateVD->getType() : BD->getType();
1684 const auto *CanonicalVD =
1685 PrivateVD ? PrivateVD->getCanonicalDecl() : nullptr;
1686
1687 // Check if already emitted.
1688 bool ShouldEmit =
1689 AlreadyEmittedVars
1690 .insert(V: PrivateVD ? static_cast<const ValueDecl *>(CanonicalVD)
1691 : static_cast<const ValueDecl *>(BD))
1692 .second;
1693 if (ShouldEmit) {
1694 // If lastprivate variable is a loop control variable for loop-based
1695 // directive, update its value before copyin back to original
1696 // variable.
1697 if (CanonicalVD) {
1698 if (const Expr *FinalExpr =
1699 LoopCountersAndUpdates.lookup(Val: CanonicalVD))
1700 EmitIgnoredExpr(E: FinalExpr);
1701 }
1702 const auto *SrcVD =
1703 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1704 const auto *DestVD =
1705 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1706
1707 // Get the address of the private variable.
1708 Address PrivateAddr = Address::invalid();
1709 if (PrivateVD) {
1710 PrivateAddr = GetAddrOfLocalVar(VD: PrivateVD);
1711 } else {
1712 auto It = OMPPrivatizedBindings.find(Val: BD);
1713 assert(It != OMPPrivatizedBindings.end() &&
1714 "BindingDecl should be privatized");
1715 PrivateAddr = It->second;
1716 }
1717 if (PrivateVD) {
1718 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1719 PrivateAddr = Address(
1720 Builder.CreateLoad(Addr: PrivateAddr),
1721 CGM.getTypes().ConvertTypeForMem(T: RefTy->getPointeeType()),
1722 CGM.getNaturalTypeAlignment(T: RefTy->getPointeeType()));
1723 }
1724 // Store the last value to the private copy in the last iteration.
1725 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1726 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1727 CGF&: *this, PrivLVal: MakeAddrLValue(Addr: PrivateAddr, T: (*IRef)->getType()), VD: PrivateVD,
1728 Loc: (*IRef)->getExprLoc());
1729 // Get the address of the original variable.
1730 Address OriginalAddr = GetAddrOfLocalVar(VD: DestVD);
1731 EmitOMPCopy(OriginalType: Type, DestAddr: OriginalAddr, SrcAddr: PrivateAddr, DestVD, SrcVD, Copy: AssignOp);
1732 }
1733 ++IRef;
1734 ++ISrcRef;
1735 ++IDestRef;
1736 }
1737 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1738 EmitIgnoredExpr(E: PostUpdate);
1739 }
1740 if (IsLastIterCond)
1741 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
1742}
1743
1744void CodeGenFunction::EmitOMPReductionClauseInit(
1745 const OMPExecutableDirective &D,
1746 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1747 if (!HaveInsertPoint())
1748 return;
1749 SmallVector<const Expr *, 4> Shareds;
1750 SmallVector<const Expr *, 4> Privates;
1751 SmallVector<const Expr *, 4> ReductionOps;
1752 SmallVector<const Expr *, 4> LHSs;
1753 SmallVector<const Expr *, 4> RHSs;
1754 OMPTaskDataTy Data;
1755 SmallVector<const Expr *, 4> TaskLHSs;
1756 SmallVector<const Expr *, 4> TaskRHSs;
1757 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1758 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1759 continue;
1760 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
1761 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
1762 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
1763 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1764 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1765 if (C->getModifier() == OMPC_REDUCTION_task) {
1766 Data.ReductionVars.append(in_start: C->privates().begin(), in_end: C->privates().end());
1767 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
1768 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
1769 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
1770 in_end: C->reduction_ops().end());
1771 TaskLHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1772 TaskRHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1773 }
1774 }
1775 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1776 unsigned Count = 0;
1777 auto *ILHS = LHSs.begin();
1778 auto *IRHS = RHSs.begin();
1779 auto *IPriv = Privates.begin();
1780 for (const Expr *IRef : Shareds) {
1781 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IPriv)->getDecl());
1782 RedCG.emitSharedOrigLValue(CGF&: *this, N: Count);
1783 RedCG.emitAggregateType(CGF&: *this, N: Count);
1784 AutoVarEmission Emission = EmitAutoVarAlloca(var: *PrivateVD);
1785 RedCG.emitInitialization(CGF&: *this, N: Count, PrivateAddr: Emission.getAllocatedAddress(),
1786 SharedAddr: RedCG.getSharedLValue(N: Count).getAddress(),
1787 DefaultInit: [&Emission](CodeGenFunction &CGF) {
1788 CGF.EmitAutoVarInit(emission: Emission);
1789 return true;
1790 });
1791 EmitAutoVarCleanups(emission: Emission);
1792 Address BaseAddr = RedCG.adjustPrivateAddress(
1793 CGF&: *this, N: Count, PrivateAddr: Emission.getAllocatedAddress());
1794 bool IsRegistered =
1795 PrivateScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Count), Addr: BaseAddr);
1796 assert(IsRegistered && "private var already registered as private");
1797 // Silence the warning about unused variable.
1798 (void)IsRegistered;
1799
1800 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
1801 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
1802 QualType Type = PrivateVD->getType();
1803 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(Val: IRef);
1804 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1805 // Store the address of the original variable associated with the LHS
1806 // implicit variable.
1807 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: RedCG.getSharedLValue(N: Count).getAddress());
1808 PrivateScope.addPrivate(LocalVD: RHSVD, Addr: GetAddrOfLocalVar(VD: PrivateVD));
1809 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1810 isa<ArraySubscriptExpr>(Val: IRef)) {
1811 // Store the address of the original variable associated with the LHS
1812 // implicit variable.
1813 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: RedCG.getSharedLValue(N: Count).getAddress());
1814 PrivateScope.addPrivate(LocalVD: RHSVD,
1815 Addr: GetAddrOfLocalVar(VD: PrivateVD).withElementType(
1816 ElemTy: ConvertTypeForMem(T: RHSVD->getType())));
1817 } else {
1818 QualType Type = PrivateVD->getType();
1819 bool IsArray = getContext().getAsArrayType(T: Type) != nullptr;
1820 Address OriginalAddr = RedCG.getSharedLValue(N: Count).getAddress();
1821 // Store the address of the original variable associated with the LHS
1822 // implicit variable.
1823 if (IsArray) {
1824 OriginalAddr =
1825 OriginalAddr.withElementType(ElemTy: ConvertTypeForMem(T: LHSVD->getType()));
1826 }
1827 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: OriginalAddr);
1828 PrivateScope.addPrivate(
1829 LocalVD: RHSVD, Addr: IsArray ? GetAddrOfLocalVar(VD: PrivateVD).withElementType(
1830 ElemTy: ConvertTypeForMem(T: RHSVD->getType()))
1831 : GetAddrOfLocalVar(VD: PrivateVD));
1832 }
1833 ++ILHS;
1834 ++IRHS;
1835 ++IPriv;
1836 ++Count;
1837 }
1838 if (!Data.ReductionVars.empty()) {
1839 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1840 Data.IsReductionWithTaskMod = true;
1841 Data.IsWorksharingReduction = isOpenMPWorksharingDirective(DKind: EKind);
1842 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1843 CGF&: *this, Loc: D.getBeginLoc(), LHSExprs: TaskLHSs, RHSExprs: TaskRHSs, Data);
1844 const Expr *TaskRedRef = nullptr;
1845 switch (EKind) {
1846 case OMPD_parallel:
1847 TaskRedRef = cast<OMPParallelDirective>(Val: D).getTaskReductionRefExpr();
1848 break;
1849 case OMPD_for:
1850 TaskRedRef = cast<OMPForDirective>(Val: D).getTaskReductionRefExpr();
1851 break;
1852 case OMPD_sections:
1853 TaskRedRef = cast<OMPSectionsDirective>(Val: D).getTaskReductionRefExpr();
1854 break;
1855 case OMPD_parallel_for:
1856 TaskRedRef = cast<OMPParallelForDirective>(Val: D).getTaskReductionRefExpr();
1857 break;
1858 case OMPD_parallel_master:
1859 TaskRedRef =
1860 cast<OMPParallelMasterDirective>(Val: D).getTaskReductionRefExpr();
1861 break;
1862 case OMPD_parallel_sections:
1863 TaskRedRef =
1864 cast<OMPParallelSectionsDirective>(Val: D).getTaskReductionRefExpr();
1865 break;
1866 case OMPD_target_parallel:
1867 TaskRedRef =
1868 cast<OMPTargetParallelDirective>(Val: D).getTaskReductionRefExpr();
1869 break;
1870 case OMPD_target_parallel_for:
1871 TaskRedRef =
1872 cast<OMPTargetParallelForDirective>(Val: D).getTaskReductionRefExpr();
1873 break;
1874 case OMPD_distribute_parallel_for:
1875 TaskRedRef =
1876 cast<OMPDistributeParallelForDirective>(Val: D).getTaskReductionRefExpr();
1877 break;
1878 case OMPD_teams_distribute_parallel_for:
1879 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(Val: D)
1880 .getTaskReductionRefExpr();
1881 break;
1882 case OMPD_target_teams_distribute_parallel_for:
1883 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(Val: D)
1884 .getTaskReductionRefExpr();
1885 break;
1886 case OMPD_simd:
1887 case OMPD_for_simd:
1888 case OMPD_section:
1889 case OMPD_single:
1890 case OMPD_master:
1891 case OMPD_critical:
1892 case OMPD_parallel_for_simd:
1893 case OMPD_task:
1894 case OMPD_taskyield:
1895 case OMPD_error:
1896 case OMPD_barrier:
1897 case OMPD_taskwait:
1898 case OMPD_taskgroup:
1899 case OMPD_flush:
1900 case OMPD_depobj:
1901 case OMPD_scan:
1902 case OMPD_ordered_standalone:
1903 case OMPD_ordered_blockassoc:
1904 case OMPD_atomic:
1905 case OMPD_teams:
1906 case OMPD_target:
1907 case OMPD_cancellation_point:
1908 case OMPD_cancel:
1909 case OMPD_target_data:
1910 case OMPD_target_enter_data:
1911 case OMPD_target_exit_data:
1912 case OMPD_taskloop:
1913 case OMPD_taskloop_simd:
1914 case OMPD_master_taskloop:
1915 case OMPD_master_taskloop_simd:
1916 case OMPD_parallel_master_taskloop:
1917 case OMPD_parallel_master_taskloop_simd:
1918 case OMPD_distribute:
1919 case OMPD_target_update:
1920 case OMPD_distribute_parallel_for_simd:
1921 case OMPD_distribute_simd:
1922 case OMPD_target_parallel_for_simd:
1923 case OMPD_target_simd:
1924 case OMPD_teams_distribute:
1925 case OMPD_teams_distribute_simd:
1926 case OMPD_teams_distribute_parallel_for_simd:
1927 case OMPD_target_teams:
1928 case OMPD_target_teams_distribute:
1929 case OMPD_target_teams_distribute_parallel_for_simd:
1930 case OMPD_target_teams_distribute_simd:
1931 case OMPD_declare_target:
1932 case OMPD_end_declare_target:
1933 case OMPD_threadprivate:
1934 case OMPD_allocate:
1935 case OMPD_declare_reduction:
1936 case OMPD_declare_mapper:
1937 case OMPD_declare_simd:
1938 case OMPD_requires:
1939 case OMPD_declare_variant:
1940 case OMPD_begin_declare_variant:
1941 case OMPD_end_declare_variant:
1942 case OMPD_unknown:
1943 default:
1944 llvm_unreachable("Unexpected directive with task reductions.");
1945 }
1946
1947 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: TaskRedRef)->getDecl());
1948 EmitVarDecl(D: *VD);
1949 EmitStoreOfScalar(Value: ReductionDesc, Addr: GetAddrOfLocalVar(VD),
1950 /*Volatile=*/false, Ty: TaskRedRef->getType());
1951 }
1952}
1953
1954void CodeGenFunction::EmitOMPReductionClauseFinal(
1955 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1956 if (!HaveInsertPoint())
1957 return;
1958 llvm::SmallVector<const Expr *, 8> Privates;
1959 llvm::SmallVector<const Expr *, 8> LHSExprs;
1960 llvm::SmallVector<const Expr *, 8> RHSExprs;
1961 llvm::SmallVector<const Expr *, 8> ReductionOps;
1962 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1963 bool HasAtLeastOneReduction = false;
1964 bool IsReductionWithTaskMod = false;
1965 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1966 // Do not emit for inscan reductions.
1967 if (C->getModifier() == OMPC_REDUCTION_inscan)
1968 continue;
1969 HasAtLeastOneReduction = true;
1970 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
1971 LHSExprs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1972 RHSExprs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1973 IsPrivateVarReduction.append(in_start: C->private_var_reduction_flags().begin(),
1974 in_end: C->private_var_reduction_flags().end());
1975 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
1976 IsReductionWithTaskMod =
1977 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1978 }
1979 if (HasAtLeastOneReduction) {
1980 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1981 if (IsReductionWithTaskMod) {
1982 CGM.getOpenMPRuntime().emitTaskReductionFini(
1983 CGF&: *this, Loc: D.getBeginLoc(), IsWorksharingReduction: isOpenMPWorksharingDirective(DKind: EKind));
1984 }
1985 bool TeamsLoopCanBeParallel = false;
1986 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(Val: &D))
1987 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1988 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1989 isOpenMPParallelDirective(DKind: EKind) ||
1990 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1991 bool SimpleReduction = ReductionKind == OMPD_simd;
1992 // Emit nowait reduction if nowait clause is present or directive is a
1993 // parallel directive (it always has implicit barrier).
1994 CGM.getOpenMPRuntime().emitReduction(
1995 CGF&: *this, Loc: D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1996 Options: {.WithNowait: WithNowait, .SimpleReduction: SimpleReduction, .IsPrivateVarReduction: IsPrivateVarReduction, .ReductionKind: ReductionKind});
1997 }
1998}
1999
2000static void emitPostUpdateForReductionClause(
2001 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2002 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2003 if (!CGF.HaveInsertPoint())
2004 return;
2005 llvm::BasicBlock *DoneBB = nullptr;
2006 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
2007 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
2008 if (!DoneBB) {
2009 if (llvm::Value *Cond = CondGen(CGF)) {
2010 // If the first post-update expression is found, emit conditional
2011 // block if it was requested.
2012 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: ".omp.reduction.pu");
2013 DoneBB = CGF.createBasicBlock(name: ".omp.reduction.pu.done");
2014 CGF.Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
2015 CGF.EmitBlock(BB: ThenBB);
2016 }
2017 }
2018 CGF.EmitIgnoredExpr(E: PostUpdate);
2019 }
2020 }
2021 if (DoneBB)
2022 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
2023}
2024
2025namespace {
2026/// Codegen lambda for appending distribute lower and upper bounds to outlined
2027/// parallel function. This is necessary for combined constructs such as
2028/// 'distribute parallel for'
2029typedef llvm::function_ref<void(CodeGenFunction &,
2030 const OMPExecutableDirective &,
2031 llvm::SmallVectorImpl<llvm::Value *> &)>
2032 CodeGenBoundParametersTy;
2033} // anonymous namespace
2034
2035static void
2036checkForLastprivateConditionalUpdate(CodeGenFunction &CGF,
2037 const OMPExecutableDirective &S) {
2038 if (CGF.getLangOpts().OpenMP < 50)
2039 return;
2040 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
2041 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2042 for (const Expr *Ref : C->varlist()) {
2043 if (!Ref->getType()->isScalarType())
2044 continue;
2045 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2046 if (!DRE)
2047 continue;
2048 // Skip BindingDecls - lastprivate conditional only applies to VarDecls.
2049 const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl());
2050 if (!VD)
2051 continue;
2052 PrivateDecls.insert(V: VD);
2053 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
2054 }
2055 }
2056 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2057 for (const Expr *Ref : C->varlist()) {
2058 if (!Ref->getType()->isScalarType())
2059 continue;
2060 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2061 if (!DRE)
2062 continue;
2063 // Skip BindingDecls - they don't use the same conditional lastprivate
2064 // mechanism.
2065 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
2066 PrivateDecls.insert(V: VD);
2067 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
2068 }
2069 }
2070 }
2071 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
2072 for (const Expr *Ref : C->varlist()) {
2073 if (!Ref->getType()->isScalarType())
2074 continue;
2075 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2076 if (!DRE)
2077 continue;
2078 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
2079 PrivateDecls.insert(V: VD);
2080 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
2081 }
2082 }
2083 }
2084 // Privates should ne analyzed since they are not captured at all.
2085 // Task reductions may be skipped - tasks are ignored.
2086 // Firstprivates do not return value but may be passed by reference - no need
2087 // to check for updated lastprivate conditional.
2088 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2089 for (const Expr *Ref : C->varlist()) {
2090 if (!Ref->getType()->isScalarType())
2091 continue;
2092 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
2093 if (!DRE)
2094 continue;
2095 // Only track VarDecl, not BindingDecl.
2096 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl()))
2097 PrivateDecls.insert(V: VD);
2098 }
2099 }
2100 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional(
2101 CGF, D: S, IgnoredDecls: PrivateDecls);
2102}
2103
2104static void emitCommonOMPParallelDirective(
2105 CodeGenFunction &CGF, const OMPExecutableDirective &S,
2106 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
2107 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
2108 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
2109 llvm::Value *NumThreads = nullptr;
2110 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown;
2111 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is as
2112 // if sev-level is fatal."
2113 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal;
2114 clang::Expr *Message = nullptr;
2115 SourceLocation SeverityLoc = SourceLocation();
2116 SourceLocation MessageLoc = SourceLocation();
2117
2118 llvm::Function *OutlinedFn =
2119 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
2120 CGF, D: S, ThreadIDVar: *CS->getCapturedDecl()->param_begin(), InnermostKind,
2121 CodeGen);
2122
2123 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
2124 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
2125 NumThreads = CGF.EmitScalarExpr(E: NumThreadsClause->getNumThreads().front(),
2126 /*IgnoreResultAssign=*/true);
2127 Modifier = NumThreadsClause->getPrescriptivenessModifier();
2128 if (const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {
2129 Message = MessageClause->getMessageString();
2130 MessageLoc = MessageClause->getBeginLoc();
2131 }
2132 if (const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {
2133 Severity = SeverityClause->getSeverityKind();
2134 SeverityLoc = SeverityClause->getBeginLoc();
2135 }
2136 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
2137 CGF, NumThreads, Loc: NumThreadsClause->getBeginLoc(), Modifier, Severity,
2138 SeverityLoc, Message, MessageLoc);
2139 }
2140 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
2141 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
2142 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
2143 CGF, ProcBind: ProcBindClause->getProcBindKind(), Loc: ProcBindClause->getBeginLoc());
2144 }
2145 const Expr *IfCond = nullptr;
2146 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2147 if (C->getNameModifier() == OMPD_unknown ||
2148 C->getNameModifier() == OMPD_parallel) {
2149 IfCond = C->getCondition();
2150 break;
2151 }
2152 }
2153
2154 OMPParallelScope Scope(CGF, S);
2155 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
2156 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
2157 // lower and upper bounds with the pragma 'for' chunking mechanism.
2158 // The following lambda takes care of appending the lower and upper bound
2159 // parameters when necessary
2160 CodeGenBoundParameters(CGF, S, CapturedVars);
2161 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
2162 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, Loc: S.getBeginLoc(), OutlinedFn,
2163 CapturedVars, IfCond, NumThreads,
2164 NumThreadsModifier: Modifier, Severity, Message);
2165}
2166
2167static bool isAllocatableDecl(const VarDecl *VD) {
2168 const VarDecl *CVD = VD->getCanonicalDecl();
2169 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
2170 return false;
2171 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
2172 // Use the default allocation.
2173 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
2174 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
2175 !AA->getAllocator());
2176}
2177
2178static void emitEmptyBoundParameters(CodeGenFunction &,
2179 const OMPExecutableDirective &,
2180 llvm::SmallVectorImpl<llvm::Value *> &) {}
2181
2182static void emitOMPCopyinClause(CodeGenFunction &CGF,
2183 const OMPExecutableDirective &S) {
2184 bool Copyins = CGF.EmitOMPCopyinClause(D: S);
2185 if (Copyins) {
2186 // Emit implicit barrier to synchronize threads and avoid data races on
2187 // propagation master's thread values of threadprivate variables to local
2188 // instances of that variables of all other implicit threads.
2189 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2190 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
2191 /*ForceSimpleCall=*/true);
2192 }
2193}
2194
2195Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable(
2196 CodeGenFunction &CGF, const VarDecl *VD) {
2197 CodeGenModule &CGM = CGF.CGM;
2198 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2199
2200 if (!VD)
2201 return Address::invalid();
2202 const VarDecl *CVD = VD->getCanonicalDecl();
2203 if (!isAllocatableDecl(VD: CVD))
2204 return Address::invalid();
2205 llvm::Value *Size;
2206 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
2207 if (CVD->getType()->isVariablyModifiedType()) {
2208 Size = CGF.getTypeSize(Ty: CVD->getType());
2209 // Align the size: ((size + align - 1) / align) * align
2210 Size = CGF.Builder.CreateNUWAdd(
2211 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
2212 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
2213 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
2214 } else {
2215 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
2216 Size = CGM.getSize(numChars: Sz.alignTo(Align));
2217 }
2218
2219 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
2220 assert(AA->getAllocator() &&
2221 "Expected allocator expression for non-default allocator.");
2222 llvm::Value *Allocator = CGF.EmitScalarExpr(E: AA->getAllocator());
2223 // According to the standard, the original allocator type is a enum (integer).
2224 // Convert to pointer type, if required.
2225 if (Allocator->getType()->isIntegerTy())
2226 Allocator = CGF.Builder.CreateIntToPtr(V: Allocator, DestTy: CGM.VoidPtrTy);
2227 else if (Allocator->getType()->isPointerTy())
2228 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: Allocator,
2229 DestTy: CGM.VoidPtrTy);
2230
2231 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
2232 Loc: CGF.Builder, Size, Allocator,
2233 Name: getNameWithSeparators(Parts: {CVD->getName(), ".void.addr"}, FirstSeparator: ".", Separator: "."));
2234 llvm::CallInst *FreeCI =
2235 OMPBuilder.createOMPFree(Loc: CGF.Builder, Addr, Allocator);
2236
2237 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(Kind: NormalAndEHCleanup, A: FreeCI);
2238 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2239 V: Addr,
2240 DestTy: CGF.ConvertTypeForMem(T: CGM.getContext().getPointerType(T: CVD->getType())),
2241 Name: getNameWithSeparators(Parts: {CVD->getName(), ".addr"}, FirstSeparator: ".", Separator: "."));
2242 return Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
2243}
2244
2245Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2246 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
2247 SourceLocation Loc) {
2248 CodeGenModule &CGM = CGF.CGM;
2249 if (CGM.getLangOpts().OpenMPUseTLS &&
2250 CGM.getContext().getTargetInfo().isTLSSupported())
2251 return VDAddr;
2252
2253 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2254
2255 llvm::Type *VarTy = VDAddr.getElementType();
2256 llvm::Value *Data =
2257 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy);
2258 llvm::ConstantInt *Size = CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy));
2259 std::string Suffix = getNameWithSeparators(Parts: {"cache", ""});
2260 llvm::Twine CacheName = Twine(CGM.getMangledName(GD: VD)).concat(Suffix);
2261
2262 llvm::CallInst *ThreadPrivateCacheCall =
2263 OMPBuilder.createCachedThreadPrivate(Loc: CGF.Builder, Pointer: Data, Size, Name: CacheName);
2264
2265 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
2266}
2267
2268std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(
2269 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
2270 SmallString<128> Buffer;
2271 llvm::raw_svector_ostream OS(Buffer);
2272 StringRef Sep = FirstSeparator;
2273 for (StringRef Part : Parts) {
2274 OS << Sep << Part;
2275 Sep = Separator;
2276 }
2277 return OS.str().str();
2278}
2279
2280void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
2281 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2282 InsertPointTy CodeGenIP, Twine RegionName) {
2283 CGBuilderTy &Builder = CGF.Builder;
2284 Builder.restoreIP(IP: CodeGenIP);
2285 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2286 Suffix: "." + RegionName + ".after");
2287
2288 {
2289 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2290 CGF.EmitStmt(S: RegionBodyStmt);
2291 }
2292
2293 if (Builder.saveIP().isValid())
2294 Builder.CreateBr(Dest: FiniBB);
2295}
2296
2297void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
2298 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2299 InsertPointTy CodeGenIP, Twine RegionName) {
2300 CGBuilderTy &Builder = CGF.Builder;
2301 Builder.restoreIP(IP: CodeGenIP);
2302 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2303 Suffix: "." + RegionName + ".after");
2304
2305 {
2306 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2307 CGF.EmitStmt(S: RegionBodyStmt);
2308 }
2309
2310 if (Builder.saveIP().isValid())
2311 Builder.CreateBr(Dest: FiniBB);
2312}
2313
2314void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
2315 if (CGM.getLangOpts().OpenMPIRBuilder) {
2316 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2317 // Check if we have any if clause associated with the directive.
2318 llvm::Value *IfCond = nullptr;
2319 if (const auto *C = S.getSingleClause<OMPIfClause>())
2320 IfCond = EmitScalarExpr(E: C->getCondition(),
2321 /*IgnoreResultAssign=*/true);
2322
2323 llvm::Value *NumThreads = nullptr;
2324 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
2325 NumThreads = EmitScalarExpr(E: NumThreadsClause->getNumThreads().front(),
2326 /*IgnoreResultAssign=*/true);
2327
2328 ProcBindKind ProcBind = OMP_PROC_BIND_default;
2329 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
2330 ProcBind = ProcBindClause->getProcBindKind();
2331
2332 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2333
2334 // The cleanup callback that finalizes all variables at the given location,
2335 // thus calls destructors etc.
2336 auto FiniCB = [this](InsertPointTy IP) {
2337 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
2338 return llvm::Error::success();
2339 };
2340
2341 // Privatization callback that performs appropriate action for
2342 // shared/private/firstprivate/lastprivate/copyin/... variables.
2343 //
2344 // TODO: This defaults to shared right now.
2345 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
2346 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
2347 // The next line is appropriate only for variables (Val) with the
2348 // data-sharing attribute "shared".
2349 ReplVal = &Val;
2350
2351 return CodeGenIP;
2352 };
2353
2354 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
2355 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
2356
2357 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
2358 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
2359 OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
2360 CGF&: *this, RegionBodyStmt: ParallelRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "parallel");
2361 return llvm::Error::success();
2362 };
2363
2364 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
2365 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
2366 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
2367 AllocaInsertPt->getIterator());
2368 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2369 cantFail(ValOrErr: OMPBuilder.createParallel(
2370 Loc: Builder, AllocaIP, /*DeallocBlocks=*/{}, BodyGenCB, PrivCB, FiniCB,
2371 IfCondition: IfCond, NumThreads, ProcBind, IsCancellable: S.hasCancel()));
2372 Builder.restoreIP(IP: AfterIP);
2373 return;
2374 }
2375
2376 // Emit parallel region as a standalone region.
2377 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2378 Action.Enter(CGF);
2379 OMPPrivateScope PrivateScope(CGF);
2380 emitOMPCopyinClause(CGF, S);
2381 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
2382 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
2383 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
2384 (void)PrivateScope.Privatize();
2385 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_parallel)->getCapturedStmt());
2386 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
2387 };
2388 {
2389 auto LPCRegion =
2390 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
2391 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_parallel, CodeGen,
2392 CodeGenBoundParameters: emitEmptyBoundParameters);
2393 emitPostUpdateForReductionClause(CGF&: *this, D: S,
2394 CondGen: [](CodeGenFunction &) { return nullptr; });
2395 }
2396 // Check for outer lastprivate conditional update.
2397 checkForLastprivateConditionalUpdate(CGF&: *this, S);
2398}
2399
2400void CodeGenFunction::EmitOMPMetaDirective(const OMPMetaDirective &S) {
2401 EmitStmt(S: S.getIfStmt());
2402}
2403
2404namespace {
2405/// RAII to handle scopes for loop transformation directives.
2406class OMPTransformDirectiveScopeRAII {
2407 OMPLoopScope *Scope = nullptr;
2408 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
2409 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
2410
2411 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
2412 delete;
2413 OMPTransformDirectiveScopeRAII &
2414 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
2415
2416public:
2417 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
2418 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(Val: S)) {
2419 Scope = new OMPLoopScope(CGF, *Dir);
2420 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
2421 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2422 } else if (const auto *Dir =
2423 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(
2424 Val: S)) {
2425 // For simplicity we reuse the loop scope similarly to what we do with
2426 // OMPCanonicalLoopNestTransformationDirective do by being a subclass
2427 // of OMPLoopBasedDirective.
2428 Scope = new OMPLoopScope(CGF, *Dir);
2429 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
2430 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2431 }
2432 }
2433 ~OMPTransformDirectiveScopeRAII() {
2434 if (!Scope)
2435 return;
2436 delete CapInfoRAII;
2437 delete CGSI;
2438 delete Scope;
2439 }
2440};
2441} // namespace
2442
2443static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
2444 int MaxLevel, int Level = 0) {
2445 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
2446 const Stmt *SimplifiedS = S->IgnoreContainers();
2447 if (const auto *CS = dyn_cast<CompoundStmt>(Val: SimplifiedS)) {
2448 PrettyStackTraceLoc CrashInfo(
2449 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
2450 "LLVM IR generation of compound statement ('{}')");
2451
2452 // Keep track of the current cleanup stack depth, including debug scopes.
2453 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
2454 for (const Stmt *CurStmt : CS->body())
2455 emitBody(CGF, S: CurStmt, NextLoop, MaxLevel, Level);
2456 return;
2457 }
2458
2459 // `tryToFindNextInnerLoop` keeps the intra-tile hint wrapper around, so match
2460 // against the loop it annotates. The tile overshoot guard is emitted
2461 // separately via EmitOMPLoopBody's finals-conditions handling.
2462 if (SimplifiedS == OMPLoopBasedDirective::ignoreIntraTileHint(S: NextLoop)) {
2463 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(Val: SimplifiedS))
2464 SimplifiedS = Dir->getTransformedStmt();
2465 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Val: SimplifiedS))
2466 SimplifiedS = CanonLoop->getLoopStmt();
2467 if (const auto *For = dyn_cast<ForStmt>(Val: SimplifiedS)) {
2468 S = For->getBody();
2469 } else {
2470 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
2471 "Expected canonical for loop or range-based for loop.");
2472 const auto *CXXFor = cast<CXXForRangeStmt>(Val: SimplifiedS);
2473 CGF.EmitStmt(S: CXXFor->getLoopVarStmt());
2474 S = CXXFor->getBody();
2475 }
2476 if (Level + 1 < MaxLevel) {
2477 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2478 CurStmt: S, /*TryImperfectlyNestedLoops=*/true);
2479 emitBody(CGF, S, NextLoop, MaxLevel, Level: Level + 1);
2480 return;
2481 }
2482 }
2483 CGF.EmitStmt(S);
2484}
2485
2486void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
2487 JumpDest LoopExit) {
2488 RunCleanupsScope BodyScope(*this);
2489 // Update counters values on current iteration.
2490 for (const Expr *UE : D.updates())
2491 EmitIgnoredExpr(E: UE);
2492 // Update the linear variables.
2493 // In distribute directives only loop counters may be marked as linear, no
2494 // need to generate the code for them.
2495 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2496 if (!isOpenMPDistributeDirective(DKind: EKind)) {
2497 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2498 for (const Expr *UE : C->updates())
2499 EmitIgnoredExpr(E: UE);
2500 }
2501 }
2502
2503 // On a continue in the body, jump to the end.
2504 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.body.continue");
2505 BreakContinueStack.push_back(Elt: BreakContinue(D, LoopExit, Continue));
2506 for (const Expr *E : D.finals_conditions()) {
2507 if (!E)
2508 continue;
2509 // Check that loop counter in non-rectangular nest fits into the iteration
2510 // space.
2511 llvm::BasicBlock *NextBB = createBasicBlock(name: "omp.body.next");
2512 EmitBranchOnBoolExpr(Cond: E, TrueBlock: NextBB, FalseBlock: Continue.getBlock(),
2513 TrueCount: getProfileCount(S: D.getBody()));
2514 EmitBlock(BB: NextBB);
2515 }
2516
2517 OMPPrivateScope InscanScope(*this);
2518 EmitOMPReductionClauseInit(D, PrivateScope&: InscanScope, /*ForInscan=*/true);
2519 bool IsInscanRegion = InscanScope.Privatize();
2520 if (IsInscanRegion) {
2521 // Need to remember the block before and after scan directive
2522 // to dispatch them correctly depending on the clause used in
2523 // this directive, inclusive or exclusive. For inclusive scan the natural
2524 // order of the blocks is used, for exclusive clause the blocks must be
2525 // executed in reverse order.
2526 OMPBeforeScanBlock = createBasicBlock(name: "omp.before.scan.bb");
2527 OMPAfterScanBlock = createBasicBlock(name: "omp.after.scan.bb");
2528 // No need to allocate inscan exit block, in simd mode it is selected in the
2529 // codegen for the scan directive.
2530 if (EKind != OMPD_simd && !getLangOpts().OpenMPSimd)
2531 OMPScanExitBlock = createBasicBlock(name: "omp.exit.inscan.bb");
2532 OMPScanDispatch = createBasicBlock(name: "omp.inscan.dispatch");
2533 EmitBranch(Block: OMPScanDispatch);
2534 EmitBlock(BB: OMPBeforeScanBlock);
2535 }
2536
2537 // Emit loop variables for C++ range loops.
2538 const Stmt *Body =
2539 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2540 // Emit loop body.
2541 emitBody(CGF&: *this, S: Body,
2542 NextLoop: OMPLoopBasedDirective::tryToFindNextInnerLoop(
2543 CurStmt: Body, /*TryImperfectlyNestedLoops=*/true),
2544 MaxLevel: D.getLoopsNumber());
2545
2546 // Jump to the dispatcher at the end of the loop body.
2547 if (IsInscanRegion)
2548 EmitBranch(Block: OMPScanExitBlock);
2549
2550 // The end (updates/cleanups).
2551 EmitBlock(BB: Continue.getBlock());
2552 BreakContinueStack.pop_back();
2553}
2554
2555using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
2556
2557/// Emit a captured statement and return the function as well as its captured
2558/// closure context.
2559static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF,
2560 const CapturedStmt *S) {
2561 LValue CapStruct = ParentCGF.InitCapturedStruct(S: *S);
2562 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
2563 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2564 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(args: *S);
2565 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
2566 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S: *S);
2567
2568 return {F, CapStruct.getPointer(CGF&: ParentCGF)};
2569}
2570
2571/// Emit a call to a previously captured closure.
2572static llvm::CallInst *
2573emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap,
2574 llvm::ArrayRef<llvm::Value *> Args) {
2575 // Append the closure context to the argument.
2576 SmallVector<llvm::Value *> EffectiveArgs;
2577 EffectiveArgs.reserve(N: Args.size() + 1);
2578 llvm::append_range(C&: EffectiveArgs, R&: Args);
2579 EffectiveArgs.push_back(Elt: Cap.second);
2580
2581 return ParentCGF.Builder.CreateCall(Callee: Cap.first, Args: EffectiveArgs);
2582}
2583
2584llvm::CanonicalLoopInfo *
2585CodeGenFunction::EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth) {
2586 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
2587
2588 // The caller is processing the loop-associated directive processing the \p
2589 // Depth loops nested in \p S. Put the previous pending loop-associated
2590 // directive to the stack. If the current loop-associated directive is a loop
2591 // transformation directive, it will push its generated loops onto the stack
2592 // such that together with the loops left here they form the combined loop
2593 // nest for the parent loop-associated directive.
2594 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
2595 ExpectedOMPLoopDepth = Depth;
2596
2597 EmitStmt(S);
2598 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
2599
2600 // The last added loop is the outermost one.
2601 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
2602
2603 // Pop the \p Depth loops requested by the call from that stack and restore
2604 // the previous context.
2605 OMPLoopNestStack.pop_back_n(NumItems: Depth);
2606 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2607
2608 return Result;
2609}
2610
2611void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
2612 const Stmt *SyntacticalLoop = S->getLoopStmt();
2613 if (!getLangOpts().OpenMPIRBuilder) {
2614 // Ignore if OpenMPIRBuilder is not enabled.
2615 EmitStmt(S: SyntacticalLoop);
2616 return;
2617 }
2618
2619 LexicalScope ForScope(*this, S->getSourceRange());
2620
2621 // Emit init statements. The Distance/LoopVar funcs may reference variable
2622 // declarations they contain.
2623 const Stmt *BodyStmt;
2624 if (const auto *For = dyn_cast<ForStmt>(Val: SyntacticalLoop)) {
2625 if (const Stmt *InitStmt = For->getInit())
2626 EmitStmt(S: InitStmt);
2627 BodyStmt = For->getBody();
2628 } else if (const auto *RangeFor =
2629 dyn_cast<CXXForRangeStmt>(Val: SyntacticalLoop)) {
2630 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2631 EmitStmt(S: RangeStmt);
2632 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2633 EmitStmt(S: BeginStmt);
2634 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2635 EmitStmt(S: EndStmt);
2636 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2637 EmitStmt(S: LoopVarStmt);
2638 BodyStmt = RangeFor->getBody();
2639 } else
2640 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2641
2642 // Emit closure for later use. By-value captures will be captured here.
2643 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2644 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(ParentCGF&: *this, S: DistanceFunc);
2645 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2646 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(ParentCGF&: *this, S: LoopVarFunc);
2647
2648 // Call the distance function to get the number of iterations of the loop to
2649 // come.
2650 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2651 ->getParam(i: 0)
2652 ->getType()
2653 .getNonReferenceType();
2654 RawAddress CountAddr = CreateMemTemp(T: LogicalTy, Name: ".count.addr");
2655 emitCapturedStmtCall(ParentCGF&: *this, Cap: DistanceClosure, Args: {CountAddr.getPointer()});
2656 llvm::Value *DistVal = Builder.CreateLoad(Addr: CountAddr, Name: ".count");
2657
2658 // Emit the loop structure.
2659 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2660 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2661 llvm::Value *IndVar) {
2662 Builder.restoreIP(IP: CodeGenIP);
2663
2664 // Emit the loop body: Convert the logical iteration number to the loop
2665 // variable and emit the body.
2666 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2667 LValue LCVal = EmitLValue(E: LoopVarRef);
2668 Address LoopVarAddress = LCVal.getAddress();
2669 emitCapturedStmtCall(ParentCGF&: *this, Cap: LoopVarClosure,
2670 Args: {LoopVarAddress.emitRawPointer(CGF&: *this), IndVar});
2671
2672 RunCleanupsScope BodyScope(*this);
2673 EmitStmt(S: BodyStmt);
2674 return llvm::Error::success();
2675 };
2676
2677 llvm::CanonicalLoopInfo *CL =
2678 cantFail(ValOrErr: OMPBuilder.createCanonicalLoop(Loc: Builder, BodyGenCB: BodyGen, TripCount: DistVal));
2679
2680 // Finish up the loop.
2681 Builder.restoreIP(IP: CL->getAfterIP());
2682 ForScope.ForceCleanup();
2683
2684 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2685 OMPLoopNestStack.push_back(Elt: CL);
2686}
2687
2688void CodeGenFunction::EmitOMPInnerLoop(
2689 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2690 const Expr *IncExpr,
2691 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2692 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2693 auto LoopExit = getJumpDestInCurrentScope(Name: "omp.inner.for.end");
2694
2695 // Start the loop with a block that tests the condition.
2696 auto CondBlock = createBasicBlock(name: "omp.inner.for.cond");
2697 EmitBlock(BB: CondBlock);
2698 const SourceRange R = S.getSourceRange();
2699
2700 // If attributes are attached, push to the basic block with them.
2701 const auto &OMPED = cast<OMPExecutableDirective>(Val: S);
2702 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2703 const Stmt *SS = ICS->getCapturedStmt();
2704 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(Val: SS);
2705 OMPLoopNestStack.clear();
2706 if (AS)
2707 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
2708 Attrs: AS->getAttrs(), StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
2709 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
2710 else
2711 LoopStack.push(Header: CondBlock, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
2712 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
2713
2714 // If there are any cleanups between here and the loop-exit scope,
2715 // create a block to stage a loop exit along.
2716 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2717 if (RequiresCleanup)
2718 ExitBlock = createBasicBlock(name: "omp.inner.for.cond.cleanup");
2719
2720 llvm::BasicBlock *LoopBody = createBasicBlock(name: "omp.inner.for.body");
2721
2722 // Emit condition.
2723 EmitBranchOnBoolExpr(Cond: LoopCond, TrueBlock: LoopBody, FalseBlock: ExitBlock, TrueCount: getProfileCount(S: &S));
2724 if (ExitBlock != LoopExit.getBlock()) {
2725 EmitBlock(BB: ExitBlock);
2726 EmitBranchThroughCleanup(Dest: LoopExit);
2727 }
2728
2729 EmitBlock(BB: LoopBody);
2730 incrementProfileCounter(S: &S);
2731
2732 // Create a block for the increment.
2733 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.inner.for.inc");
2734 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
2735
2736 BodyGen(*this);
2737
2738 // Emit "IV = IV + 1" and a back-edge to the condition block.
2739 EmitBlock(BB: Continue.getBlock());
2740 EmitIgnoredExpr(E: IncExpr);
2741 PostIncGen(*this);
2742 BreakContinueStack.pop_back();
2743 EmitBranch(Block: CondBlock);
2744 LoopStack.pop();
2745 // Emit the fall-through block.
2746 EmitBlock(BB: LoopExit.getBlock());
2747}
2748
2749bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
2750 if (!HaveInsertPoint())
2751 return false;
2752 // Emit inits for the linear variables.
2753 bool HasLinears = false;
2754 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2755 for (const Expr *Init : C->inits()) {
2756 HasLinears = true;
2757 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Init)->getDecl());
2758 if (const auto *Ref =
2759 dyn_cast<DeclRefExpr>(Val: VD->getInit()->IgnoreImpCasts())) {
2760 if (isa<BindingDecl>(Val: Ref->getDecl())) {
2761 EmitVarDecl(D: *VD);
2762 } else {
2763 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
2764 const auto *OrigVD = cast<VarDecl>(Val: Ref->getDecl());
2765 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2766 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
2767 VD->getInit()->getType(), VK_LValue,
2768 VD->getInit()->getExprLoc());
2769 EmitExprAsInit(
2770 init: &DRE, D: VD,
2771 lvalue: MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: VD->getType()),
2772 /*capturedByInit=*/false);
2773 EmitAutoVarCleanups(emission: Emission);
2774 }
2775 } else {
2776 EmitVarDecl(D: *VD);
2777 }
2778 }
2779 // Emit the linear steps for the linear clauses.
2780 // If a step is not constant, it is pre-calculated before the loop.
2781 if (const auto *CS = cast_or_null<BinaryOperator>(Val: C->getCalcStep()))
2782 if (const auto *SaveRef = cast<DeclRefExpr>(Val: CS->getLHS())) {
2783 EmitVarDecl(D: *cast<VarDecl>(Val: SaveRef->getDecl()));
2784 // Emit calculation of the linear step.
2785 EmitIgnoredExpr(E: CS);
2786 }
2787 }
2788 return HasLinears;
2789}
2790
2791void CodeGenFunction::EmitOMPLinearClauseFinal(
2792 const OMPLoopDirective &D,
2793 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2794 if (!HaveInsertPoint())
2795 return;
2796 llvm::BasicBlock *DoneBB = nullptr;
2797 // Emit the final values of the linear variables.
2798 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2799 auto IC = C->varlist_begin();
2800 for (const Expr *F : C->finals()) {
2801 if (!DoneBB) {
2802 if (llvm::Value *Cond = CondGen(*this)) {
2803 // If the first post-update expression is found, emit conditional
2804 // block if it was requested.
2805 llvm::BasicBlock *ThenBB = createBasicBlock(name: ".omp.linear.pu");
2806 DoneBB = createBasicBlock(name: ".omp.linear.pu.done");
2807 Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
2808 EmitBlock(BB: ThenBB);
2809 }
2810 }
2811 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IC)->getDecl();
2812 Address OrigAddr = Address::invalid();
2813 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
2814 OrigAddr = EmitOMPBindingOriginalAddr(BD, Loc: (*IC)->getExprLoc());
2815 } else {
2816 const auto *OrigVD = cast<VarDecl>(Val: OrigDecl);
2817 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2818 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
2819 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2820 OrigAddr = EmitLValue(E: &DRE).getAddress();
2821 }
2822 CodeGenFunction::OMPPrivateScope VarScope(*this);
2823 VarScope.addPrivate(LocalVD: OrigDecl, Addr: OrigAddr);
2824 (void)VarScope.Privatize();
2825 EmitIgnoredExpr(E: F);
2826 ++IC;
2827 }
2828 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2829 EmitIgnoredExpr(E: PostUpdate);
2830 }
2831 if (DoneBB)
2832 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
2833}
2834
2835static void emitAlignedClause(CodeGenFunction &CGF,
2836 const OMPExecutableDirective &D) {
2837 if (!CGF.HaveInsertPoint())
2838 return;
2839 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2840 llvm::APInt ClauseAlignment(64, 0);
2841 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2842 auto *AlignmentCI =
2843 cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AlignmentExpr));
2844 ClauseAlignment = AlignmentCI->getValue();
2845 }
2846 for (const Expr *E : Clause->varlist()) {
2847 llvm::APInt Alignment(ClauseAlignment);
2848 if (Alignment == 0) {
2849 // OpenMP [2.8.1, Description]
2850 // If no optional parameter is specified, implementation-defined default
2851 // alignments for SIMD instructions on the target platforms are assumed.
2852 Alignment =
2853 CGF.getContext()
2854 .toCharUnitsFromBits(BitSize: CGF.getContext().getOpenMPDefaultSimdAlign(
2855 T: E->getType()->getPointeeType()))
2856 .getQuantity();
2857 }
2858 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2859 "alignment is not power of 2");
2860 if (Alignment != 0) {
2861 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2862 CGF.emitAlignmentAssumption(
2863 PtrValue, E, /*No second loc needed*/ AssumptionLoc: SourceLocation(),
2864 Alignment: llvm::ConstantInt::get(Context&: CGF.getLLVMContext(), V: Alignment));
2865 }
2866 }
2867 }
2868}
2869
2870void CodeGenFunction::EmitOMPPrivateLoopCounters(
2871 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
2872 if (!HaveInsertPoint())
2873 return;
2874 auto I = S.private_counters().begin();
2875 for (const Expr *E : S.counters()) {
2876 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
2877 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
2878 // Emit var without initialization.
2879 AutoVarEmission VarEmission = EmitAutoVarAlloca(var: *PrivateVD);
2880 EmitAutoVarCleanups(emission: VarEmission);
2881 LocalDeclMap.erase(Val: PrivateVD);
2882 (void)LoopScope.addPrivate(LocalVD: VD, Addr: VarEmission.getAllocatedAddress());
2883 if (LocalDeclMap.count(Val: VD) || CapturedStmtInfo->lookup(VD) ||
2884 VD->hasGlobalStorage()) {
2885 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2886 LocalDeclMap.count(Val: VD) || CapturedStmtInfo->lookup(VD),
2887 E->getType(), VK_LValue, E->getExprLoc());
2888 (void)LoopScope.addPrivate(LocalVD: PrivateVD, Addr: EmitLValue(E: &DRE).getAddress());
2889 } else {
2890 (void)LoopScope.addPrivate(LocalVD: PrivateVD, Addr: VarEmission.getAllocatedAddress());
2891 }
2892 ++I;
2893 }
2894 // Privatize extra loop counters used in loops for ordered(n) clauses.
2895 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2896 if (!C->getNumForLoops())
2897 continue;
2898 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2899 I < E; ++I) {
2900 const auto *DRE = cast<DeclRefExpr>(Val: C->getLoopCounter(NumLoop: I));
2901 const auto *VD = cast<VarDecl>(Val: DRE->getDecl());
2902 // Override only those variables that can be captured to avoid re-emission
2903 // of the variables declared within the loops.
2904 if (DRE->refersToEnclosingVariableOrCapture()) {
2905 (void)LoopScope.addPrivate(
2906 LocalVD: VD, Addr: CreateMemTemp(T: DRE->getType(), Name: VD->getName()));
2907 }
2908 }
2909 }
2910}
2911
2912static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
2913 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2914 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2915 if (!CGF.HaveInsertPoint())
2916 return;
2917 {
2918 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2919 CGF.EmitOMPPrivateLoopCounters(S, LoopScope&: PreCondScope);
2920 (void)PreCondScope.Privatize();
2921 // Get initial values of real counters.
2922 for (const Expr *I : S.inits()) {
2923 CGF.EmitIgnoredExpr(E: I);
2924 }
2925 }
2926 // Create temp loop control variables with their init values to support
2927 // non-rectangular loops.
2928 CodeGenFunction::OMPMapVars PreCondVars;
2929 for (const Expr *E : S.dependent_counters()) {
2930 if (!E)
2931 continue;
2932 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2933 "dependent counter must not be an iterator.");
2934 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
2935 Address CounterAddr =
2936 CGF.CreateMemTemp(T: VD->getType().getNonReferenceType());
2937 (void)PreCondVars.setVarAddr(CGF, LocalVD: VD, TempAddr: CounterAddr);
2938 }
2939 (void)PreCondVars.apply(CGF);
2940 for (const Expr *E : S.dependent_inits()) {
2941 if (!E)
2942 continue;
2943 CGF.EmitIgnoredExpr(E);
2944 }
2945 // Check that loop is executed at least one time.
2946 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2947 PreCondVars.restore(CGF);
2948}
2949
2950void CodeGenFunction::EmitOMPLinearClause(
2951 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2952 if (!HaveInsertPoint())
2953 return;
2954 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2955 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2956 if (isOpenMPSimdDirective(DKind: EKind)) {
2957 const auto *LoopDirective = cast<OMPLoopDirective>(Val: &D);
2958 for (const Expr *C : LoopDirective->counters()) {
2959 SIMDLCVs.insert(
2960 V: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: C)->getDecl())->getCanonicalDecl());
2961 }
2962 }
2963 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2964 auto CurPrivate = C->privates().begin();
2965 for (const Expr *E : C->varlist()) {
2966 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
2967 const auto *PrivateVD =
2968 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *CurPrivate)->getDecl());
2969 bool IsSIMDLCV = false;
2970 if (const auto *VarD = dyn_cast<VarDecl>(Val: VD))
2971 IsSIMDLCV = SIMDLCVs.count(V: VarD->getCanonicalDecl());
2972 if (!IsSIMDLCV) {
2973 // Emit private VarDecl with copy init.
2974 EmitVarDecl(D: *PrivateVD);
2975 Address PrivateAddr = GetAddrOfLocalVar(VD: PrivateVD);
2976 bool IsRegistered = PrivateScope.addPrivate(LocalVD: VD, Addr: PrivateAddr);
2977 assert(IsRegistered && "linear var already registered as private");
2978 // Silence the warning about unused variable.
2979 (void)IsRegistered;
2980 } else {
2981 EmitVarDecl(D: *PrivateVD);
2982 }
2983 ++CurPrivate;
2984 }
2985 }
2986}
2987
2988static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
2989 const OMPExecutableDirective &D) {
2990 if (!CGF.HaveInsertPoint())
2991 return;
2992 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
2993 RValue Len = CGF.EmitAnyExpr(E: C->getSimdlen(), aggSlot: AggValueSlot::ignored(),
2994 /*ignoreResult=*/true);
2995 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
2996 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2997 // In presence of finite 'safelen', it may be unsafe to mark all
2998 // the memory instructions parallel, because loop-carried
2999 // dependences of 'safelen' iterations are possible.
3000 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
3001 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
3002 RValue Len = CGF.EmitAnyExpr(E: C->getSafelen(), aggSlot: AggValueSlot::ignored(),
3003 /*ignoreResult=*/true);
3004 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3005 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
3006 // In presence of finite 'safelen', it may be unsafe to mark all
3007 // the memory instructions parallel, because loop-carried
3008 // dependences of 'safelen' iterations are possible.
3009 CGF.LoopStack.setParallel(/*Enable=*/false);
3010 }
3011}
3012
3013// Check for the presence of an `OMPOrderedBlockAssocDirective`,
3014// i.e., `ordered` in `#pragma omp ordered simd`.
3015//
3016// Consider the following source code:
3017// ```
3018// __attribute__((noinline)) void omp_simd_loop(float X[ARRAY_SIZE][ARRAY_SIZE])
3019// {
3020// for (int r = 1; r < ARRAY_SIZE; ++r) {
3021// for (int c = 1; c < ARRAY_SIZE; ++c) {
3022// #pragma omp simd
3023// for (int k = 2; k < ARRAY_SIZE; ++k) {
3024// #pragma omp ordered simd
3025// X[r][k] = X[r][k - 2] + sinf((float)(r / c));
3026// }
3027// }
3028// }
3029// }
3030// ```
3031//
3032// Suppose we are in `CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective
3033// &D)`. By examining `D.dump()` we have the following AST containing
3034// `OMPOrderedBlockAssocDirective`:
3035//
3036// ```
3037// OMPSimdDirective 0x1c32950
3038// `-CapturedStmt 0x1c32028
3039// |-CapturedDecl 0x1c310e8
3040// | |-ForStmt 0x1c31e30
3041// | | |-DeclStmt 0x1c31298
3042// | | | `-VarDecl 0x1c31208 used k 'int' cinit
3043// | | | `-IntegerLiteral 0x1c31278 'int' 2
3044// | | |-<<<NULL>>>
3045// | | |-BinaryOperator 0x1c31308 'int' '<'
3046// | | | |-ImplicitCastExpr 0x1c312f0 'int' <LValueToRValue>
3047// | | | | `-DeclRefExpr 0x1c312b0 'int' lvalue Var 0x1c31208 'k' 'int'
3048// | | | `-IntegerLiteral 0x1c312d0 'int' 256
3049// | | |-UnaryOperator 0x1c31348 'int' prefix '++'
3050// | | | `-DeclRefExpr 0x1c31328 'int' lvalue Var 0x1c31208 'k' 'int'
3051// | | `-CompoundStmt 0x1c31e18
3052// | | `-OMPOrderedBlockAssocDirective 0x1c31dd8
3053// | | |-OMPSimdClause 0x1c31380
3054// | | `-CapturedStmt 0x1c31cd0
3055// ```
3056//
3057// Note the presence of `OMPOrderedBlockAssocDirective` above:
3058// It's (transitively) nested in a `CapturedStmt` representing the pragma
3059// annotated compound statement. Thus, we need to consider this nesting and
3060// include checking the `getCapturedStmt` in this case.
3061static bool hasOrderedBlockAssocDirective(const Stmt *S) {
3062 if (isa<OMPOrderedBlockAssocDirective>(Val: S))
3063 return true;
3064
3065 if (const auto *CS = dyn_cast<CapturedStmt>(Val: S))
3066 return hasOrderedBlockAssocDirective(S: CS->getCapturedStmt());
3067
3068 for (const Stmt *Child : S->children()) {
3069 if (Child && hasOrderedBlockAssocDirective(S: Child))
3070 return true;
3071 }
3072
3073 return false;
3074}
3075
3076static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt,
3077 LoopInfoStack &LoopStack) {
3078 // Check for the presence of an `OMPOrderedBlockAssocDirective`
3079 // i.e., `ordered` in `#pragma omp ordered simd`
3080 bool HasOrderedDirective = hasOrderedBlockAssocDirective(S: &AssociatedStmt);
3081 // If present then conservatively disable loop vectorization
3082 // analogously to how `emitSimdlenSafelenClause` does.
3083 if (HasOrderedDirective)
3084 LoopStack.setParallel(/*Enable=*/false);
3085}
3086
3087void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) {
3088 // Walk clauses and process safelen/lastprivate.
3089 LoopStack.setParallel(/*Enable=*/true);
3090 LoopStack.setVectorizeEnable();
3091 const Stmt *AssociatedStmt = D.getAssociatedStmt();
3092 applyConservativeSimdOrderedDirective(AssociatedStmt: *AssociatedStmt, LoopStack);
3093 emitSimdlenSafelenClause(CGF&: *this, D);
3094 if (const auto *C = D.getSingleClause<OMPOrderClause>())
3095 if (C->getKind() == OMPC_ORDER_concurrent)
3096 LoopStack.setParallel(/*Enable=*/true);
3097 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
3098 if ((EKind == OMPD_simd ||
3099 (getLangOpts().OpenMPSimd && isOpenMPSimdDirective(DKind: EKind))) &&
3100 llvm::any_of(Range: D.getClausesOfKind<OMPReductionClause>(),
3101 P: [](const OMPReductionClause *C) {
3102 return C->getModifier() == OMPC_REDUCTION_inscan;
3103 }))
3104 // Disable parallel access in case of prefix sum.
3105 LoopStack.setParallel(/*Enable=*/false);
3106}
3107
3108void CodeGenFunction::EmitOMPSimdFinal(
3109 const OMPLoopDirective &D,
3110 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
3111 if (!HaveInsertPoint())
3112 return;
3113 llvm::BasicBlock *DoneBB = nullptr;
3114 auto IC = D.counters().begin();
3115 auto IPC = D.private_counters().begin();
3116 for (const Expr *F : D.finals()) {
3117 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: (*IC))->getDecl());
3118 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: (*IPC))->getDecl());
3119 const auto *CED = dyn_cast<OMPCapturedExprDecl>(Val: OrigVD);
3120 if (LocalDeclMap.count(Val: OrigVD) || CapturedStmtInfo->lookup(VD: OrigVD) ||
3121 OrigVD->hasGlobalStorage() || CED) {
3122 if (!DoneBB) {
3123 if (llvm::Value *Cond = CondGen(*this)) {
3124 // If the first post-update expression is found, emit conditional
3125 // block if it was requested.
3126 llvm::BasicBlock *ThenBB = createBasicBlock(name: ".omp.final.then");
3127 DoneBB = createBasicBlock(name: ".omp.final.done");
3128 Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
3129 EmitBlock(BB: ThenBB);
3130 }
3131 }
3132 Address OrigAddr = Address::invalid();
3133 if (CED) {
3134 OrigAddr = EmitLValue(E: CED->getInit()->IgnoreImpCasts()).getAddress();
3135 } else {
3136 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
3137 /*RefersToEnclosingVariableOrCapture=*/false,
3138 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
3139 OrigAddr = EmitLValue(E: &DRE).getAddress();
3140 }
3141 OMPPrivateScope VarScope(*this);
3142 VarScope.addPrivate(LocalVD: OrigVD, Addr: OrigAddr);
3143 (void)VarScope.Privatize();
3144 EmitIgnoredExpr(E: F);
3145 }
3146 ++IC;
3147 ++IPC;
3148 }
3149 if (DoneBB)
3150 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
3151}
3152
3153static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
3154 const OMPLoopDirective &S,
3155 CodeGenFunction::JumpDest LoopExit) {
3156 CGF.EmitOMPLoopBody(D: S, LoopExit);
3157 CGF.EmitStopPoint(S: &S);
3158}
3159
3160/// Emit a helper variable and return corresponding lvalue.
3161static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
3162 const DeclRefExpr *Helper) {
3163 auto VDecl = cast<VarDecl>(Val: Helper->getDecl());
3164 CGF.EmitVarDecl(D: *VDecl);
3165 return CGF.EmitLValue(E: Helper);
3166}
3167
3168static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
3169 const RegionCodeGenTy &SimdInitGen,
3170 const RegionCodeGenTy &BodyCodeGen) {
3171 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
3172 PrePostActionTy &) {
3173 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
3174 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3175 SimdInitGen(CGF);
3176
3177 BodyCodeGen(CGF);
3178 };
3179 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
3180 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
3181 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
3182
3183 BodyCodeGen(CGF);
3184 };
3185 const Expr *IfCond = nullptr;
3186 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3187 if (isOpenMPSimdDirective(DKind: EKind)) {
3188 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3189 if (CGF.getLangOpts().OpenMP >= 50 &&
3190 (C->getNameModifier() == OMPD_unknown ||
3191 C->getNameModifier() == OMPD_simd)) {
3192 IfCond = C->getCondition();
3193 break;
3194 }
3195 }
3196 }
3197 if (IfCond) {
3198 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
3199 } else {
3200 RegionCodeGenTy ThenRCG(ThenGen);
3201 ThenRCG(CGF);
3202 }
3203}
3204
3205static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
3206 PrePostActionTy &Action) {
3207 Action.Enter(CGF);
3208 OMPLoopScope PreInitScope(CGF, S);
3209 // if (PreCond) {
3210 // for (IV in 0..LastIteration) BODY;
3211 // <Final counter/linear vars updates>;
3212 // }
3213
3214 // The presence of lower/upper bound variable depends on the actual directive
3215 // kind in the AST node. The variables must be emitted because some of the
3216 // expressions associated with the loop will use them.
3217 OpenMPDirectiveKind DKind = S.getDirectiveKind();
3218 if (isOpenMPDistributeDirective(DKind) ||
3219 isOpenMPWorksharingDirective(DKind) || isOpenMPTaskLoopDirective(DKind) ||
3220 isOpenMPGenericLoopDirective(DKind)) {
3221 (void)EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: S.getLowerBoundVariable()));
3222 (void)EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: S.getUpperBoundVariable()));
3223 }
3224
3225 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3226 // Emit: if (PreCond) - begin.
3227 // If the condition constant folds and can be elided, avoid emitting the
3228 // whole loop.
3229 bool CondConstant;
3230 llvm::BasicBlock *ContBlock = nullptr;
3231 if (CGF.ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
3232 if (!CondConstant)
3233 return;
3234 } else {
3235 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "simd.if.then");
3236 ContBlock = CGF.createBasicBlock(name: "simd.if.end");
3237 emitPreCond(CGF, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
3238 TrueCount: CGF.getProfileCount(S: &S));
3239 CGF.EmitBlock(BB: ThenBlock);
3240 CGF.incrementProfileCounter(S: &S);
3241 }
3242
3243 // Emit the loop iteration variable.
3244 const Expr *IVExpr = S.getIterationVariable();
3245 const auto *IVDecl = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IVExpr)->getDecl());
3246 CGF.EmitVarDecl(D: *IVDecl);
3247 CGF.EmitIgnoredExpr(E: S.getInit());
3248
3249 // Emit the iterations count variable.
3250 // If it is not a variable, Sema decided to calculate iterations count on
3251 // each iteration (e.g., it is foldable into a constant).
3252 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
3253 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
3254 // Emit calculation of the iterations count.
3255 CGF.EmitIgnoredExpr(E: S.getCalcLastIteration());
3256 }
3257
3258 emitAlignedClause(CGF, D: S);
3259 (void)CGF.EmitOMPLinearClauseInit(D: S);
3260 {
3261 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
3262 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
3263 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
3264 CGF.EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
3265 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
3266 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
3267 CGF, S, CGF.EmitLValue(E: S.getIterationVariable()));
3268 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
3269 (void)LoopScope.Privatize();
3270 if (isOpenMPTargetExecutionDirective(DKind: EKind))
3271 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
3272
3273 emitCommonSimdLoop(
3274 CGF, S,
3275 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3276 CGF.EmitOMPSimdInit(D: S);
3277 },
3278 BodyCodeGen: [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3279 CGF.EmitOMPInnerLoop(
3280 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: S.getCond(), IncExpr: S.getInc(),
3281 BodyGen: [&S](CodeGenFunction &CGF) {
3282 emitOMPLoopBodyWithStopPoint(CGF, S,
3283 LoopExit: CodeGenFunction::JumpDest());
3284 },
3285 PostIncGen: [](CodeGenFunction &) {});
3286 });
3287 CGF.EmitOMPSimdFinal(D: S, CondGen: [](CodeGenFunction &) { return nullptr; });
3288 // Emit final copy of the lastprivate variables at the end of loops.
3289 if (HasLastprivateClause)
3290 CGF.EmitOMPLastprivateClauseFinal(D: S, /*NoFinals=*/true);
3291 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_simd);
3292 emitPostUpdateForReductionClause(CGF, D: S,
3293 CondGen: [](CodeGenFunction &) { return nullptr; });
3294 LoopScope.restoreMap();
3295 CGF.EmitOMPLinearClauseFinal(D: S, CondGen: [](CodeGenFunction &) { return nullptr; });
3296 }
3297 // Emit: if (PreCond) - end.
3298 if (ContBlock) {
3299 CGF.EmitBranch(Block: ContBlock);
3300 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
3301 }
3302}
3303
3304// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3305// available for "loop bind(thread)", which maps to "simd".
3306static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S) {
3307 // Check for unsupported clauses
3308 for (OMPClause *C : S.clauses()) {
3309 // Currently only order, simdlen and safelen clauses are supported
3310 if (!(isa<OMPSimdlenClause>(Val: C) || isa<OMPSafelenClause>(Val: C) ||
3311 isa<OMPOrderClause>(Val: C) || isa<OMPAlignedClause>(Val: C)))
3312 return false;
3313 }
3314
3315 // Check if we have a statement with the ordered-blockassoc directive.
3316 // Visit the statement hierarchy to find a compound statement
3317 // with a ordered-blockassoc directive in it.
3318 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Val: S.getRawStmt())) {
3319 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
3320 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
3321 if (!SubStmt)
3322 continue;
3323 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: SubStmt)) {
3324 for (const Stmt *CSSubStmt : CS->children()) {
3325 if (!CSSubStmt)
3326 continue;
3327 if (isa<OMPOrderedBlockAssocDirective>(Val: CSSubStmt)) {
3328 return false;
3329 }
3330 }
3331 }
3332 }
3333 }
3334 }
3335 return true;
3336}
3337
3338static llvm::MapVector<llvm::Value *, llvm::Value *>
3339GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF) {
3340 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
3341 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
3342 llvm::APInt ClauseAlignment(64, 0);
3343 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
3344 auto *AlignmentCI =
3345 cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AlignmentExpr));
3346 ClauseAlignment = AlignmentCI->getValue();
3347 }
3348 for (const Expr *E : Clause->varlist()) {
3349 llvm::APInt Alignment(ClauseAlignment);
3350 if (Alignment == 0) {
3351 // OpenMP [2.8.1, Description]
3352 // If no optional parameter is specified, implementation-defined default
3353 // alignments for SIMD instructions on the target platforms are assumed.
3354 Alignment =
3355 CGF.getContext()
3356 .toCharUnitsFromBits(BitSize: CGF.getContext().getOpenMPDefaultSimdAlign(
3357 T: E->getType()->getPointeeType()))
3358 .getQuantity();
3359 }
3360 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
3361 "alignment is not power of 2");
3362 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
3363 AlignedVars[PtrValue] = CGF.Builder.getInt64(C: Alignment.getSExtValue());
3364 }
3365 }
3366 return AlignedVars;
3367}
3368
3369// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3370// available for "loop bind(thread)", which maps to "simd".
3371static void emitOMPSimdDirective(const OMPLoopDirective &S,
3372 CodeGenFunction &CGF, CodeGenModule &CGM) {
3373 bool UseOMPIRBuilder =
3374 CGM.getLangOpts().OpenMPIRBuilder && isSimdSupportedByOpenMPIRBuilder(S);
3375 if (UseOMPIRBuilder) {
3376 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,
3377 PrePostActionTy &) {
3378 // Use the OpenMPIRBuilder if enabled.
3379 if (UseOMPIRBuilder) {
3380 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
3381 GetAlignedMapping(S, CGF);
3382 // Emit the associated statement and get its loop representation.
3383 const Stmt *Inner = S.getRawStmt();
3384 llvm::CanonicalLoopInfo *CLI =
3385 CGF.EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
3386
3387 llvm::OpenMPIRBuilder &OMPBuilder =
3388 CGM.getOpenMPRuntime().getOMPBuilder();
3389 // Add SIMD specific metadata
3390 llvm::ConstantInt *Simdlen = nullptr;
3391 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
3392 RValue Len = CGF.EmitAnyExpr(E: C->getSimdlen(), aggSlot: AggValueSlot::ignored(),
3393 /*ignoreResult=*/true);
3394 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3395 Simdlen = Val;
3396 }
3397 llvm::ConstantInt *Safelen = nullptr;
3398 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
3399 RValue Len = CGF.EmitAnyExpr(E: C->getSafelen(), aggSlot: AggValueSlot::ignored(),
3400 /*ignoreResult=*/true);
3401 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3402 Safelen = Val;
3403 }
3404 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
3405 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3406 if (C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
3407 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
3408 }
3409 }
3410 // Add simd metadata to the collapsed loop. Do not generate
3411 // another loop for if clause. Support for if clause is done earlier.
3412 OMPBuilder.applySimd(Loop: CLI, AlignedVars,
3413 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
3414 return;
3415 }
3416 };
3417 {
3418 auto LPCRegion =
3419 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
3420 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3421 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd,
3422 CodeGen: CodeGenIRBuilder);
3423 }
3424 return;
3425 }
3426
3427 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3428 CGF.OMPFirstScanLoop = true;
3429 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3430 emitOMPSimdRegion(CGF, S, Action);
3431 };
3432 {
3433 auto LPCRegion =
3434 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
3435 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3436 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd, CodeGen);
3437 }
3438 // Check for outer lastprivate conditional update.
3439 checkForLastprivateConditionalUpdate(CGF, S);
3440}
3441
3442void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
3443 emitOMPSimdDirective(S, CGF&: *this, CGM);
3444}
3445
3446void CodeGenFunction::EmitOMPTileDirective(const OMPTileDirective &S) {
3447 // Emit the de-sugared statement.
3448 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
3449 EmitStmt(S: S.getTransformedStmt());
3450}
3451
3452void CodeGenFunction::EmitOMPStripeDirective(const OMPStripeDirective &S) {
3453 // Emit the de-sugared statement.
3454 OMPTransformDirectiveScopeRAII StripeScope(*this, &S);
3455 EmitStmt(S: S.getTransformedStmt());
3456}
3457
3458void CodeGenFunction::EmitOMPReverseDirective(const OMPReverseDirective &S) {
3459 // Emit the de-sugared statement.
3460 OMPTransformDirectiveScopeRAII ReverseScope(*this, &S);
3461 EmitStmt(S: S.getTransformedStmt());
3462}
3463
3464void CodeGenFunction::EmitOMPSplitDirective(const OMPSplitDirective &S) {
3465 // Emit the de-sugared statement (the split loops).
3466 OMPTransformDirectiveScopeRAII SplitScope(*this, &S);
3467 EmitStmt(S: S.getTransformedStmt());
3468}
3469
3470void CodeGenFunction::EmitOMPInterchangeDirective(
3471 const OMPInterchangeDirective &S) {
3472 // Emit the de-sugared statement.
3473 OMPTransformDirectiveScopeRAII InterchangeScope(*this, &S);
3474 EmitStmt(S: S.getTransformedStmt());
3475}
3476
3477void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
3478 // Emit the de-sugared statement.
3479 OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
3480 EmitStmt(S: S.getTransformedStmt());
3481 EmitStmt(S: S.getFinals());
3482}
3483
3484void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
3485 // Emit the de-sugared statement
3486 OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
3487 EmitStmt(S: S.getTransformedStmt());
3488}
3489
3490void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
3491 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
3492
3493 if (UseOMPIRBuilder) {
3494 auto DL = SourceLocToDebugLoc(Location: S.getBeginLoc());
3495 const Stmt *Inner = S.getRawStmt();
3496
3497 // Consume nested loop. Clear the entire remaining loop stack because a
3498 // fully unrolled loop is non-transformable. For partial unrolling the
3499 // generated outer loop is pushed back to the stack.
3500 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
3501 OMPLoopNestStack.clear();
3502
3503 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
3504
3505 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
3506 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
3507
3508 if (S.hasClausesOfKind<OMPFullClause>()) {
3509 assert(ExpectedOMPLoopDepth == 0);
3510 OMPBuilder.unrollLoopFull(DL, Loop: CLI);
3511 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3512 uint64_t Factor = 0;
3513 if (Expr *FactorExpr = PartialClause->getFactor()) {
3514 Factor =
3515 FactorExpr->EvaluateKnownConstInt(Ctx: getContext()).getLimitedValue();
3516 assert(Factor >= 1 && "Only positive factors are valid");
3517 }
3518 OMPBuilder.unrollLoopPartial(DL, Loop: CLI, Factor,
3519 UnrolledCLI: NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
3520 } else {
3521 OMPBuilder.unrollLoopHeuristic(DL, Loop: CLI);
3522 }
3523
3524 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3525 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3526 if (UnrolledCLI)
3527 OMPLoopNestStack.push_back(Elt: UnrolledCLI);
3528
3529 return;
3530 }
3531
3532 // This function is only called if the unrolled loop is not consumed by any
3533 // other loop-associated construct. Such a loop-associated construct will have
3534 // used the transformed AST.
3535
3536 // Set the unroll metadata for the next emitted loop.
3537 LoopStack.setUnrollState(LoopAttributes::Enable);
3538
3539 if (S.hasClausesOfKind<OMPFullClause>()) {
3540 LoopStack.setUnrollState(LoopAttributes::Full);
3541 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3542 if (Expr *FactorExpr = PartialClause->getFactor()) {
3543 uint64_t Factor =
3544 FactorExpr->EvaluateKnownConstInt(Ctx: getContext()).getLimitedValue();
3545 assert(Factor >= 1 && "Only positive factors are valid");
3546 LoopStack.setUnrollCount(Factor);
3547 }
3548 }
3549
3550 EmitStmt(S: S.getAssociatedStmt());
3551}
3552
3553void CodeGenFunction::EmitOMPOuterLoop(
3554 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
3555 CodeGenFunction::OMPPrivateScope &LoopScope,
3556 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3557 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
3558 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
3559 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3560
3561 const Expr *IVExpr = S.getIterationVariable();
3562 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3563 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3564
3565 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "omp.dispatch.end");
3566
3567 // Start the loop with a block that tests the condition.
3568 llvm::BasicBlock *CondBlock = createBasicBlock(name: "omp.dispatch.cond");
3569 EmitBlock(BB: CondBlock);
3570 const SourceRange R = S.getSourceRange();
3571 OMPLoopNestStack.clear();
3572 LoopStack.push(Header: CondBlock, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
3573 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
3574
3575 llvm::Value *BoolCondVal = nullptr;
3576 if (!DynamicOrOrdered) {
3577 // UB = min(UB, GlobalUB) or
3578 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
3579 // 'distribute parallel for')
3580 EmitIgnoredExpr(E: LoopArgs.EUB);
3581 // IV = LB
3582 EmitIgnoredExpr(E: LoopArgs.Init);
3583 // IV < UB
3584 BoolCondVal = EvaluateExprAsBool(E: LoopArgs.Cond);
3585 } else {
3586 BoolCondVal =
3587 RT.emitForNext(CGF&: *this, Loc: S.getBeginLoc(), IVSize, IVSigned, IL: LoopArgs.IL,
3588 LB: LoopArgs.LB, UB: LoopArgs.UB, ST: LoopArgs.ST);
3589 }
3590
3591 // If there are any cleanups between here and the loop-exit scope,
3592 // create a block to stage a loop exit along.
3593 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
3594 if (LoopScope.requiresCleanups())
3595 ExitBlock = createBasicBlock(name: "omp.dispatch.cleanup");
3596
3597 llvm::BasicBlock *LoopBody = createBasicBlock(name: "omp.dispatch.body");
3598 Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock);
3599 if (ExitBlock != LoopExit.getBlock()) {
3600 EmitBlock(BB: ExitBlock);
3601 EmitBranchThroughCleanup(Dest: LoopExit);
3602 }
3603 EmitBlock(BB: LoopBody);
3604
3605 // Emit "IV = LB" (in case of static schedule, we have already calculated new
3606 // LB for loop condition and emitted it above).
3607 if (DynamicOrOrdered)
3608 EmitIgnoredExpr(E: LoopArgs.Init);
3609
3610 // Create a block for the increment.
3611 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.dispatch.inc");
3612 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
3613
3614 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3615 emitCommonSimdLoop(
3616 CGF&: *this, S,
3617 SimdInitGen: [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3618 // Generate !llvm.loop.parallel metadata for loads and stores for loops
3619 // with dynamic/guided scheduling and without ordered clause.
3620 if (!isOpenMPSimdDirective(DKind: EKind)) {
3621 CGF.LoopStack.setParallel(!IsMonotonic);
3622 if (const auto *C = S.getSingleClause<OMPOrderClause>())
3623 if (C->getKind() == OMPC_ORDER_concurrent)
3624 CGF.LoopStack.setParallel(/*Enable=*/true);
3625 } else {
3626 CGF.EmitOMPSimdInit(D: S);
3627 }
3628 },
3629 BodyCodeGen: [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3630 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3631 SourceLocation Loc = S.getBeginLoc();
3632 // when 'distribute' is not combined with a 'for':
3633 // while (idx <= UB) { BODY; ++idx; }
3634 // when 'distribute' is combined with a 'for'
3635 // (e.g. 'distribute parallel for')
3636 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3637 CGF.EmitOMPInnerLoop(
3638 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: LoopArgs.Cond, IncExpr: LoopArgs.IncExpr,
3639 BodyGen: [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3640 CodeGenLoop(CGF, S, LoopExit);
3641 },
3642 PostIncGen: [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3643 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3644 });
3645 });
3646
3647 EmitBlock(BB: Continue.getBlock());
3648 BreakContinueStack.pop_back();
3649 if (!DynamicOrOrdered) {
3650 // Emit "LB = LB + Stride", "UB = UB + Stride".
3651 EmitIgnoredExpr(E: LoopArgs.NextLB);
3652 EmitIgnoredExpr(E: LoopArgs.NextUB);
3653 }
3654
3655 EmitBranch(Block: CondBlock);
3656 OMPLoopNestStack.clear();
3657 LoopStack.pop();
3658 // Emit the fall-through block.
3659 EmitBlock(BB: LoopExit.getBlock());
3660
3661 // Tell the runtime we are done.
3662 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3663 if (!DynamicOrOrdered)
3664 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
3665 DKind: LoopArgs.DKind);
3666 };
3667 OMPCancelStack.emitExit(CGF&: *this, Kind: EKind, CodeGen);
3668}
3669
3670void CodeGenFunction::EmitOMPForOuterLoop(
3671 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
3672 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
3673 const OMPLoopArguments &LoopArgs,
3674 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3675 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3676
3677 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
3678 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind: ScheduleKind.Schedule);
3679
3680 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
3681 LoopArgs.Chunk != nullptr)) &&
3682 "static non-chunked schedule does not need outer loop");
3683
3684 // Emit outer loop.
3685 //
3686 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3687 // When schedule(dynamic,chunk_size) is specified, the iterations are
3688 // distributed to threads in the team in chunks as the threads request them.
3689 // Each thread executes a chunk of iterations, then requests another chunk,
3690 // until no chunks remain to be distributed. Each chunk contains chunk_size
3691 // iterations, except for the last chunk to be distributed, which may have
3692 // fewer iterations. When no chunk_size is specified, it defaults to 1.
3693 //
3694 // When schedule(guided,chunk_size) is specified, the iterations are assigned
3695 // to threads in the team in chunks as the executing threads request them.
3696 // Each thread executes a chunk of iterations, then requests another chunk,
3697 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
3698 // each chunk is proportional to the number of unassigned iterations divided
3699 // by the number of threads in the team, decreasing to 1. For a chunk_size
3700 // with value k (greater than 1), the size of each chunk is determined in the
3701 // same way, with the restriction that the chunks do not contain fewer than k
3702 // iterations (except for the last chunk to be assigned, which may have fewer
3703 // than k iterations).
3704 //
3705 // When schedule(auto) is specified, the decision regarding scheduling is
3706 // delegated to the compiler and/or runtime system. The programmer gives the
3707 // implementation the freedom to choose any possible mapping of iterations to
3708 // threads in the team.
3709 //
3710 // When schedule(runtime) is specified, the decision regarding scheduling is
3711 // deferred until run time, and the schedule and chunk size are taken from the
3712 // run-sched-var ICV. If the ICV is set to auto, the schedule is
3713 // implementation defined
3714 //
3715 // __kmpc_dispatch_init();
3716 // while(__kmpc_dispatch_next(&LB, &UB)) {
3717 // idx = LB;
3718 // while (idx <= UB) { BODY; ++idx;
3719 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
3720 // } // inner loop
3721 // }
3722 // __kmpc_dispatch_deinit();
3723 //
3724 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3725 // When schedule(static, chunk_size) is specified, iterations are divided into
3726 // chunks of size chunk_size, and the chunks are assigned to the threads in
3727 // the team in a round-robin fashion in the order of the thread number.
3728 //
3729 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
3730 // while (idx <= UB) { BODY; ++idx; } // inner loop
3731 // LB = LB + ST;
3732 // UB = UB + ST;
3733 // }
3734 //
3735
3736 const Expr *IVExpr = S.getIterationVariable();
3737 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3738 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3739
3740 if (DynamicOrOrdered) {
3741 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3742 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
3743 llvm::Value *LBVal = DispatchBounds.first;
3744 llvm::Value *UBVal = DispatchBounds.second;
3745 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3746 LoopArgs.Chunk};
3747 RT.emitForDispatchInit(CGF&: *this, Loc: S.getBeginLoc(), ScheduleKind, IVSize,
3748 IVSigned, Ordered, DispatchValues: DipatchRTInputValues);
3749 } else {
3750 CGOpenMPRuntime::StaticRTInput StaticInit(
3751 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3752 LoopArgs.ST, LoopArgs.Chunk);
3753 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3754 RT.emitForStaticInit(CGF&: *this, Loc: S.getBeginLoc(), DKind: EKind, ScheduleKind,
3755 Values: StaticInit);
3756 }
3757
3758 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3759 const unsigned IVSize,
3760 const bool IVSigned) {
3761 if (Ordered) {
3762 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3763 IVSigned);
3764 }
3765 };
3766
3767 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3768 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3769 OuterLoopArgs.IncExpr = S.getInc();
3770 OuterLoopArgs.Init = S.getInit();
3771 OuterLoopArgs.Cond = S.getCond();
3772 OuterLoopArgs.NextLB = S.getNextLowerBound();
3773 OuterLoopArgs.NextUB = S.getNextUpperBound();
3774 OuterLoopArgs.DKind = LoopArgs.DKind;
3775 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, LoopArgs: OuterLoopArgs,
3776 CodeGenLoop: emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3777 if (DynamicOrOrdered) {
3778 RT.emitForDispatchDeinit(CGF&: *this, Loc: S.getBeginLoc());
3779 }
3780}
3781
3782static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
3783 const unsigned IVSize, const bool IVSigned) {}
3784
3785void CodeGenFunction::EmitOMPDistributeOuterLoop(
3786 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3787 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3788 const CodeGenLoopTy &CodeGenLoopContent) {
3789
3790 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3791
3792 // Emit outer loop.
3793 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3794 // dynamic
3795 //
3796
3797 const Expr *IVExpr = S.getIterationVariable();
3798 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3799 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3800 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3801
3802 CGOpenMPRuntime::StaticRTInput StaticInit(
3803 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3804 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3805 RT.emitDistributeStaticInit(CGF&: *this, Loc: S.getBeginLoc(), SchedKind: ScheduleKind, Values: StaticInit);
3806
3807 // for combined 'distribute' and 'for' the increment expression of distribute
3808 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3809 Expr *IncExpr;
3810 if (isOpenMPLoopBoundSharingDirective(Kind: EKind))
3811 IncExpr = S.getDistInc();
3812 else
3813 IncExpr = S.getInc();
3814
3815 // this routine is shared by 'omp distribute parallel for' and
3816 // 'omp distribute': select the right EUB expression depending on the
3817 // directive
3818 OMPLoopArguments OuterLoopArgs;
3819 OuterLoopArgs.LB = LoopArgs.LB;
3820 OuterLoopArgs.UB = LoopArgs.UB;
3821 OuterLoopArgs.ST = LoopArgs.ST;
3822 OuterLoopArgs.IL = LoopArgs.IL;
3823 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3824 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3825 ? S.getCombinedEnsureUpperBound()
3826 : S.getEnsureUpperBound();
3827 OuterLoopArgs.IncExpr = IncExpr;
3828 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3829 ? S.getCombinedInit()
3830 : S.getInit();
3831 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3832 ? S.getCombinedCond()
3833 : S.getCond();
3834 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3835 ? S.getCombinedNextLowerBound()
3836 : S.getNextLowerBound();
3837 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3838 ? S.getCombinedNextUpperBound()
3839 : S.getNextUpperBound();
3840 OuterLoopArgs.DKind = OMPD_distribute;
3841
3842 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3843 LoopScope, LoopArgs: OuterLoopArgs, CodeGenLoop: CodeGenLoopContent,
3844 CodeGenOrdered: emitEmptyOrdered);
3845}
3846
3847static std::pair<LValue, LValue>
3848emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
3849 const OMPExecutableDirective &S) {
3850 const OMPLoopDirective &LS = cast<OMPLoopDirective>(Val: S);
3851 LValue LB =
3852 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getLowerBoundVariable()));
3853 LValue UB =
3854 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getUpperBoundVariable()));
3855
3856 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3857 // parallel for') we need to use the 'distribute'
3858 // chunk lower and upper bounds rather than the whole loop iteration
3859 // space. These are parameters to the outlined function for 'parallel'
3860 // and we copy the bounds of the previous schedule into the
3861 // the current ones.
3862 LValue PrevLB = CGF.EmitLValue(E: LS.getPrevLowerBoundVariable());
3863 LValue PrevUB = CGF.EmitLValue(E: LS.getPrevUpperBoundVariable());
3864 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3865 lvalue: PrevLB, Loc: LS.getPrevLowerBoundVariable()->getExprLoc());
3866 PrevLBVal = CGF.EmitScalarConversion(
3867 Src: PrevLBVal, SrcTy: LS.getPrevLowerBoundVariable()->getType(),
3868 DstTy: LS.getIterationVariable()->getType(),
3869 Loc: LS.getPrevLowerBoundVariable()->getExprLoc());
3870 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3871 lvalue: PrevUB, Loc: LS.getPrevUpperBoundVariable()->getExprLoc());
3872 PrevUBVal = CGF.EmitScalarConversion(
3873 Src: PrevUBVal, SrcTy: LS.getPrevUpperBoundVariable()->getType(),
3874 DstTy: LS.getIterationVariable()->getType(),
3875 Loc: LS.getPrevUpperBoundVariable()->getExprLoc());
3876
3877 CGF.EmitStoreOfScalar(value: PrevLBVal, lvalue: LB);
3878 CGF.EmitStoreOfScalar(value: PrevUBVal, lvalue: UB);
3879
3880 return {LB, UB};
3881}
3882
3883/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3884/// we need to use the LB and UB expressions generated by the worksharing
3885/// code generation support, whereas in non combined situations we would
3886/// just emit 0 and the LastIteration expression
3887/// This function is necessary due to the difference of the LB and UB
3888/// types for the RT emission routines for 'for_static_init' and
3889/// 'for_dispatch_init'
3890static std::pair<llvm::Value *, llvm::Value *>
3891emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
3892 const OMPExecutableDirective &S,
3893 Address LB, Address UB) {
3894 const OMPLoopDirective &LS = cast<OMPLoopDirective>(Val: S);
3895 const Expr *IVExpr = LS.getIterationVariable();
3896 // when implementing a dynamic schedule for a 'for' combined with a
3897 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3898 // is not normalized as each team only executes its own assigned
3899 // distribute chunk
3900 QualType IteratorTy = IVExpr->getType();
3901 llvm::Value *LBVal =
3902 CGF.EmitLoadOfScalar(Addr: LB, /*Volatile=*/false, Ty: IteratorTy, Loc: S.getBeginLoc());
3903 llvm::Value *UBVal =
3904 CGF.EmitLoadOfScalar(Addr: UB, /*Volatile=*/false, Ty: IteratorTy, Loc: S.getBeginLoc());
3905 return {LBVal, UBVal};
3906}
3907
3908static void emitDistributeParallelForDistributeInnerBoundParams(
3909 CodeGenFunction &CGF, const OMPExecutableDirective &S,
3910 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
3911 const auto &Dir = cast<OMPLoopDirective>(Val: S);
3912 LValue LB =
3913 CGF.EmitLValue(E: cast<DeclRefExpr>(Val: Dir.getCombinedLowerBoundVariable()));
3914 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3915 V: CGF.Builder.CreateLoad(Addr: LB.getAddress()), DestTy: CGF.SizeTy, /*isSigned=*/false);
3916 CapturedVars.push_back(Elt: LBCast);
3917 LValue UB =
3918 CGF.EmitLValue(E: cast<DeclRefExpr>(Val: Dir.getCombinedUpperBoundVariable()));
3919
3920 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3921 V: CGF.Builder.CreateLoad(Addr: UB.getAddress()), DestTy: CGF.SizeTy, /*isSigned=*/false);
3922 CapturedVars.push_back(Elt: UBCast);
3923}
3924
3925static void
3926emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
3927 const OMPLoopDirective &S,
3928 CodeGenFunction::JumpDest LoopExit) {
3929 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3930 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,
3931 PrePostActionTy &Action) {
3932 Action.Enter(CGF);
3933 bool HasCancel = false;
3934 if (!isOpenMPSimdDirective(DKind: EKind)) {
3935 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &S))
3936 HasCancel = D->hasCancel();
3937 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(Val: &S))
3938 HasCancel = D->hasCancel();
3939 else if (const auto *D =
3940 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &S))
3941 HasCancel = D->hasCancel();
3942 }
3943 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3944 CGF.EmitOMPWorksharingLoop(S, EUB: S.getPrevEnsureUpperBound(),
3945 CodeGenLoopBounds: emitDistributeParallelForInnerBounds,
3946 CGDispatchBounds: emitDistributeParallelForDispatchBounds);
3947 };
3948
3949 emitCommonOMPParallelDirective(
3950 CGF, S, InnermostKind: isOpenMPSimdDirective(DKind: EKind) ? OMPD_for_simd : OMPD_for,
3951 CodeGen: CGInlinedWorksharingLoop,
3952 CodeGenBoundParameters: emitDistributeParallelForDistributeInnerBoundParams);
3953}
3954
3955void CodeGenFunction::EmitOMPDistributeParallelForDirective(
3956 const OMPDistributeParallelForDirective &S) {
3957 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3958 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
3959 IncExpr: S.getDistInc());
3960 };
3961 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3962 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_distribute, CodeGen);
3963}
3964
3965void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
3966 const OMPDistributeParallelForSimdDirective &S) {
3967 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3968 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
3969 IncExpr: S.getDistInc());
3970 };
3971 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3972 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_distribute, CodeGen);
3973}
3974
3975void CodeGenFunction::EmitOMPDistributeSimdDirective(
3976 const OMPDistributeSimdDirective &S) {
3977 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3978 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
3979 };
3980 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3981 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_simd, CodeGen);
3982}
3983
3984void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
3985 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
3986 // Emit SPMD target parallel for region as a standalone region.
3987 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3988 emitOMPSimdRegion(CGF, S, Action);
3989 };
3990 llvm::Function *Fn;
3991 llvm::Constant *Addr;
3992 // Emit target region as a standalone region.
3993 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3994 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
3995 assert(Fn && Addr && "Target device function emission failed.");
3996}
3997
3998void CodeGenFunction::EmitOMPTargetSimdDirective(
3999 const OMPTargetSimdDirective &S) {
4000 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4001 emitOMPSimdRegion(CGF, S, Action);
4002 };
4003 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
4004}
4005
4006namespace {
4007struct ScheduleKindModifiersTy {
4008 OpenMPScheduleClauseKind Kind;
4009 OpenMPScheduleClauseModifier M1;
4010 OpenMPScheduleClauseModifier M2;
4011 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
4012 OpenMPScheduleClauseModifier M1,
4013 OpenMPScheduleClauseModifier M2)
4014 : Kind(Kind), M1(M1), M2(M2) {}
4015};
4016} // namespace
4017
4018bool CodeGenFunction::EmitOMPWorksharingLoop(
4019 const OMPLoopDirective &S, Expr *EUB,
4020 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4021 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
4022 // Emit the loop iteration variable.
4023 const auto *IVExpr = cast<DeclRefExpr>(Val: S.getIterationVariable());
4024 const auto *IVDecl = cast<VarDecl>(Val: IVExpr->getDecl());
4025 EmitVarDecl(D: *IVDecl);
4026
4027 // Emit the iterations count variable.
4028 // If it is not a variable, Sema decided to calculate iterations count on each
4029 // iteration (e.g., it is foldable into a constant).
4030 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
4031 EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
4032 // Emit calculation of the iterations count.
4033 EmitIgnoredExpr(E: S.getCalcLastIteration());
4034 }
4035
4036 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
4037
4038 bool HasLastprivateClause;
4039 // Check pre-condition.
4040 {
4041 OMPLoopScope PreInitScope(*this, S);
4042 // Skip the entire loop if we don't meet the precondition.
4043 // If the condition constant folds and can be elided, avoid emitting the
4044 // whole loop.
4045 bool CondConstant;
4046 llvm::BasicBlock *ContBlock = nullptr;
4047 if (ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
4048 if (!CondConstant)
4049 return false;
4050 } else {
4051 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "omp.precond.then");
4052 ContBlock = createBasicBlock(name: "omp.precond.end");
4053 emitPreCond(CGF&: *this, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
4054 TrueCount: getProfileCount(S: &S));
4055 EmitBlock(BB: ThenBlock);
4056 incrementProfileCounter(S: &S);
4057 }
4058
4059 RunCleanupsScope DoacrossCleanupScope(*this);
4060 bool Ordered = false;
4061 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
4062 if (OrderedClause->getNumForLoops())
4063 RT.emitDoacrossInit(CGF&: *this, D: S, NumIterations: OrderedClause->getLoopNumIterations());
4064 else
4065 Ordered = true;
4066 }
4067
4068 emitAlignedClause(CGF&: *this, D: S);
4069 bool HasLinears = EmitOMPLinearClauseInit(D: S);
4070 // Emit helper vars inits.
4071
4072 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
4073 LValue LB = Bounds.first;
4074 LValue UB = Bounds.second;
4075 LValue ST =
4076 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()));
4077 LValue IL =
4078 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()));
4079
4080 // Emit 'then' code.
4081 {
4082 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4083 OMPPrivateScope LoopScope(*this);
4084 if (EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope) || HasLinears) {
4085 // Emit implicit barrier to synchronize threads and avoid data races on
4086 // initialization of firstprivate variables and post-update of
4087 // lastprivate variables.
4088 CGM.getOpenMPRuntime().emitBarrierCall(
4089 CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4090 /*ForceSimpleCall=*/true);
4091 }
4092 EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
4093 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
4094 *this, S, EmitLValue(E: S.getIterationVariable()));
4095 HasLastprivateClause = EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
4096 EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
4097 EmitOMPPrivateLoopCounters(S, LoopScope);
4098 EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
4099 (void)LoopScope.Privatize();
4100 if (isOpenMPTargetExecutionDirective(DKind: EKind))
4101 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF&: *this, D: S);
4102
4103 // Detect the loop schedule kind and chunk.
4104 const Expr *ChunkExpr = nullptr;
4105 OpenMPScheduleTy ScheduleKind;
4106 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
4107 ScheduleKind.Schedule = C->getScheduleKind();
4108 ScheduleKind.M1 = C->getFirstScheduleModifier();
4109 ScheduleKind.M2 = C->getSecondScheduleModifier();
4110 ChunkExpr = C->getChunkSize();
4111 } else {
4112 // Default behaviour for schedule clause.
4113 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
4114 CGF&: *this, S, ScheduleKind&: ScheduleKind.Schedule, ChunkExpr);
4115 }
4116 bool HasChunkSizeOne = false;
4117 llvm::Value *Chunk = nullptr;
4118 if (ChunkExpr) {
4119 Chunk = EmitScalarExpr(E: ChunkExpr);
4120 Chunk = EmitScalarConversion(Src: Chunk, SrcTy: ChunkExpr->getType(),
4121 DstTy: S.getIterationVariable()->getType(),
4122 Loc: S.getBeginLoc());
4123 Expr::EvalResult Result;
4124 if (ChunkExpr->EvaluateAsInt(Result, Ctx: getContext())) {
4125 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
4126 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
4127 }
4128 }
4129 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
4130 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
4131 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
4132 // If the static schedule kind is specified or if the ordered clause is
4133 // specified, and if no monotonic modifier is specified, the effect will
4134 // be as if the monotonic modifier was specified.
4135 bool StaticChunkedOne =
4136 RT.isStaticChunked(ScheduleKind: ScheduleKind.Schedule,
4137 /* Chunked */ Chunk != nullptr) &&
4138 HasChunkSizeOne && isOpenMPLoopBoundSharingDirective(Kind: EKind);
4139 // GPU combined `distribute parallel for`: emit a single
4140 // for_static_init with the fused distr_static_chunk + static_chunkone
4141 // schedule (enum 93). The surrounding EmitOMPDistributeLoop must skip
4142 // its distribute_static_init under the same conditions. Both sites are
4143 // guarded by canEmitGPUFusedDistSchedule() alone so they cannot
4144 // disagree; the assert guards the invariant that makes this safe today,
4145 // aka that the implicit GPU default schedule is always static chunk-one.
4146 ScheduleKind.UseFusedDistChunkSchedule =
4147 canEmitGPUFusedDistSchedule(CGM, S, DKind: EKind);
4148 assert((!ScheduleKind.UseFusedDistChunkSchedule || StaticChunkedOne) &&
4149 "fused distribute schedule requires a static chunk-one schedule");
4150 bool IsMonotonic =
4151 Ordered ||
4152 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
4153 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
4154 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
4155 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
4156 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
4157 if ((RT.isStaticNonchunked(ScheduleKind: ScheduleKind.Schedule,
4158 /* Chunked */ Chunk != nullptr) ||
4159 StaticChunkedOne) &&
4160 !Ordered) {
4161 JumpDest LoopExit =
4162 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
4163 emitCommonSimdLoop(
4164 CGF&: *this, S,
4165 SimdInitGen: [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
4166 if (isOpenMPSimdDirective(DKind: EKind)) {
4167 CGF.EmitOMPSimdInit(D: S);
4168 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
4169 if (C->getKind() == OMPC_ORDER_concurrent)
4170 CGF.LoopStack.setParallel(/*Enable=*/true);
4171 }
4172 },
4173 BodyCodeGen: [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
4174 &S, ScheduleKind, LoopExit, EKind,
4175 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
4176 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
4177 // When no chunk_size is specified, the iteration space is divided
4178 // into chunks that are approximately equal in size, and at most
4179 // one chunk is distributed to each thread. Note that the size of
4180 // the chunks is unspecified in this case.
4181 CGOpenMPRuntime::StaticRTInput StaticInit(
4182 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
4183 UB.getAddress(), ST.getAddress(),
4184 StaticChunkedOne ? Chunk : nullptr);
4185 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
4186 CGF, Loc: S.getBeginLoc(), DKind: EKind, ScheduleKind, Values: StaticInit);
4187 // UB = min(UB, GlobalUB);
4188 if (!StaticChunkedOne)
4189 CGF.EmitIgnoredExpr(E: S.getEnsureUpperBound());
4190 // IV = LB;
4191 CGF.EmitIgnoredExpr(E: S.getInit());
4192 // For unchunked static schedule generate:
4193 //
4194 // while (idx <= UB) {
4195 // BODY;
4196 // ++idx;
4197 // }
4198 //
4199 // For static schedule with chunk one:
4200 //
4201 // while (IV <= PrevUB) {
4202 // BODY;
4203 // IV += ST;
4204 // }
4205 CGF.EmitOMPInnerLoop(
4206 S, RequiresCleanup: LoopScope.requiresCleanups(),
4207 LoopCond: StaticChunkedOne ? S.getCombinedParForInDistCond()
4208 : S.getCond(),
4209 IncExpr: StaticChunkedOne ? S.getDistInc() : S.getInc(),
4210 BodyGen: [&S, LoopExit](CodeGenFunction &CGF) {
4211 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
4212 },
4213 PostIncGen: [](CodeGenFunction &) {});
4214 });
4215 EmitBlock(BB: LoopExit.getBlock());
4216 // Tell the runtime we are done.
4217 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4218 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
4219 DKind: OMPD_for);
4220 };
4221 OMPCancelStack.emitExit(CGF&: *this, Kind: EKind, CodeGen);
4222 } else {
4223 // Emit the outer loop, which requests its work chunk [LB..UB] from
4224 // runtime and runs the inner loop to process it.
4225 OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
4226 ST.getAddress(), IL.getAddress(), Chunk,
4227 EUB);
4228 LoopArguments.DKind = OMPD_for;
4229 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
4230 LoopArgs: LoopArguments, CGDispatchBounds);
4231 }
4232 if (isOpenMPSimdDirective(DKind: EKind)) {
4233 EmitOMPSimdFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4234 return CGF.Builder.CreateIsNotNull(
4235 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4236 });
4237 }
4238 EmitOMPReductionClauseFinal(
4239 D: S, /*ReductionKind=*/isOpenMPSimdDirective(DKind: EKind)
4240 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
4241 : /*Parallel only*/ OMPD_parallel);
4242 // Emit post-update of the reduction variables if IsLastIter != 0.
4243 emitPostUpdateForReductionClause(
4244 CGF&: *this, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4245 return CGF.Builder.CreateIsNotNull(
4246 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4247 });
4248 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4249 if (HasLastprivateClause)
4250 EmitOMPLastprivateClauseFinal(
4251 D: S, NoFinals: isOpenMPSimdDirective(DKind: EKind),
4252 IsLastIterCond: Builder.CreateIsNotNull(Arg: EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
4253 LoopScope.restoreMap();
4254 EmitOMPLinearClauseFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4255 return CGF.Builder.CreateIsNotNull(
4256 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4257 });
4258 }
4259 DoacrossCleanupScope.ForceCleanup();
4260 // We're now done with the loop, so jump to the continuation block.
4261 if (ContBlock) {
4262 EmitBranch(Block: ContBlock);
4263 EmitBlock(BB: ContBlock, /*IsFinished=*/true);
4264 }
4265 }
4266 return HasLastprivateClause;
4267}
4268
4269/// The following two functions generate expressions for the loop lower
4270/// and upper bounds in case of static and dynamic (dispatch) schedule
4271/// of the associated 'for' or 'distribute' loop.
4272static std::pair<LValue, LValue>
4273emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4274 const auto &LS = cast<OMPLoopDirective>(Val: S);
4275 LValue LB =
4276 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getLowerBoundVariable()));
4277 LValue UB =
4278 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getUpperBoundVariable()));
4279 return {LB, UB};
4280}
4281
4282/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
4283/// consider the lower and upper bound expressions generated by the
4284/// worksharing loop support, but we use 0 and the iteration space size as
4285/// constants
4286static std::pair<llvm::Value *, llvm::Value *>
4287emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
4288 Address LB, Address UB) {
4289 const auto &LS = cast<OMPLoopDirective>(Val: S);
4290 const Expr *IVExpr = LS.getIterationVariable();
4291 const unsigned IVSize = CGF.getContext().getTypeSize(T: IVExpr->getType());
4292 llvm::Value *LBVal = CGF.Builder.getIntN(N: IVSize, C: 0);
4293 llvm::Value *UBVal = CGF.EmitScalarExpr(E: LS.getLastIteration());
4294 return {LBVal, UBVal};
4295}
4296
4297/// Emits internal temp array declarations for the directive with inscan
4298/// reductions.
4299/// The code is the following:
4300/// \code
4301/// size num_iters = <num_iters>;
4302/// <type> buffer[num_iters];
4303/// \endcode
4304static void emitScanBasedDirectiveDecls(
4305 CodeGenFunction &CGF, const OMPLoopDirective &S,
4306 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4307 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4308 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4309 SmallVector<const Expr *, 4> Shareds;
4310 SmallVector<const Expr *, 4> Privates;
4311 SmallVector<const Expr *, 4> ReductionOps;
4312 SmallVector<const Expr *, 4> CopyArrayTemps;
4313 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4314 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4315 "Only inscan reductions are expected.");
4316 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
4317 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4318 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
4319 CopyArrayTemps.append(in_start: C->copy_array_temps().begin(),
4320 in_end: C->copy_array_temps().end());
4321 }
4322 {
4323 // Emit buffers for each reduction variables.
4324 // ReductionCodeGen is required to emit correctly the code for array
4325 // reductions.
4326 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
4327 unsigned Count = 0;
4328 auto *ITA = CopyArrayTemps.begin();
4329 for (const Expr *IRef : Privates) {
4330 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IRef)->getDecl());
4331 // Emit variably modified arrays, used for arrays/array sections
4332 // reductions.
4333 if (PrivateVD->getType()->isVariablyModifiedType()) {
4334 RedCG.emitSharedOrigLValue(CGF, N: Count);
4335 RedCG.emitAggregateType(CGF, N: Count);
4336 }
4337 CodeGenFunction::OpaqueValueMapping DimMapping(
4338 CGF,
4339 cast<OpaqueValueExpr>(
4340 Val: cast<VariableArrayType>(Val: (*ITA)->getType()->getAsArrayTypeUnsafe())
4341 ->getSizeExpr()),
4342 RValue::get(V: OMPScanNumIterations));
4343 // Emit temp buffer.
4344 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ITA)->getDecl()));
4345 ++ITA;
4346 ++Count;
4347 }
4348 }
4349}
4350
4351/// Copies final inscan reductions values to the original variables.
4352/// The code is the following:
4353/// \code
4354/// <orig_var> = buffer[num_iters-1];
4355/// \endcode
4356static void emitScanBasedDirectiveFinals(
4357 CodeGenFunction &CGF, const OMPLoopDirective &S,
4358 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4359 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4360 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4361 SmallVector<const Expr *, 4> Shareds;
4362 SmallVector<const Expr *, 4> LHSs;
4363 SmallVector<const Expr *, 4> RHSs;
4364 SmallVector<const Expr *, 4> Privates;
4365 SmallVector<const Expr *, 4> CopyOps;
4366 SmallVector<const Expr *, 4> CopyArrayElems;
4367 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4368 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4369 "Only inscan reductions are expected.");
4370 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
4371 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
4372 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
4373 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4374 CopyOps.append(in_start: C->copy_ops().begin(), in_end: C->copy_ops().end());
4375 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
4376 in_end: C->copy_array_elems().end());
4377 }
4378 // Create temp var and copy LHS value to this temp value.
4379 // LHS = TMP[LastIter];
4380 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
4381 LHS: OMPScanNumIterations,
4382 RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1, /*isSigned=*/IsSigned: false));
4383 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
4384 const Expr *PrivateExpr = Privates[I];
4385 const Expr *OrigExpr = Shareds[I];
4386 const Expr *CopyArrayElem = CopyArrayElems[I];
4387 CodeGenFunction::OpaqueValueMapping IdxMapping(
4388 CGF,
4389 cast<OpaqueValueExpr>(
4390 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4391 RValue::get(V: OMPLast));
4392 LValue DestLVal = CGF.EmitLValue(E: OrigExpr);
4393 LValue SrcLVal = CGF.EmitLValue(E: CopyArrayElem);
4394 CGF.EmitOMPCopy(
4395 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
4396 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
4397 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
4398 }
4399}
4400
4401/// Emits the code for the directive with inscan reductions.
4402/// The code is the following:
4403/// \code
4404/// #pragma omp ...
4405/// for (i: 0..<num_iters>) {
4406/// <input phase>;
4407/// buffer[i] = red;
4408/// }
4409/// #pragma omp master // in parallel region
4410/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
4411/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
4412/// buffer[i] op= buffer[i-pow(2,k)];
4413/// #pragma omp barrier // in parallel region
4414/// #pragma omp ...
4415/// for (0..<num_iters>) {
4416/// red = InclusiveScan ? buffer[i] : buffer[i-1];
4417/// <scan phase>;
4418/// }
4419/// \endcode
4420static void emitScanBasedDirective(
4421 CodeGenFunction &CGF, const OMPLoopDirective &S,
4422 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
4423 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
4424 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
4425 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4426 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4427 SmallVector<const Expr *, 4> Privates;
4428 SmallVector<const Expr *, 4> ReductionOps;
4429 SmallVector<const Expr *, 4> LHSs;
4430 SmallVector<const Expr *, 4> RHSs;
4431 SmallVector<const Expr *, 4> CopyArrayElems;
4432 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4433 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4434 "Only inscan reductions are expected.");
4435 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4436 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
4437 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
4438 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
4439 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
4440 in_end: C->copy_array_elems().end());
4441 }
4442 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
4443 {
4444 // Emit loop with input phase:
4445 // #pragma omp ...
4446 // for (i: 0..<num_iters>) {
4447 // <input phase>;
4448 // buffer[i] = red;
4449 // }
4450 CGF.OMPFirstScanLoop = true;
4451 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4452 FirstGen(CGF);
4453 }
4454 // #pragma omp barrier // in parallel region
4455 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
4456 &ReductionOps,
4457 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
4458 Action.Enter(CGF);
4459 // Emit prefix reduction:
4460 // #pragma omp master // in parallel region
4461 // for (int k = 0; k <= ceil(log2(n)); ++k)
4462 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
4463 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(name: "omp.outer.log.scan.body");
4464 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "omp.outer.log.scan.exit");
4465 llvm::Function *F =
4466 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::log2, Tys: CGF.DoubleTy);
4467 llvm::Value *Arg =
4468 CGF.Builder.CreateUIToFP(V: OMPScanNumIterations, DestTy: CGF.DoubleTy);
4469 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(callee: F, args: Arg);
4470 F = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ceil, Tys: CGF.DoubleTy);
4471 LogVal = CGF.EmitNounwindRuntimeCall(callee: F, args: LogVal);
4472 LogVal = CGF.Builder.CreateFPToUI(V: LogVal, DestTy: CGF.IntTy);
4473 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
4474 LHS: OMPScanNumIterations, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1));
4475 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: S.getBeginLoc());
4476 CGF.EmitBlock(BB: LoopBB);
4477 auto *Counter = CGF.Builder.CreatePHI(Ty: CGF.IntTy, NumReservedValues: 2);
4478 // size pow2k = 1;
4479 auto *Pow2K = CGF.Builder.CreatePHI(Ty: CGF.SizeTy, NumReservedValues: 2);
4480 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0), BB: InputBB);
4481 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), BB: InputBB);
4482 // for (size i = n - 1; i >= 2 ^ k; --i)
4483 // tmp[i] op= tmp[i-pow2k];
4484 llvm::BasicBlock *InnerLoopBB =
4485 CGF.createBasicBlock(name: "omp.inner.log.scan.body");
4486 llvm::BasicBlock *InnerExitBB =
4487 CGF.createBasicBlock(name: "omp.inner.log.scan.exit");
4488 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
4489 CGF.Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
4490 CGF.EmitBlock(BB: InnerLoopBB);
4491 auto *IVal = CGF.Builder.CreatePHI(Ty: CGF.SizeTy, NumReservedValues: 2);
4492 IVal->addIncoming(V: NMin1, BB: LoopBB);
4493 {
4494 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
4495 auto *ILHS = LHSs.begin();
4496 auto *IRHS = RHSs.begin();
4497 for (const Expr *CopyArrayElem : CopyArrayElems) {
4498 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
4499 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
4500 Address LHSAddr = Address::invalid();
4501 {
4502 CodeGenFunction::OpaqueValueMapping IdxMapping(
4503 CGF,
4504 cast<OpaqueValueExpr>(
4505 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4506 RValue::get(V: IVal));
4507 LHSAddr = CGF.EmitLValue(E: CopyArrayElem).getAddress();
4508 }
4509 PrivScope.addPrivate(LocalVD: LHSVD, Addr: LHSAddr);
4510 Address RHSAddr = Address::invalid();
4511 {
4512 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(LHS: IVal, RHS: Pow2K);
4513 CodeGenFunction::OpaqueValueMapping IdxMapping(
4514 CGF,
4515 cast<OpaqueValueExpr>(
4516 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4517 RValue::get(V: OffsetIVal));
4518 RHSAddr = CGF.EmitLValue(E: CopyArrayElem).getAddress();
4519 }
4520 PrivScope.addPrivate(LocalVD: RHSVD, Addr: RHSAddr);
4521 ++ILHS;
4522 ++IRHS;
4523 }
4524 PrivScope.Privatize();
4525 CGF.CGM.getOpenMPRuntime().emitReduction(
4526 CGF, Loc: S.getEndLoc(), Privates, LHSExprs: LHSs, RHSExprs: RHSs, ReductionOps,
4527 Options: {/*WithNowait=*/true, /*SimpleReduction=*/true,
4528 /*IsPrivateVarReduction*/ {}, .ReductionKind: OMPD_unknown});
4529 }
4530 llvm::Value *NextIVal =
4531 CGF.Builder.CreateNUWSub(LHS: IVal, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1));
4532 IVal->addIncoming(V: NextIVal, BB: CGF.Builder.GetInsertBlock());
4533 CmpI = CGF.Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
4534 CGF.Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
4535 CGF.EmitBlock(BB: InnerExitBB);
4536 llvm::Value *Next =
4537 CGF.Builder.CreateNUWAdd(LHS: Counter, RHS: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1));
4538 Counter->addIncoming(V: Next, BB: CGF.Builder.GetInsertBlock());
4539 // pow2k <<= 1;
4540 llvm::Value *NextPow2K =
4541 CGF.Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
4542 Pow2K->addIncoming(V: NextPow2K, BB: CGF.Builder.GetInsertBlock());
4543 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
4544 CGF.Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
4545 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: S.getEndLoc());
4546 CGF.EmitBlock(BB: ExitBB);
4547 };
4548 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4549 if (isOpenMPParallelDirective(DKind: EKind)) {
4550 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
4551 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4552 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4553 /*ForceSimpleCall=*/true);
4554 } else {
4555 RegionCodeGenTy RCG(CodeGen);
4556 RCG(CGF);
4557 }
4558
4559 CGF.OMPFirstScanLoop = false;
4560 SecondGen(CGF);
4561}
4562
4563static bool emitWorksharingDirective(CodeGenFunction &CGF,
4564 const OMPLoopDirective &S,
4565 bool HasCancel) {
4566 bool HasLastprivates;
4567 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4568 if (llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
4569 P: [](const OMPReductionClause *C) {
4570 return C->getModifier() == OMPC_REDUCTION_inscan;
4571 })) {
4572 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4573 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4574 OMPLoopScope LoopScope(CGF, S);
4575 return CGF.EmitScalarExpr(E: S.getNumIterations());
4576 };
4577 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {
4578 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4579 (void)CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4580 CodeGenLoopBounds: emitForLoopBounds,
4581 CGDispatchBounds: emitDispatchForLoopBounds);
4582 // Emit an implicit barrier at the end.
4583 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc: S.getBeginLoc(),
4584 Kind: OMPD_for);
4585 };
4586 const auto &&SecondGen = [&S, HasCancel, EKind,
4587 &HasLastprivates](CodeGenFunction &CGF) {
4588 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4589 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4590 CodeGenLoopBounds: emitForLoopBounds,
4591 CGDispatchBounds: emitDispatchForLoopBounds);
4592 };
4593 if (!isOpenMPParallelDirective(DKind: EKind))
4594 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
4595 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
4596 if (!isOpenMPParallelDirective(DKind: EKind))
4597 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
4598 } else {
4599 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4600 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4601 CodeGenLoopBounds: emitForLoopBounds,
4602 CGDispatchBounds: emitDispatchForLoopBounds);
4603 }
4604 return HasLastprivates;
4605}
4606
4607// Pass OMPLoopDirective (instead of OMPForDirective) to make this check
4608// available for "loop bind(parallel)", which maps to "for".
4609static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S,
4610 bool HasCancel) {
4611 if (HasCancel)
4612 return false;
4613 for (OMPClause *C : S.clauses()) {
4614 if (isa<OMPNowaitClause, OMPBindClause>(Val: C))
4615 continue;
4616
4617 if (auto *SC = dyn_cast<OMPScheduleClause>(Val: C)) {
4618 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4619 return false;
4620 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4621 return false;
4622 switch (SC->getScheduleKind()) {
4623 case OMPC_SCHEDULE_auto:
4624 case OMPC_SCHEDULE_dynamic:
4625 case OMPC_SCHEDULE_runtime:
4626 case OMPC_SCHEDULE_guided:
4627 case OMPC_SCHEDULE_static:
4628 continue;
4629 case OMPC_SCHEDULE_unknown:
4630 return false;
4631 }
4632 }
4633
4634 return false;
4635 }
4636
4637 return true;
4638}
4639
4640static llvm::omp::ScheduleKind
4641convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind) {
4642 switch (ScheduleClauseKind) {
4643 case OMPC_SCHEDULE_unknown:
4644 return llvm::omp::OMP_SCHEDULE_Default;
4645 case OMPC_SCHEDULE_auto:
4646 return llvm::omp::OMP_SCHEDULE_Auto;
4647 case OMPC_SCHEDULE_dynamic:
4648 return llvm::omp::OMP_SCHEDULE_Dynamic;
4649 case OMPC_SCHEDULE_guided:
4650 return llvm::omp::OMP_SCHEDULE_Guided;
4651 case OMPC_SCHEDULE_runtime:
4652 return llvm::omp::OMP_SCHEDULE_Runtime;
4653 case OMPC_SCHEDULE_static:
4654 return llvm::omp::OMP_SCHEDULE_Static;
4655 }
4656 llvm_unreachable("Unhandled schedule kind");
4657}
4658
4659// Pass OMPLoopDirective (instead of OMPForDirective) to make this function
4660// available for "loop bind(parallel)", which maps to "for".
4661static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF,
4662 CodeGenModule &CGM, bool HasCancel) {
4663 bool HasLastprivates = false;
4664 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder &&
4665 isForSupportedByOpenMPIRBuilder(S, HasCancel);
4666 auto &&CodeGen = [&S, &CGM, HasCancel, &HasLastprivates,
4667 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
4668 // Use the OpenMPIRBuilder if enabled.
4669 if (UseOMPIRBuilder) {
4670 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
4671
4672 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4673 llvm::Value *ChunkSize = nullptr;
4674 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
4675 SchedKind =
4676 convertClauseKindToSchedKind(ScheduleClauseKind: SchedClause->getScheduleKind());
4677 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4678 ChunkSize = CGF.EmitScalarExpr(E: ChunkSizeExpr);
4679 }
4680
4681 // Emit the associated statement and get its loop representation.
4682 const Stmt *Inner = S.getRawStmt();
4683 llvm::CanonicalLoopInfo *CLI =
4684 CGF.EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
4685
4686 llvm::OpenMPIRBuilder &OMPBuilder =
4687 CGM.getOpenMPRuntime().getOMPBuilder();
4688 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4689 CGF.AllocaInsertPt->getIterator());
4690 cantFail(ValOrErr: OMPBuilder.applyWorkshareLoop(
4691 DL: CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4692 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
4693 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
4694 /*HasOrderedClause=*/false));
4695 return;
4696 }
4697
4698 HasLastprivates = emitWorksharingDirective(CGF, S, HasCancel);
4699 };
4700 {
4701 auto LPCRegion =
4702 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
4703 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
4704 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_for, CodeGen,
4705 HasCancel);
4706 }
4707
4708 if (!UseOMPIRBuilder) {
4709 // Emit an implicit barrier at the end.
4710 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4711 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc: S.getBeginLoc(), Kind: OMPD_for);
4712 }
4713 // Check for outer lastprivate conditional update.
4714 checkForLastprivateConditionalUpdate(CGF, S);
4715}
4716
4717void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
4718 return emitOMPForDirective(S, CGF&: *this, CGM, HasCancel: S.hasCancel());
4719}
4720
4721void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
4722 bool HasLastprivates = false;
4723 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4724 PrePostActionTy &) {
4725 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4726 };
4727 {
4728 auto LPCRegion =
4729 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4730 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4731 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_simd, CodeGen);
4732 }
4733
4734 // Emit an implicit barrier at the end.
4735 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4736 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_for);
4737 // Check for outer lastprivate conditional update.
4738 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4739}
4740
4741static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
4742 const Twine &Name,
4743 llvm::Value *Init = nullptr) {
4744 LValue LVal = CGF.MakeAddrLValue(Addr: CGF.CreateMemTemp(T: Ty, Name), T: Ty);
4745 if (Init)
4746 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Init), Dst: LVal, /*isInit*/ true);
4747 return LVal;
4748}
4749
4750void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
4751 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4752 const auto *CS = dyn_cast<CompoundStmt>(Val: CapturedStmt);
4753 bool HasLastprivates = false;
4754 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4755 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4756 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4757 const ASTContext &C = CGF.getContext();
4758 QualType KmpInt32Ty =
4759 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4760 // Emit helper vars inits.
4761 LValue LB = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.lb.",
4762 Init: CGF.Builder.getInt32(C: 0));
4763 llvm::ConstantInt *GlobalUBVal = CS != nullptr
4764 ? CGF.Builder.getInt32(C: CS->size() - 1)
4765 : CGF.Builder.getInt32(C: 0);
4766 LValue UB =
4767 createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.ub.", Init: GlobalUBVal);
4768 LValue ST = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.st.",
4769 Init: CGF.Builder.getInt32(C: 1));
4770 LValue IL = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.il.",
4771 Init: CGF.Builder.getInt32(C: 0));
4772 // Loop counter.
4773 LValue IV = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.iv.");
4774 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4775 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4776 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4777 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4778 // Generate condition for loop.
4779 BinaryOperator *Cond = BinaryOperator::Create(
4780 C, lhs: &IVRefExpr, rhs: &UBRefExpr, opc: BO_LE, ResTy: C.BoolTy, VK: VK_PRValue, OK: OK_Ordinary,
4781 opLoc: S.getBeginLoc(), FPFeatures: FPOptionsOverride());
4782 // Increment for loop counter.
4783 UnaryOperator *Inc = UnaryOperator::Create(
4784 C, input: &IVRefExpr, opc: UO_PreInc, type: KmpInt32Ty, VK: VK_PRValue, OK: OK_Ordinary,
4785 l: S.getBeginLoc(), CanOverflow: true, FPFeatures: FPOptionsOverride());
4786 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4787 // Iterate through all sections and emit a switch construct:
4788 // switch (IV) {
4789 // case 0:
4790 // <SectionStmt[0]>;
4791 // break;
4792 // ...
4793 // case <NumSection> - 1:
4794 // <SectionStmt[<NumSection> - 1]>;
4795 // break;
4796 // }
4797 // .omp.sections.exit:
4798 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".omp.sections.exit");
4799 llvm::SwitchInst *SwitchStmt =
4800 CGF.Builder.CreateSwitch(V: CGF.EmitLoadOfScalar(lvalue: IV, Loc: S.getBeginLoc()),
4801 Dest: ExitBB, NumCases: CS == nullptr ? 1 : CS->size());
4802 if (CS) {
4803 unsigned CaseNumber = 0;
4804 for (const Stmt *SubStmt : CS->children()) {
4805 auto CaseBB = CGF.createBasicBlock(name: ".omp.sections.case");
4806 CGF.EmitBlock(BB: CaseBB);
4807 SwitchStmt->addCase(OnVal: CGF.Builder.getInt32(C: CaseNumber), Dest: CaseBB);
4808 CGF.EmitStmt(S: SubStmt);
4809 CGF.EmitBranch(Block: ExitBB);
4810 ++CaseNumber;
4811 }
4812 } else {
4813 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(name: ".omp.sections.case");
4814 CGF.EmitBlock(BB: CaseBB);
4815 SwitchStmt->addCase(OnVal: CGF.Builder.getInt32(C: 0), Dest: CaseBB);
4816 CGF.EmitStmt(S: CapturedStmt);
4817 CGF.EmitBranch(Block: ExitBB);
4818 }
4819 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
4820 };
4821
4822 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4823 if (CGF.EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope)) {
4824 // Emit implicit barrier to synchronize threads and avoid data races on
4825 // initialization of firstprivate variables and post-update of lastprivate
4826 // variables.
4827 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4828 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4829 /*ForceSimpleCall=*/true);
4830 }
4831 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
4832 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4833 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
4834 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
4835 (void)LoopScope.Privatize();
4836 if (isOpenMPTargetExecutionDirective(DKind: EKind))
4837 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
4838
4839 // Emit static non-chunked loop.
4840 OpenMPScheduleTy ScheduleKind;
4841 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4842 CGOpenMPRuntime::StaticRTInput StaticInit(
4843 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
4844 LB.getAddress(), UB.getAddress(), ST.getAddress());
4845 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, Loc: S.getBeginLoc(), DKind: EKind,
4846 ScheduleKind, Values: StaticInit);
4847 // UB = min(UB, GlobalUB);
4848 llvm::Value *UBVal = CGF.EmitLoadOfScalar(lvalue: UB, Loc: S.getBeginLoc());
4849 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4850 C: CGF.Builder.CreateICmpSLT(LHS: UBVal, RHS: GlobalUBVal), True: UBVal, False: GlobalUBVal);
4851 CGF.EmitStoreOfScalar(value: MinUBGlobalUB, lvalue: UB);
4852 // IV = LB;
4853 CGF.EmitStoreOfScalar(value: CGF.EmitLoadOfScalar(lvalue: LB, Loc: S.getBeginLoc()), lvalue: IV);
4854 // while (idx <= UB) { BODY; ++idx; }
4855 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, LoopCond: Cond, IncExpr: Inc, BodyGen,
4856 PostIncGen: [](CodeGenFunction &) {});
4857 // Tell the runtime we are done.
4858 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4859 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
4860 DKind: OMPD_sections);
4861 };
4862 CGF.OMPCancelStack.emitExit(CGF, Kind: EKind, CodeGen);
4863 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
4864 // Emit post-update of the reduction variables if IsLastIter != 0.
4865 emitPostUpdateForReductionClause(CGF, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4866 return CGF.Builder.CreateIsNotNull(
4867 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4868 });
4869
4870 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4871 if (HasLastprivates)
4872 CGF.EmitOMPLastprivateClauseFinal(
4873 D: S, /*NoFinals=*/false,
4874 IsLastIterCond: CGF.Builder.CreateIsNotNull(
4875 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
4876 };
4877
4878 bool HasCancel = false;
4879 if (auto *OSD = dyn_cast<OMPSectionsDirective>(Val: &S))
4880 HasCancel = OSD->hasCancel();
4881 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &S))
4882 HasCancel = OPSD->hasCancel();
4883 OMPCancelStackRAII CancelRegion(*this, EKind, HasCancel);
4884 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_sections, CodeGen,
4885 HasCancel);
4886 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4887 // clause. Otherwise the barrier will be generated by the codegen for the
4888 // directive.
4889 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4890 // Emit implicit barrier to synchronize threads and avoid data races on
4891 // initialization of firstprivate variables.
4892 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(),
4893 Kind: OMPD_unknown);
4894 }
4895}
4896
4897void CodeGenFunction::EmitOMPScopeDirective(const OMPScopeDirective &S) {
4898 {
4899 // Emit code for 'scope' region
4900 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4901 Action.Enter(CGF);
4902 OMPPrivateScope PrivateScope(CGF);
4903 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
4904 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
4905 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
4906 (void)PrivateScope.Privatize();
4907 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
4908 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
4909 };
4910 auto LPCRegion =
4911 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4912 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4913 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_scope, CodeGen);
4914 }
4915 // Emit an implicit barrier at the end.
4916 if (!S.getSingleClause<OMPNowaitClause>()) {
4917 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_scope);
4918 }
4919 // Check for outer lastprivate conditional update.
4920 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4921}
4922
4923void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
4924 if (CGM.getLangOpts().OpenMPIRBuilder) {
4925 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4926 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4927 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4928
4929 auto FiniCB = [](InsertPointTy IP) {
4930 // Don't FinalizeOMPRegion because this is done inside of OMPIRBuilder for
4931 // sections.
4932 return llvm::Error::success();
4933 };
4934
4935 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4936 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4937 const auto *CS = dyn_cast<CompoundStmt>(Val: CapturedStmt);
4938 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4939 if (CS) {
4940 for (const Stmt *SubStmt : CS->children()) {
4941 auto SectionCB = [this, SubStmt](
4942 InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4943 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4944 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(CGF&: *this, RegionBodyStmt: SubStmt, AllocaIP: AllocIP,
4945 CodeGenIP, RegionName: "section");
4946 return llvm::Error::success();
4947 };
4948 SectionCBVector.push_back(Elt: SectionCB);
4949 }
4950 } else {
4951 auto SectionCB =
4952 [this, CapturedStmt](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4953 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4954 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4955 CGF&: *this, RegionBodyStmt: CapturedStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "section");
4956 return llvm::Error::success();
4957 };
4958 SectionCBVector.push_back(Elt: SectionCB);
4959 }
4960
4961 // Privatization callback that performs appropriate action for
4962 // shared/private/firstprivate/lastprivate/copyin/... variables.
4963 //
4964 // TODO: This defaults to shared right now.
4965 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4966 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4967 // The next line is appropriate only for variables (Val) with the
4968 // data-sharing attribute "shared".
4969 ReplVal = &Val;
4970
4971 return CodeGenIP;
4972 };
4973
4974 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
4975 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
4976 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4977 AllocaInsertPt->getIterator());
4978 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4979 cantFail(ValOrErr: OMPBuilder.createSections(
4980 Loc: Builder, AllocaIP, SectionCBs: SectionCBVector, PrivCB, FiniCB, IsCancellable: S.hasCancel(),
4981 IsNowait: S.getSingleClause<OMPNowaitClause>()));
4982 Builder.restoreIP(IP: AfterIP);
4983 return;
4984 }
4985 {
4986 auto LPCRegion =
4987 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4988 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4989 EmitSections(S);
4990 }
4991 // Emit an implicit barrier at the end.
4992 if (!S.getSingleClause<OMPNowaitClause>()) {
4993 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(),
4994 Kind: OMPD_sections);
4995 }
4996 // Check for outer lastprivate conditional update.
4997 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4998}
4999
5000void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
5001 if (CGM.getLangOpts().OpenMPIRBuilder) {
5002 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5003 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5004
5005 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
5006 auto FiniCB = [this](InsertPointTy IP) {
5007 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5008 return llvm::Error::success();
5009 };
5010
5011 auto BodyGenCB = [SectionRegionBodyStmt,
5012 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5013 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5014 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5015 CGF&: *this, RegionBodyStmt: SectionRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "section");
5016 return llvm::Error::success();
5017 };
5018
5019 LexicalScope Scope(*this, S.getSourceRange());
5020 EmitStopPoint(S: &S);
5021 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5022 cantFail(ValOrErr: OMPBuilder.createSection(Loc: Builder, BodyGenCB, FiniCB));
5023 Builder.restoreIP(IP: AfterIP);
5024
5025 return;
5026 }
5027 LexicalScope Scope(*this, S.getSourceRange());
5028 EmitStopPoint(S: &S);
5029 EmitStmt(S: S.getAssociatedStmt());
5030}
5031
5032void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
5033 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
5034 llvm::SmallVector<const Expr *, 8> DestExprs;
5035 llvm::SmallVector<const Expr *, 8> SrcExprs;
5036 llvm::SmallVector<const Expr *, 8> AssignmentOps;
5037 // Check if there are any 'copyprivate' clauses associated with this
5038 // 'single' construct.
5039 // Build a list of copyprivate variables along with helper expressions
5040 // (<source>, <destination>, <destination>=<source> expressions)
5041 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
5042 CopyprivateVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5043 DestExprs.append(in_start: C->destination_exprs().begin(),
5044 in_end: C->destination_exprs().end());
5045 SrcExprs.append(in_start: C->source_exprs().begin(), in_end: C->source_exprs().end());
5046 AssignmentOps.append(in_start: C->assignment_ops().begin(),
5047 in_end: C->assignment_ops().end());
5048 }
5049 // Emit code for 'single' region along with 'copyprivate' clauses
5050 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5051 Action.Enter(CGF);
5052 OMPPrivateScope SingleScope(CGF);
5053 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope&: SingleScope);
5054 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: SingleScope);
5055 (void)SingleScope.Privatize();
5056 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
5057 };
5058 {
5059 auto LPCRegion =
5060 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5061 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5062 CGM.getOpenMPRuntime().emitSingleRegion(CGF&: *this, SingleOpGen: CodeGen, Loc: S.getBeginLoc(),
5063 CopyprivateVars, DestExprs,
5064 SrcExprs, AssignmentOps);
5065 }
5066 // Emit an implicit barrier at the end (to avoid data race on firstprivate
5067 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
5068 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
5069 CGM.getOpenMPRuntime().emitBarrierCall(
5070 CGF&: *this, Loc: S.getBeginLoc(),
5071 Kind: S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
5072 }
5073 // Check for outer lastprivate conditional update.
5074 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5075}
5076
5077static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
5078 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5079 Action.Enter(CGF);
5080 CGF.EmitStmt(S: S.getRawStmt());
5081 };
5082 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
5083}
5084
5085void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
5086 if (CGM.getLangOpts().OpenMPIRBuilder) {
5087 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5088 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5089
5090 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
5091
5092 auto FiniCB = [this](InsertPointTy IP) {
5093 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5094 return llvm::Error::success();
5095 };
5096
5097 auto BodyGenCB = [MasterRegionBodyStmt,
5098 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5099 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5100 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5101 CGF&: *this, RegionBodyStmt: MasterRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "master");
5102 return llvm::Error::success();
5103 };
5104
5105 LexicalScope Scope(*this, S.getSourceRange());
5106 EmitStopPoint(S: &S);
5107 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5108 cantFail(ValOrErr: OMPBuilder.createMaster(Loc: Builder, BodyGenCB, FiniCB));
5109 Builder.restoreIP(IP: AfterIP);
5110
5111 return;
5112 }
5113 LexicalScope Scope(*this, S.getSourceRange());
5114 EmitStopPoint(S: &S);
5115 emitMaster(CGF&: *this, S);
5116}
5117
5118static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
5119 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5120 Action.Enter(CGF);
5121 CGF.EmitStmt(S: S.getRawStmt());
5122 };
5123 Expr *Filter = nullptr;
5124 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
5125 Filter = FilterClause->getThreadID();
5126 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: CodeGen, Loc: S.getBeginLoc(),
5127 Filter);
5128}
5129
5130void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {
5131 if (CGM.getLangOpts().OpenMPIRBuilder) {
5132 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5133 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5134
5135 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
5136 const Expr *Filter = nullptr;
5137 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
5138 Filter = FilterClause->getThreadID();
5139 llvm::Value *FilterVal = Filter
5140 ? EmitScalarExpr(E: Filter, IgnoreResultAssign: CGM.Int32Ty)
5141 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
5142
5143 auto FiniCB = [this](InsertPointTy IP) {
5144 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5145 return llvm::Error::success();
5146 };
5147
5148 auto BodyGenCB = [MaskedRegionBodyStmt,
5149 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5150 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5151 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5152 CGF&: *this, RegionBodyStmt: MaskedRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "masked");
5153 return llvm::Error::success();
5154 };
5155
5156 LexicalScope Scope(*this, S.getSourceRange());
5157 EmitStopPoint(S: &S);
5158 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
5159 ValOrErr: OMPBuilder.createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal));
5160 Builder.restoreIP(IP: AfterIP);
5161
5162 return;
5163 }
5164 LexicalScope Scope(*this, S.getSourceRange());
5165 EmitStopPoint(S: &S);
5166 emitMasked(CGF&: *this, S);
5167}
5168
5169void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
5170 if (CGM.getLangOpts().OpenMPIRBuilder) {
5171 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5172 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5173
5174 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
5175 const Expr *Hint = nullptr;
5176 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
5177 Hint = HintClause->getHint();
5178
5179 // TODO: This is slightly different from what's currently being done in
5180 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
5181 // about typing is final.
5182 llvm::Value *HintInst = nullptr;
5183 if (Hint)
5184 HintInst =
5185 Builder.CreateIntCast(V: EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, isSigned: false);
5186
5187 auto FiniCB = [this](InsertPointTy IP) {
5188 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
5189 return llvm::Error::success();
5190 };
5191
5192 auto BodyGenCB = [CriticalRegionBodyStmt,
5193 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5194 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5195 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
5196 CGF&: *this, RegionBodyStmt: CriticalRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "critical");
5197 return llvm::Error::success();
5198 };
5199
5200 LexicalScope Scope(*this, S.getSourceRange());
5201 EmitStopPoint(S: &S);
5202 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5203 cantFail(ValOrErr: OMPBuilder.createCritical(Loc: Builder, BodyGenCB, FiniCB,
5204 CriticalName: S.getDirectiveName().getAsString(),
5205 HintInst));
5206 Builder.restoreIP(IP: AfterIP);
5207
5208 return;
5209 }
5210
5211 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5212 Action.Enter(CGF);
5213 CGF.EmitStmt(S: S.getAssociatedStmt());
5214 };
5215 const Expr *Hint = nullptr;
5216 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
5217 Hint = HintClause->getHint();
5218 LexicalScope Scope(*this, S.getSourceRange());
5219 EmitStopPoint(S: &S);
5220 CGM.getOpenMPRuntime().emitCriticalRegion(CGF&: *this,
5221 CriticalName: S.getDirectiveName().getAsString(),
5222 CriticalOpGen: CodeGen, Loc: S.getBeginLoc(), Hint);
5223}
5224
5225void CodeGenFunction::EmitOMPParallelForDirective(
5226 const OMPParallelForDirective &S) {
5227 // Emit directive as a combined directive that consists of two implicit
5228 // directives: 'parallel' with 'for' directive.
5229 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5230 Action.Enter(CGF);
5231 emitOMPCopyinClause(CGF, S);
5232 (void)emitWorksharingDirective(CGF, S, HasCancel: S.hasCancel());
5233 };
5234 {
5235 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5236 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
5237 CGCapturedStmtInfo CGSI(CR_OpenMP);
5238 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5239 OMPLoopScope LoopScope(CGF, S);
5240 return CGF.EmitScalarExpr(E: S.getNumIterations());
5241 };
5242 bool IsInscan = llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
5243 P: [](const OMPReductionClause *C) {
5244 return C->getModifier() == OMPC_REDUCTION_inscan;
5245 });
5246 if (IsInscan)
5247 emitScanBasedDirectiveDecls(CGF&: *this, S, NumIteratorsGen);
5248 auto LPCRegion =
5249 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5250 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for, CodeGen,
5251 CodeGenBoundParameters: emitEmptyBoundParameters);
5252 if (IsInscan)
5253 emitScanBasedDirectiveFinals(CGF&: *this, S, NumIteratorsGen);
5254 }
5255 // Check for outer lastprivate conditional update.
5256 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5257}
5258
5259void CodeGenFunction::EmitOMPParallelForSimdDirective(
5260 const OMPParallelForSimdDirective &S) {
5261 // Emit directive as a combined directive that consists of two implicit
5262 // directives: 'parallel' with 'for' directive.
5263 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5264 Action.Enter(CGF);
5265 emitOMPCopyinClause(CGF, S);
5266 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
5267 };
5268 {
5269 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5270 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
5271 CGCapturedStmtInfo CGSI(CR_OpenMP);
5272 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5273 OMPLoopScope LoopScope(CGF, S);
5274 return CGF.EmitScalarExpr(E: S.getNumIterations());
5275 };
5276 bool IsInscan = llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
5277 P: [](const OMPReductionClause *C) {
5278 return C->getModifier() == OMPC_REDUCTION_inscan;
5279 });
5280 if (IsInscan)
5281 emitScanBasedDirectiveDecls(CGF&: *this, S, NumIteratorsGen);
5282 auto LPCRegion =
5283 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5284 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for_simd, CodeGen,
5285 CodeGenBoundParameters: emitEmptyBoundParameters);
5286 if (IsInscan)
5287 emitScanBasedDirectiveFinals(CGF&: *this, S, NumIteratorsGen);
5288 }
5289 // Check for outer lastprivate conditional update.
5290 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5291}
5292
5293void CodeGenFunction::EmitOMPParallelMasterDirective(
5294 const OMPParallelMasterDirective &S) {
5295 // Emit directive as a combined directive that consists of two implicit
5296 // directives: 'parallel' with 'master' directive.
5297 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5298 Action.Enter(CGF);
5299 OMPPrivateScope PrivateScope(CGF);
5300 emitOMPCopyinClause(CGF, S);
5301 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
5302 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
5303 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
5304 (void)PrivateScope.Privatize();
5305 emitMaster(CGF, S);
5306 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
5307 };
5308 {
5309 auto LPCRegion =
5310 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5311 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master, CodeGen,
5312 CodeGenBoundParameters: emitEmptyBoundParameters);
5313 emitPostUpdateForReductionClause(CGF&: *this, D: S,
5314 CondGen: [](CodeGenFunction &) { return nullptr; });
5315 }
5316 // Check for outer lastprivate conditional update.
5317 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5318}
5319
5320void CodeGenFunction::EmitOMPParallelMaskedDirective(
5321 const OMPParallelMaskedDirective &S) {
5322 // Emit directive as a combined directive that consists of two implicit
5323 // directives: 'parallel' with 'masked' directive.
5324 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5325 Action.Enter(CGF);
5326 OMPPrivateScope PrivateScope(CGF);
5327 emitOMPCopyinClause(CGF, S);
5328 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
5329 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
5330 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
5331 (void)PrivateScope.Privatize();
5332 emitMasked(CGF, S);
5333 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
5334 };
5335 {
5336 auto LPCRegion =
5337 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5338 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked, CodeGen,
5339 CodeGenBoundParameters: emitEmptyBoundParameters);
5340 emitPostUpdateForReductionClause(CGF&: *this, D: S,
5341 CondGen: [](CodeGenFunction &) { return nullptr; });
5342 }
5343 // Check for outer lastprivate conditional update.
5344 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5345}
5346
5347void CodeGenFunction::EmitOMPParallelSectionsDirective(
5348 const OMPParallelSectionsDirective &S) {
5349 // Emit directive as a combined directive that consists of two implicit
5350 // directives: 'parallel' with 'sections' directive.
5351 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5352 Action.Enter(CGF);
5353 emitOMPCopyinClause(CGF, S);
5354 CGF.EmitSections(S);
5355 };
5356 {
5357 auto LPCRegion =
5358 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5359 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_sections, CodeGen,
5360 CodeGenBoundParameters: emitEmptyBoundParameters);
5361 }
5362 // Check for outer lastprivate conditional update.
5363 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5364}
5365
5366namespace {
5367/// Get the list of variables declared in the context of the untied tasks.
5368class CheckVarsEscapingUntiedTaskDeclContext final
5369 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
5370 llvm::SmallVector<const VarDecl *, 4> PrivateDecls;
5371
5372public:
5373 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
5374 ~CheckVarsEscapingUntiedTaskDeclContext() = default;
5375 void VisitDeclStmt(const DeclStmt *S) {
5376 if (!S)
5377 return;
5378 // Need to privatize only local vars, static locals can be processed as is.
5379 for (const Decl *D : S->decls()) {
5380 if (const auto *VD = dyn_cast_or_null<VarDecl>(Val: D))
5381 if (VD->hasLocalStorage())
5382 PrivateDecls.push_back(Elt: VD);
5383 }
5384 }
5385 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
5386 void VisitCapturedStmt(const CapturedStmt *) {}
5387 void VisitLambdaExpr(const LambdaExpr *) {}
5388 void VisitBlockExpr(const BlockExpr *) {}
5389 void VisitStmt(const Stmt *S) {
5390 if (!S)
5391 return;
5392 for (const Stmt *Child : S->children())
5393 if (Child)
5394 Visit(S: Child);
5395 }
5396
5397 /// Swaps list of vars with the provided one.
5398 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
5399};
5400} // anonymous namespace
5401
5402static void buildDependences(const OMPExecutableDirective &S,
5403 OMPTaskDataTy &Data) {
5404
5405 // First look for 'omp_all_memory' and add this first.
5406 bool OmpAllMemory = false;
5407 if (llvm::any_of(
5408 Range: S.getClausesOfKind<OMPDependClause>(), P: [](const OMPDependClause *C) {
5409 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
5410 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
5411 })) {
5412 OmpAllMemory = true;
5413 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
5414 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
5415 // simplify.
5416 OMPTaskDataTy::DependData &DD =
5417 Data.Dependences.emplace_back(Args: OMPC_DEPEND_outallmemory,
5418 /*IteratorExpr=*/Args: nullptr);
5419 // Add a nullptr Expr to simplify the codegen in emitDependData.
5420 DD.DepExprs.push_back(Elt: nullptr);
5421 }
5422 // Add remaining dependences skipping any 'out' or 'inout' if they are
5423 // overridden by 'omp_all_memory'.
5424 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
5425 OpenMPDependClauseKind Kind = C->getDependencyKind();
5426 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
5427 continue;
5428 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
5429 continue;
5430 OMPTaskDataTy::DependData &DD =
5431 Data.Dependences.emplace_back(Args: C->getDependencyKind(), Args: C->getModifier());
5432 DD.DepExprs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5433 }
5434}
5435
5436void CodeGenFunction::EmitOMPTaskBasedDirective(
5437 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
5438 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
5439 OMPTaskDataTy &Data) {
5440 // Emit outlined function for task construct.
5441 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CapturedRegion);
5442 auto I = CS->getCapturedDecl()->param_begin();
5443 auto PartId = std::next(x: I);
5444 auto TaskT = std::next(x: I, n: 4);
5445 // Check if the task is final
5446 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
5447 // If the condition constant folds and can be elided, try to avoid emitting
5448 // the condition and the dead arm of the if/else.
5449 const Expr *Cond = Clause->getCondition();
5450 bool CondConstant;
5451 if (ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant))
5452 Data.Final.setInt(CondConstant);
5453 else
5454 Data.Final.setPointer(EvaluateExprAsBool(E: Cond));
5455 } else {
5456 // By default the task is not final.
5457 Data.Final.setInt(/*IntVal=*/false);
5458 }
5459 // Check if the task has 'priority' clause.
5460 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
5461 const Expr *Prio = Clause->getPriority();
5462 Data.Priority.setInt(/*IntVal=*/true);
5463 Data.Priority.setPointer(EmitScalarConversion(
5464 Src: EmitScalarExpr(E: Prio), SrcTy: Prio->getType(),
5465 DstTy: getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
5466 Loc: Prio->getExprLoc()));
5467 }
5468 // The first function argument for tasks is a thread id, the second one is a
5469 // part id (0 for tied tasks, >=0 for untied task).
5470 llvm::DenseSet<const ValueDecl *> EmittedAsPrivate;
5471 // Get list of private variables.
5472 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
5473 auto IRef = C->varlist_begin();
5474 for (const Expr *IInit : C->private_copies()) {
5475 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
5476 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
5477 .second) {
5478 Data.PrivateVars.push_back(Elt: *IRef);
5479 Data.PrivateCopies.push_back(Elt: IInit);
5480 }
5481 ++IRef;
5482 }
5483 }
5484 EmittedAsPrivate.clear();
5485 // Get list of firstprivate variables.
5486 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5487 auto IRef = C->varlist_begin();
5488 auto IElemInitRef = C->inits().begin();
5489 for (const Expr *IInit : C->private_copies()) {
5490 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
5491 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
5492 .second) {
5493 Data.FirstprivateVars.push_back(Elt: *IRef);
5494 Data.FirstprivateCopies.push_back(Elt: IInit);
5495 Data.FirstprivateInits.push_back(Elt: *IElemInitRef);
5496 }
5497 ++IRef;
5498 ++IElemInitRef;
5499 }
5500 }
5501 // Get list of lastprivate variables (for taskloops).
5502 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
5503 llvm::MapVector<const ValueDecl *, const DeclRefExpr *> LastprivateSrcsOrigs;
5504 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
5505 auto IRef = C->varlist_begin();
5506 auto ID = C->destination_exprs().begin();
5507 auto IS = C->source_exprs().begin();
5508 for (const Expr *IInit : C->private_copies()) {
5509 const auto *OrigDecl = cast<DeclRefExpr>(Val: *IRef)->getDecl();
5510 if (EmittedAsPrivate.insert(V: cast<ValueDecl>(Val: OrigDecl->getCanonicalDecl()))
5511 .second) {
5512 Data.LastprivateVars.push_back(Elt: *IRef);
5513 Data.LastprivateCopies.push_back(Elt: IInit);
5514 }
5515 LastprivateDstsOrigs.insert(KV: std::make_pair(
5516 x: cast<DeclRefExpr>(Val: *ID)->getDecl(), y: cast<DeclRefExpr>(Val: *IRef)));
5517 LastprivateSrcsOrigs.insert(KV: std::make_pair(
5518 x: cast<DeclRefExpr>(Val: *IS)->getDecl(), y: cast<DeclRefExpr>(Val: *IRef)));
5519 ++IRef;
5520 ++ID;
5521 ++IS;
5522 }
5523 }
5524 SmallVector<const Expr *, 4> LHSs;
5525 SmallVector<const Expr *, 4> RHSs;
5526 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
5527 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5528 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5529 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
5530 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
5531 in_end: C->reduction_ops().end());
5532 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
5533 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
5534 }
5535 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
5536 CGF&: *this, Loc: S.getBeginLoc(), LHSExprs: LHSs, RHSExprs: RHSs, Data);
5537 // Build list of dependences.
5538 buildDependences(S, Data);
5539 // Get list of local vars for untied tasks.
5540 if (!Data.Tied) {
5541 CheckVarsEscapingUntiedTaskDeclContext Checker;
5542 Checker.Visit(S: S.getInnermostCapturedStmt()->getCapturedStmt());
5543 Data.PrivateLocals.append(in_start: Checker.getPrivateDecls().begin(),
5544 in_end: Checker.getPrivateDecls().end());
5545 }
5546 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5547 &LastprivateSrcsOrigs, CapturedRegion](
5548 CodeGenFunction &CGF, PrePostActionTy &Action) {
5549 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
5550 std::pair<Address, Address>>
5551 UntiedLocalVars;
5552 // Set proper addresses for generated private copies.
5553 OMPPrivateScope Scope(CGF);
5554 // Generate debug info for variables present in shared clause.
5555 if (auto *DI = CGF.getDebugInfo()) {
5556 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5557 CGF.CapturedStmtInfo->getCaptureFields();
5558 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5559 if (CaptureFields.size() && ContextValue) {
5560 unsigned CharWidth = CGF.getContext().getCharWidth();
5561 // The shared variables are packed together as members of structure.
5562 // So the address of each shared variable can be computed by adding
5563 // offset of it (within record) to the base address of record. For each
5564 // shared variable, debug intrinsic llvm.dbg.declare is generated with
5565 // appropriate expressions (DIExpression).
5566 // Ex:
5567 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
5568 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5569 // metadata !svar1,
5570 // metadata !DIExpression(DW_OP_deref))
5571 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5572 // metadata !svar2,
5573 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
5574 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5575 const VarDecl *SharedVar = It->first;
5576 RecordDecl *CaptureRecord = It->second->getParent();
5577 const ASTRecordLayout &Layout =
5578 CGF.getContext().getASTRecordLayout(D: CaptureRecord);
5579 unsigned Offset =
5580 Layout.getFieldOffset(FieldNo: It->second->getFieldIndex()) / CharWidth;
5581 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5582 (void)DI->EmitDeclareOfAutoVariable(Decl: SharedVar, AI: ContextValue,
5583 Builder&: CGF.Builder, UsePointerValue: false);
5584 // Get the call dbg.declare instruction we just created and update
5585 // its DIExpression to add offset to base address.
5586 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,
5587 unsigned Offset) {
5588 SmallVector<uint64_t, 8> Ops;
5589 // Add offset to the base address if non zero.
5590 if (Offset) {
5591 Ops.push_back(Elt: llvm::dwarf::DW_OP_plus_uconst);
5592 Ops.push_back(Elt: Offset);
5593 }
5594 Ops.push_back(Elt: llvm::dwarf::DW_OP_deref);
5595 Declare->setExpression(llvm::DIExpression::get(Context&: Ctx, Elements: Ops));
5596 };
5597 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
5598 // If we're emitting using the new debug info format into a block
5599 // without a terminator, the record will be "trailing".
5600 assert(!Last.isTerminator() && "unexpected terminator");
5601 if (auto *Marker =
5602 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5603 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5604 C: llvm::filterDbgVars(R: Marker->getDbgRecordRange()))) {
5605 UpdateExpr(Last.getContext(), &DVR, Offset);
5606 break;
5607 }
5608 }
5609 }
5610 }
5611 }
5612 llvm::SmallVector<std::pair<const ValueDecl *, Address>, 16>
5613 FirstprivatePtrs;
5614 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
5615 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
5616 enum { PrivatesParam = 2, CopyFnParam = 3 };
5617 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5618 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: CopyFnParam)));
5619 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(
5620 VD: CS->getCapturedDecl()->getParam(i: PrivatesParam)));
5621 // Map privates.
5622 llvm::SmallVector<std::pair<const ValueDecl *, Address>, 16> PrivatePtrs;
5623 llvm::SmallVector<llvm::Value *, 16> CallArgs;
5624 llvm::SmallVector<llvm::Type *, 4> ParamTypes;
5625 CallArgs.push_back(Elt: PrivatesPtr);
5626 ParamTypes.push_back(Elt: PrivatesPtr->getType());
5627 for (const Expr *E : Data.PrivateVars) {
5628 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
5629 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5630 T: CGF.getContext().getPointerType(T: E->getType()), Name: ".priv.ptr.addr");
5631 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5632 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5633 ParamTypes.push_back(Elt: PrivatePtr.getType());
5634 }
5635 for (const Expr *E : Data.FirstprivateVars) {
5636 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
5637 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5638 T: CGF.getContext().getPointerType(T: E->getType()),
5639 Name: ".firstpriv.ptr.addr");
5640 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5641 FirstprivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5642 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5643 ParamTypes.push_back(Elt: PrivatePtr.getType());
5644 }
5645 for (const Expr *E : Data.LastprivateVars) {
5646 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
5647 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5648 T: CGF.getContext().getPointerType(T: E->getType()),
5649 Name: ".lastpriv.ptr.addr");
5650 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5651 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5652 ParamTypes.push_back(Elt: PrivatePtr.getType());
5653 }
5654 for (const VarDecl *VD : Data.PrivateLocals) {
5655 QualType Ty = VD->getType().getNonReferenceType();
5656 if (VD->getType()->isLValueReferenceType())
5657 Ty = CGF.getContext().getPointerType(T: Ty);
5658 if (isAllocatableDecl(VD))
5659 Ty = CGF.getContext().getPointerType(T: Ty);
5660 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5661 T: CGF.getContext().getPointerType(T: Ty), Name: ".local.ptr.addr");
5662 auto Result = UntiedLocalVars.insert(
5663 KV: std::make_pair(x&: VD, y: std::make_pair(x&: PrivatePtr, y: Address::invalid())));
5664 // If key exists update in place.
5665 if (Result.second == false)
5666 *Result.first = std::make_pair(
5667 x&: VD, y: std::make_pair(x&: PrivatePtr, y: Address::invalid()));
5668 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5669 ParamTypes.push_back(Elt: PrivatePtr.getType());
5670 }
5671 auto *CopyFnTy = llvm::FunctionType::get(Result: CGF.Builder.getVoidTy(),
5672 Params: ParamTypes, /*isVarArg=*/false);
5673 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5674 CGF, Loc: S.getBeginLoc(), OutlinedFn: {CopyFnTy, CopyFn}, Args: CallArgs);
5675 for (const auto &Pair : LastprivateDstsOrigs) {
5676 const auto *OrigDecl = Pair.second->getDecl();
5677 if (const auto *BD = dyn_cast<BindingDecl>(Val: OrigDecl)) {
5678 // For BindingDecls, emit the binding's LValue directly.
5679 Address OrigAddr =
5680 CGF.EmitOMPBindingOriginalAddr(BD, Loc: Pair.second->getExprLoc());
5681 Scope.addPrivate(LocalVD: Pair.first, Addr: OrigAddr);
5682 } else {
5683 const auto *OrigVD = cast<VarDecl>(Val: OrigDecl);
5684 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
5685 /*RefersToEnclosingVariableOrCapture=*/
5686 CGF.CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
5687 Pair.second->getType(), VK_LValue,
5688 Pair.second->getExprLoc());
5689 Scope.addPrivate(LocalVD: Pair.first, Addr: CGF.EmitLValue(E: &DRE).getAddress());
5690 }
5691 }
5692 for (const auto &Pair : PrivatePtrs) {
5693 Address Replacement = Address(
5694 CGF.Builder.CreateLoad(Addr: Pair.second),
5695 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5696 CGF.getContext().getDeclAlign(D: Pair.first));
5697 Scope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5698
5699 // For BindingDecls with lastprivate, also map the .lastprivate.src
5700 // pseudo-variable to the same private address.
5701 if (isa<BindingDecl>(Val: Pair.first)) {
5702 for (const auto &SrcPair : LastprivateSrcsOrigs) {
5703 if (SrcPair.second->getDecl() == Pair.first) {
5704 Scope.addPrivate(LocalVD: SrcPair.first, Addr: Replacement);
5705 break;
5706 }
5707 }
5708 }
5709
5710 if (auto *DI = CGF.getDebugInfo())
5711 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5712 // Only emit debug info for VarDecls, not BindingDecls.
5713 if (const auto *VD = dyn_cast<VarDecl>(Val: Pair.first))
5714 (void)DI->EmitDeclareOfAutoVariable(
5715 Decl: VD, AI: Pair.second.getBasePointer(), Builder&: CGF.Builder,
5716 /*UsePointerValue*/ true);
5717 }
5718 // Adjust mapping for internal locals by mapping actual memory instead of
5719 // a pointer to this memory.
5720 for (auto &Pair : UntiedLocalVars) {
5721 QualType VDType = Pair.first->getType().getNonReferenceType();
5722 if (Pair.first->getType()->isLValueReferenceType())
5723 VDType = CGF.getContext().getPointerType(T: VDType);
5724 if (isAllocatableDecl(VD: Pair.first)) {
5725 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Pair.second.first);
5726 Address Replacement(
5727 Ptr,
5728 CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: VDType)),
5729 CGF.getPointerAlign());
5730 Pair.second.first = Replacement;
5731 Ptr = CGF.Builder.CreateLoad(Addr: Replacement);
5732 Replacement = Address(Ptr, CGF.ConvertTypeForMem(T: VDType),
5733 CGF.getContext().getDeclAlign(D: Pair.first));
5734 Pair.second.second = Replacement;
5735 } else {
5736 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Pair.second.first);
5737 Address Replacement(Ptr, CGF.ConvertTypeForMem(T: VDType),
5738 CGF.getContext().getDeclAlign(D: Pair.first));
5739 Pair.second.first = Replacement;
5740 }
5741 }
5742 }
5743 if (Data.Reductions) {
5744 OMPPrivateScope FirstprivateScope(CGF);
5745 for (const auto &Pair : FirstprivatePtrs) {
5746 Address Replacement(
5747 CGF.Builder.CreateLoad(Addr: Pair.second),
5748 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5749 CGF.getContext().getDeclAlign(D: Pair.first));
5750 FirstprivateScope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5751 }
5752 (void)FirstprivateScope.Privatize();
5753 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5754 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5755 Data.ReductionCopies, Data.ReductionOps);
5756 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5757 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: 9)));
5758 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5759 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5760 RedCG.emitAggregateType(CGF, N: Cnt);
5761 // FIXME: This must removed once the runtime library is fixed.
5762 // Emit required threadprivate variables for
5763 // initializer/combiner/finalizer.
5764 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5765 RCG&: RedCG, N: Cnt);
5766 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5767 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5768 Replacement = Address(
5769 CGF.EmitScalarConversion(Src: Replacement.emitRawPointer(CGF),
5770 SrcTy: CGF.getContext().VoidPtrTy,
5771 DstTy: CGF.getContext().getPointerType(
5772 T: Data.ReductionCopies[Cnt]->getType()),
5773 Loc: Data.ReductionCopies[Cnt]->getExprLoc()),
5774 CGF.ConvertTypeForMem(T: Data.ReductionCopies[Cnt]->getType()),
5775 Replacement.getAlignment());
5776 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5777 Scope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5778 }
5779 }
5780 // Privatize all private variables except for in_reduction items.
5781 (void)Scope.Privatize();
5782 SmallVector<const Expr *, 4> InRedVars;
5783 SmallVector<const Expr *, 4> InRedPrivs;
5784 SmallVector<const Expr *, 4> InRedOps;
5785 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5786 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5787 auto IPriv = C->privates().begin();
5788 auto IRed = C->reduction_ops().begin();
5789 auto ITD = C->taskgroup_descriptors().begin();
5790 for (const Expr *Ref : C->varlist()) {
5791 InRedVars.emplace_back(Args&: Ref);
5792 InRedPrivs.emplace_back(Args: *IPriv);
5793 InRedOps.emplace_back(Args: *IRed);
5794 TaskgroupDescriptors.emplace_back(Args: *ITD);
5795 std::advance(i&: IPriv, n: 1);
5796 std::advance(i&: IRed, n: 1);
5797 std::advance(i&: ITD, n: 1);
5798 }
5799 }
5800 // Privatize in_reduction items here, because taskgroup descriptors must be
5801 // privatized earlier.
5802 OMPPrivateScope InRedScope(CGF);
5803 if (!InRedVars.empty()) {
5804 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5805 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5806 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5807 RedCG.emitAggregateType(CGF, N: Cnt);
5808 // The taskgroup descriptor variable is always implicit firstprivate and
5809 // privatized already during processing of the firstprivates.
5810 // FIXME: This must removed once the runtime library is fixed.
5811 // Emit required threadprivate variables for
5812 // initializer/combiner/finalizer.
5813 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5814 RCG&: RedCG, N: Cnt);
5815 llvm::Value *ReductionsPtr;
5816 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5817 ReductionsPtr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: TRExpr),
5818 Loc: TRExpr->getExprLoc());
5819 } else {
5820 ReductionsPtr = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
5821 }
5822 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5823 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5824 Replacement = Address(
5825 CGF.EmitScalarConversion(
5826 Src: Replacement.emitRawPointer(CGF), SrcTy: CGF.getContext().VoidPtrTy,
5827 DstTy: CGF.getContext().getPointerType(T: InRedPrivs[Cnt]->getType()),
5828 Loc: InRedPrivs[Cnt]->getExprLoc()),
5829 CGF.ConvertTypeForMem(T: InRedPrivs[Cnt]->getType()),
5830 Replacement.getAlignment());
5831 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5832 InRedScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5833 }
5834 }
5835 (void)InRedScope.Privatize();
5836
5837 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,
5838 UntiedLocalVars);
5839 Action.Enter(CGF);
5840 BodyGen(CGF);
5841 };
5842 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
5843 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5844 D: S, ThreadIDVar: *I, PartIDVar: *PartId, TaskTVar: *TaskT, InnermostKind: EKind, CodeGen, Tied: Data.Tied, NumberOfParts&: Data.NumberOfParts);
5845 OMPLexicalScope Scope(*this, S, std::nullopt,
5846 !isOpenMPParallelDirective(DKind: EKind) &&
5847 !isOpenMPSimdDirective(DKind: EKind));
5848 TaskGen(*this, OutlinedFn, Data);
5849}
5850
5851static ImplicitParamDecl *
5852createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
5853 QualType Ty, CapturedDecl *CD,
5854 SourceLocation Loc) {
5855 auto *OrigVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: Ty,
5856 ParamKind: ImplicitParamKind::Other);
5857 auto *OrigRef = DeclRefExpr::Create(
5858 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: OrigVD,
5859 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: Ty, VK: VK_LValue);
5860 auto *PrivateVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: Ty,
5861 ParamKind: ImplicitParamKind::Other);
5862 auto *PrivateRef = DeclRefExpr::Create(
5863 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: PrivateVD,
5864 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: Ty, VK: VK_LValue);
5865 QualType ElemType = C.getBaseElementType(QT: Ty);
5866 auto *InitVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: ElemType,
5867 ParamKind: ImplicitParamKind::Other);
5868 auto *InitRef = DeclRefExpr::Create(
5869 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: InitVD,
5870 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: ElemType, VK: VK_LValue);
5871 PrivateVD->setInitStyle(VarDecl::CInit);
5872 PrivateVD->setInit(ImplicitCastExpr::Create(Context: C, T: ElemType, Kind: CK_LValueToRValue,
5873 Operand: InitRef, /*BasePath=*/nullptr,
5874 Cat: VK_PRValue, FPO: FPOptionsOverride()));
5875 Data.FirstprivateVars.emplace_back(Args&: OrigRef);
5876 Data.FirstprivateCopies.emplace_back(Args&: PrivateRef);
5877 Data.FirstprivateInits.emplace_back(Args&: InitRef);
5878 return OrigVD;
5879}
5880
5881void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
5882 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
5883 OMPTargetDataInfo &InputInfo) {
5884 // Emit outlined function for task construct.
5885 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_task);
5886 Address CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
5887 CanQualType SharedsTy =
5888 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
5889 auto I = CS->getCapturedDecl()->param_begin();
5890 auto PartId = std::next(x: I);
5891 auto TaskT = std::next(x: I, n: 4);
5892 OMPTaskDataTy Data;
5893 // The task is not final.
5894 Data.Final.setInt(/*IntVal=*/false);
5895 // Get list of firstprivate variables.
5896 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5897 auto IRef = C->varlist_begin();
5898 auto IElemInitRef = C->inits().begin();
5899 for (auto *IInit : C->private_copies()) {
5900 Data.FirstprivateVars.push_back(Elt: *IRef);
5901 Data.FirstprivateCopies.push_back(Elt: IInit);
5902 Data.FirstprivateInits.push_back(Elt: *IElemInitRef);
5903 ++IRef;
5904 ++IElemInitRef;
5905 }
5906 }
5907 SmallVector<const Expr *, 4> LHSs;
5908 SmallVector<const Expr *, 4> RHSs;
5909 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5910 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5911 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5912 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
5913 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
5914 in_end: C->reduction_ops().end());
5915 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
5916 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
5917 }
5918 OMPPrivateScope TargetScope(*this);
5919 VarDecl *BPVD = nullptr;
5920 VarDecl *PVD = nullptr;
5921 VarDecl *SVD = nullptr;
5922 VarDecl *MVD = nullptr;
5923 if (InputInfo.NumberOfTargetItems > 0) {
5924 auto *CD = CapturedDecl::Create(
5925 C&: getContext(), DC: getContext().getTranslationUnitDecl(), /*NumParams=*/0);
5926 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
5927 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
5928 EltTy: getContext().VoidPtrTy, ArySize: ArrSize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5929 /*IndexTypeQuals=*/0);
5930 BPVD = createImplicitFirstprivateForType(
5931 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5932 PVD = createImplicitFirstprivateForType(
5933 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5934 QualType SizesType = getContext().getConstantArrayType(
5935 EltTy: getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
5936 ArySize: ArrSize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5937 /*IndexTypeQuals=*/0);
5938 SVD = createImplicitFirstprivateForType(C&: getContext(), Data, Ty: SizesType, CD,
5939 Loc: S.getBeginLoc());
5940 TargetScope.addPrivate(LocalVD: BPVD, Addr: InputInfo.BasePointersArray);
5941 TargetScope.addPrivate(LocalVD: PVD, Addr: InputInfo.PointersArray);
5942 TargetScope.addPrivate(LocalVD: SVD, Addr: InputInfo.SizesArray);
5943 // If there is no user-defined mapper, the mapper array will be nullptr. In
5944 // this case, we don't need to privatize it.
5945 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5946 Val: InputInfo.MappersArray.emitRawPointer(CGF&: *this))) {
5947 MVD = createImplicitFirstprivateForType(
5948 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5949 TargetScope.addPrivate(LocalVD: MVD, Addr: InputInfo.MappersArray);
5950 }
5951 }
5952 (void)TargetScope.Privatize();
5953 buildDependences(S, Data);
5954 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
5955 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5956 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
5957 // Set proper addresses for generated private copies.
5958 OMPPrivateScope Scope(CGF);
5959 if (!Data.FirstprivateVars.empty()) {
5960 enum { PrivatesParam = 2, CopyFnParam = 3 };
5961 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5962 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: CopyFnParam)));
5963 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(
5964 VD: CS->getCapturedDecl()->getParam(i: PrivatesParam)));
5965 // Map privates.
5966 llvm::SmallVector<std::pair<const ValueDecl *, Address>, 16> PrivatePtrs;
5967 llvm::SmallVector<llvm::Value *, 16> CallArgs;
5968 llvm::SmallVector<llvm::Type *, 4> ParamTypes;
5969 CallArgs.push_back(Elt: PrivatesPtr);
5970 ParamTypes.push_back(Elt: PrivatesPtr->getType());
5971 for (const Expr *E : Data.FirstprivateVars) {
5972 const auto *VD = cast<DeclRefExpr>(Val: E)->getDecl();
5973 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5974 T: CGF.getContext().getPointerType(T: E->getType()),
5975 Name: ".firstpriv.ptr.addr");
5976 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5977 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5978 ParamTypes.push_back(Elt: PrivatePtr.getType());
5979 }
5980 auto *CopyFnTy = llvm::FunctionType::get(Result: CGF.Builder.getVoidTy(),
5981 Params: ParamTypes, /*isVarArg=*/false);
5982 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5983 CGF, Loc: S.getBeginLoc(), OutlinedFn: {CopyFnTy, CopyFn}, Args: CallArgs);
5984 for (const auto &Pair : PrivatePtrs) {
5985 Address Replacement(
5986 CGF.Builder.CreateLoad(Addr: Pair.second),
5987 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5988 CGF.getContext().getDeclAlign(D: Pair.first));
5989 Scope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5990 }
5991 }
5992 CGF.processInReduction(S, Data, CGF, CS, Scope);
5993 if (InputInfo.NumberOfTargetItems > 0) {
5994 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5995 Addr: CGF.GetAddrOfLocalVar(VD: BPVD), /*Index=*/0);
5996 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5997 Addr: CGF.GetAddrOfLocalVar(VD: PVD), /*Index=*/0);
5998 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5999 Addr: CGF.GetAddrOfLocalVar(VD: SVD), /*Index=*/0);
6000 // If MVD is nullptr, the mapper array is not privatized
6001 if (MVD)
6002 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
6003 Addr: CGF.GetAddrOfLocalVar(VD: MVD), /*Index=*/0);
6004 }
6005
6006 Action.Enter(CGF);
6007 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
6008 auto *TL = S.getSingleClause<OMPThreadLimitClause>();
6009 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
6010 needsTaskBasedThreadLimit(DKind: EKind) && TL) {
6011 // Emit __kmpc_set_thread_limit() to set the thread_limit for the task
6012 // enclosing this target region. This will indirectly set the thread_limit
6013 // for every applicable construct within target region.
6014 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
6015 CGF, ThreadLimit: TL->getThreadLimit().front(), Loc: S.getBeginLoc());
6016 }
6017 BodyGen(CGF);
6018 };
6019 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
6020 D: S, ThreadIDVar: *I, PartIDVar: *PartId, TaskTVar: *TaskT, InnermostKind: EKind, CodeGen, /*Tied=*/true,
6021 NumberOfParts&: Data.NumberOfParts);
6022 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
6023 IntegerLiteral IfCond(getContext(), TrueOrFalse,
6024 getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
6025 SourceLocation());
6026 CGM.getOpenMPRuntime().emitTaskCall(CGF&: *this, Loc: S.getBeginLoc(), D: S, TaskFunction: OutlinedFn,
6027 SharedsTy, Shareds: CapturedStruct, IfCond: &IfCond, Data);
6028}
6029
6030void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
6031 OMPTaskDataTy &Data,
6032 CodeGenFunction &CGF,
6033 const CapturedStmt *CS,
6034 OMPPrivateScope &Scope) {
6035 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
6036 if (Data.Reductions) {
6037 OpenMPDirectiveKind CapturedRegion = EKind;
6038 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
6039 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
6040 Data.ReductionCopies, Data.ReductionOps);
6041 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
6042 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: 4)));
6043 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
6044 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
6045 RedCG.emitAggregateType(CGF, N: Cnt);
6046 // FIXME: This must removed once the runtime library is fixed.
6047 // Emit required threadprivate variables for
6048 // initializer/combiner/finalizer.
6049 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
6050 RCG&: RedCG, N: Cnt);
6051 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
6052 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
6053 Replacement = Address(
6054 CGF.EmitScalarConversion(Src: Replacement.emitRawPointer(CGF),
6055 SrcTy: CGF.getContext().VoidPtrTy,
6056 DstTy: CGF.getContext().getPointerType(
6057 T: Data.ReductionCopies[Cnt]->getType()),
6058 Loc: Data.ReductionCopies[Cnt]->getExprLoc()),
6059 CGF.ConvertTypeForMem(T: Data.ReductionCopies[Cnt]->getType()),
6060 Replacement.getAlignment());
6061 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
6062 Scope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
6063 }
6064 }
6065 (void)Scope.Privatize();
6066 SmallVector<const Expr *, 4> InRedVars;
6067 SmallVector<const Expr *, 4> InRedPrivs;
6068 SmallVector<const Expr *, 4> InRedOps;
6069 SmallVector<const Expr *, 4> TaskgroupDescriptors;
6070 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
6071 auto IPriv = C->privates().begin();
6072 auto IRed = C->reduction_ops().begin();
6073 auto ITD = C->taskgroup_descriptors().begin();
6074 for (const Expr *Ref : C->varlist()) {
6075 InRedVars.emplace_back(Args&: Ref);
6076 InRedPrivs.emplace_back(Args: *IPriv);
6077 InRedOps.emplace_back(Args: *IRed);
6078 TaskgroupDescriptors.emplace_back(Args: *ITD);
6079 std::advance(i&: IPriv, n: 1);
6080 std::advance(i&: IRed, n: 1);
6081 std::advance(i&: ITD, n: 1);
6082 }
6083 }
6084 OMPPrivateScope InRedScope(CGF);
6085 if (!InRedVars.empty()) {
6086 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
6087 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
6088 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
6089 RedCG.emitAggregateType(CGF, N: Cnt);
6090 // FIXME: This must removed once the runtime library is fixed.
6091 // Emit required threadprivate variables for
6092 // initializer/combiner/finalizer.
6093 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
6094 RCG&: RedCG, N: Cnt);
6095 llvm::Value *ReductionsPtr;
6096 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
6097 ReductionsPtr =
6098 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: TRExpr), Loc: TRExpr->getExprLoc());
6099 } else {
6100 ReductionsPtr = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6101 }
6102 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
6103 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
6104 Replacement = Address(
6105 CGF.EmitScalarConversion(
6106 Src: Replacement.emitRawPointer(CGF), SrcTy: CGF.getContext().VoidPtrTy,
6107 DstTy: CGF.getContext().getPointerType(T: InRedPrivs[Cnt]->getType()),
6108 Loc: InRedPrivs[Cnt]->getExprLoc()),
6109 CGF.ConvertTypeForMem(T: InRedPrivs[Cnt]->getType()),
6110 Replacement.getAlignment());
6111 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
6112 InRedScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
6113 }
6114 }
6115 (void)InRedScope.Privatize();
6116}
6117
6118void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
6119 // Emit outlined function for task construct.
6120 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_task);
6121 Address CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
6122 CanQualType SharedsTy =
6123 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
6124 const Expr *IfCond = nullptr;
6125 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
6126 if (C->getNameModifier() == OMPD_unknown ||
6127 C->getNameModifier() == OMPD_task) {
6128 IfCond = C->getCondition();
6129 break;
6130 }
6131 }
6132
6133 OMPTaskDataTy Data;
6134 // Check if we should emit tied or untied task.
6135 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
6136 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
6137 CGF.EmitStmt(S: CS->getCapturedStmt());
6138 };
6139 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
6140 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
6141 const OMPTaskDataTy &Data) {
6142 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, Loc: S.getBeginLoc(), D: S, TaskFunction: OutlinedFn,
6143 SharedsTy, Shareds: CapturedStruct, IfCond,
6144 Data);
6145 };
6146 auto LPCRegion =
6147 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
6148 EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_task, BodyGen, TaskGen, Data);
6149}
6150
6151void CodeGenFunction::EmitOMPTaskyieldDirective(
6152 const OMPTaskyieldDirective &S) {
6153 CGM.getOpenMPRuntime().emitTaskyieldCall(CGF&: *this, Loc: S.getBeginLoc());
6154}
6155
6156void CodeGenFunction::EmitOMPErrorDirective(const OMPErrorDirective &S) {
6157 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();
6158 Expr *ME = MC ? MC->getMessageString() : nullptr;
6159 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();
6160 bool IsFatal = false;
6161 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
6162 IsFatal = true;
6163 CGM.getOpenMPRuntime().emitErrorCall(CGF&: *this, Loc: S.getBeginLoc(), ME, IsFatal);
6164}
6165
6166void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
6167 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_barrier);
6168}
6169
6170void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
6171 OMPTaskDataTy Data;
6172 // Build list of dependences
6173 buildDependences(S, Data);
6174 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
6175 CGM.getOpenMPRuntime().emitTaskwaitCall(CGF&: *this, Loc: S.getBeginLoc(), Data);
6176}
6177
6178static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T) {
6179 return T.clauses().empty();
6180}
6181
6182void CodeGenFunction::EmitOMPTaskgroupDirective(
6183 const OMPTaskgroupDirective &S) {
6184 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6185 if (CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(T: S)) {
6186 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6187 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6188 InsertPointTy AllocaIP(AllocaInsertPt->getIterator());
6189
6190 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
6191 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
6192 Builder.restoreIP(IP: CodeGenIP);
6193 EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
6194 return llvm::Error::success();
6195 };
6196 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
6197 if (!CapturedStmtInfo)
6198 CapturedStmtInfo = &CapStmtInfo;
6199 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
6200 cantFail(ValOrErr: OMPBuilder.createTaskgroup(Loc: Builder, AllocaIP,
6201 /*DeallocBlocks=*/{}, BodyGenCB));
6202 Builder.restoreIP(IP: AfterIP);
6203 return;
6204 }
6205 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
6206 Action.Enter(CGF);
6207 if (const Expr *E = S.getReductionRef()) {
6208 SmallVector<const Expr *, 4> LHSs;
6209 SmallVector<const Expr *, 4> RHSs;
6210 OMPTaskDataTy Data;
6211 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
6212 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
6213 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
6214 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
6215 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
6216 in_end: C->reduction_ops().end());
6217 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
6218 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
6219 }
6220 llvm::Value *ReductionDesc =
6221 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, Loc: S.getBeginLoc(),
6222 LHSExprs: LHSs, RHSExprs: RHSs, Data);
6223 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
6224 CGF.EmitVarDecl(D: *VD);
6225 CGF.EmitStoreOfScalar(Value: ReductionDesc, Addr: CGF.GetAddrOfLocalVar(VD),
6226 /*Volatile=*/false, Ty: E->getType());
6227 }
6228 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
6229 };
6230 CGM.getOpenMPRuntime().emitTaskgroupRegion(CGF&: *this, TaskgroupOpGen: CodeGen, Loc: S.getBeginLoc());
6231}
6232
6233void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
6234 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
6235 ? llvm::AtomicOrdering::NotAtomic
6236 : llvm::AtomicOrdering::AcquireRelease;
6237 CGM.getOpenMPRuntime().emitFlush(
6238 CGF&: *this,
6239 Vars: [&S]() -> ArrayRef<const Expr *> {
6240 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
6241 return llvm::ArrayRef(FlushClause->varlist_begin(),
6242 FlushClause->varlist_end());
6243 return {};
6244 }(),
6245 Loc: S.getBeginLoc(), AO);
6246}
6247
6248void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {
6249 const auto *DO = S.getSingleClause<OMPDepobjClause>();
6250 LValue DOLVal = EmitLValue(E: DO->getDepobj());
6251 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
6252 // Build list and emit dependences
6253 OMPTaskDataTy Data;
6254 buildDependences(S, Data);
6255 for (auto &Dep : Data.Dependences) {
6256 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
6257 CGF&: *this, Dependencies: Dep, Loc: DC->getBeginLoc());
6258 EmitStoreOfScalar(value: DepAddr.emitRawPointer(CGF&: *this), lvalue: DOLVal);
6259 }
6260 return;
6261 }
6262 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
6263 CGM.getOpenMPRuntime().emitDestroyClause(CGF&: *this, DepobjLVal: DOLVal, Loc: DC->getBeginLoc());
6264 return;
6265 }
6266 if (const auto *UC = S.getSingleClause<OMPUpdateDependObjectsClause>()) {
6267 CGM.getOpenMPRuntime().emitUpdateDependObjectsClause(
6268 CGF&: *this, DepobjLVal: DOLVal, NewDepKind: UC->getDependencyKind(), Loc: UC->getBeginLoc());
6269 return;
6270 }
6271}
6272
6273void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) {
6274 if (!OMPParentLoopDirectiveForScan)
6275 return;
6276 const OMPExecutableDirective &ParentDir = *OMPParentLoopDirectiveForScan;
6277 bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();
6278 SmallVector<const Expr *, 4> Shareds;
6279 SmallVector<const Expr *, 4> Privates;
6280 SmallVector<const Expr *, 4> LHSs;
6281 SmallVector<const Expr *, 4> RHSs;
6282 SmallVector<const Expr *, 4> ReductionOps;
6283 SmallVector<const Expr *, 4> CopyOps;
6284 SmallVector<const Expr *, 4> CopyArrayTemps;
6285 SmallVector<const Expr *, 4> CopyArrayElems;
6286 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
6287 if (C->getModifier() != OMPC_REDUCTION_inscan)
6288 continue;
6289 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
6290 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
6291 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
6292 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
6293 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
6294 CopyOps.append(in_start: C->copy_ops().begin(), in_end: C->copy_ops().end());
6295 CopyArrayTemps.append(in_start: C->copy_array_temps().begin(),
6296 in_end: C->copy_array_temps().end());
6297 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
6298 in_end: C->copy_array_elems().end());
6299 }
6300 if (ParentDir.getDirectiveKind() == OMPD_simd ||
6301 (getLangOpts().OpenMPSimd &&
6302 isOpenMPSimdDirective(DKind: ParentDir.getDirectiveKind()))) {
6303 // For simd directive and simd-based directives in simd only mode, use the
6304 // following codegen:
6305 // int x = 0;
6306 // #pragma omp simd reduction(inscan, +: x)
6307 // for (..) {
6308 // <first part>
6309 // #pragma omp scan inclusive(x)
6310 // <second part>
6311 // }
6312 // is transformed to:
6313 // int x = 0;
6314 // for (..) {
6315 // int x_priv = 0;
6316 // <first part>
6317 // x = x_priv + x;
6318 // x_priv = x;
6319 // <second part>
6320 // }
6321 // and
6322 // int x = 0;
6323 // #pragma omp simd reduction(inscan, +: x)
6324 // for (..) {
6325 // <first part>
6326 // #pragma omp scan exclusive(x)
6327 // <second part>
6328 // }
6329 // to
6330 // int x = 0;
6331 // for (..) {
6332 // int x_priv = 0;
6333 // <second part>
6334 // int temp = x;
6335 // x = x_priv + x;
6336 // x_priv = temp;
6337 // <first part>
6338 // }
6339 llvm::BasicBlock *OMPScanReduce = createBasicBlock(name: "omp.inscan.reduce");
6340 EmitBranch(Block: IsInclusive
6341 ? OMPScanReduce
6342 : BreakContinueStack.back().ContinueBlock.getBlock());
6343 EmitBlock(BB: OMPScanDispatch);
6344 {
6345 // New scope for correct construction/destruction of temp variables for
6346 // exclusive scan.
6347 LexicalScope Scope(*this, S.getSourceRange());
6348 EmitBranch(Block: IsInclusive ? OMPBeforeScanBlock : OMPAfterScanBlock);
6349 EmitBlock(BB: OMPScanReduce);
6350 if (!IsInclusive) {
6351 // Create temp var and copy LHS value to this temp value.
6352 // TMP = LHS;
6353 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6354 const Expr *PrivateExpr = Privates[I];
6355 const Expr *TempExpr = CopyArrayTemps[I];
6356 EmitAutoVarDecl(
6357 D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: TempExpr)->getDecl()));
6358 LValue DestLVal = EmitLValue(E: TempExpr);
6359 LValue SrcLVal = EmitLValue(E: LHSs[I]);
6360 EmitOMPCopy(OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(),
6361 SrcAddr: SrcLVal.getAddress(),
6362 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6363 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()),
6364 Copy: CopyOps[I]);
6365 }
6366 }
6367 CGM.getOpenMPRuntime().emitReduction(
6368 CGF&: *this, Loc: ParentDir.getEndLoc(), Privates, LHSExprs: LHSs, RHSExprs: RHSs, ReductionOps,
6369 Options: {/*WithNowait=*/true, /*SimpleReduction=*/true,
6370 /*IsPrivateVarReduction*/ {}, .ReductionKind: OMPD_simd});
6371 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6372 const Expr *PrivateExpr = Privates[I];
6373 LValue DestLVal;
6374 LValue SrcLVal;
6375 if (IsInclusive) {
6376 DestLVal = EmitLValue(E: RHSs[I]);
6377 SrcLVal = EmitLValue(E: LHSs[I]);
6378 } else {
6379 const Expr *TempExpr = CopyArrayTemps[I];
6380 DestLVal = EmitLValue(E: RHSs[I]);
6381 SrcLVal = EmitLValue(E: TempExpr);
6382 }
6383 EmitOMPCopy(
6384 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6385 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6386 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6387 }
6388 }
6389 EmitBranch(Block: IsInclusive ? OMPAfterScanBlock : OMPBeforeScanBlock);
6390 OMPScanExitBlock = IsInclusive
6391 ? BreakContinueStack.back().ContinueBlock.getBlock()
6392 : OMPScanReduce;
6393 EmitBlock(BB: OMPAfterScanBlock);
6394 return;
6395 }
6396 if (!IsInclusive) {
6397 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6398 EmitBlock(BB: OMPScanExitBlock);
6399 }
6400 if (OMPFirstScanLoop) {
6401 // Emit buffer[i] = red; at the end of the input phase.
6402 const auto *IVExpr = cast<OMPLoopDirective>(Val: ParentDir)
6403 .getIterationVariable()
6404 ->IgnoreParenImpCasts();
6405 LValue IdxLVal = EmitLValue(E: IVExpr);
6406 llvm::Value *IdxVal = EmitLoadOfScalar(lvalue: IdxLVal, Loc: IVExpr->getExprLoc());
6407 IdxVal = Builder.CreateIntCast(V: IdxVal, DestTy: SizeTy, /*isSigned=*/false);
6408 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6409 const Expr *PrivateExpr = Privates[I];
6410 const Expr *OrigExpr = Shareds[I];
6411 const Expr *CopyArrayElem = CopyArrayElems[I];
6412 OpaqueValueMapping IdxMapping(
6413 *this,
6414 cast<OpaqueValueExpr>(
6415 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
6416 RValue::get(V: IdxVal));
6417 LValue DestLVal = EmitLValue(E: CopyArrayElem);
6418 LValue SrcLVal = EmitLValue(E: OrigExpr);
6419 EmitOMPCopy(
6420 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6421 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6422 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6423 }
6424 }
6425 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6426 if (IsInclusive) {
6427 EmitBlock(BB: OMPScanExitBlock);
6428 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6429 }
6430 EmitBlock(BB: OMPScanDispatch);
6431 if (!OMPFirstScanLoop) {
6432 // Emit red = buffer[i]; at the entrance to the scan phase.
6433 const auto *IVExpr = cast<OMPLoopDirective>(Val: ParentDir)
6434 .getIterationVariable()
6435 ->IgnoreParenImpCasts();
6436 LValue IdxLVal = EmitLValue(E: IVExpr);
6437 llvm::Value *IdxVal = EmitLoadOfScalar(lvalue: IdxLVal, Loc: IVExpr->getExprLoc());
6438 IdxVal = Builder.CreateIntCast(V: IdxVal, DestTy: SizeTy, /*isSigned=*/false);
6439 llvm::BasicBlock *ExclusiveExitBB = nullptr;
6440 if (!IsInclusive) {
6441 llvm::BasicBlock *ContBB = createBasicBlock(name: "omp.exclusive.dec");
6442 ExclusiveExitBB = createBasicBlock(name: "omp.exclusive.copy.exit");
6443 llvm::Value *Cmp = Builder.CreateIsNull(Arg: IdxVal);
6444 Builder.CreateCondBr(Cond: Cmp, True: ExclusiveExitBB, False: ContBB);
6445 EmitBlock(BB: ContBB);
6446 // Use idx - 1 iteration for exclusive scan.
6447 IdxVal = Builder.CreateNUWSub(LHS: IdxVal, RHS: llvm::ConstantInt::get(Ty: SizeTy, V: 1));
6448 }
6449 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6450 const Expr *PrivateExpr = Privates[I];
6451 const Expr *OrigExpr = Shareds[I];
6452 const Expr *CopyArrayElem = CopyArrayElems[I];
6453 OpaqueValueMapping IdxMapping(
6454 *this,
6455 cast<OpaqueValueExpr>(
6456 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
6457 RValue::get(V: IdxVal));
6458 LValue SrcLVal = EmitLValue(E: CopyArrayElem);
6459 LValue DestLVal = EmitLValue(E: OrigExpr);
6460 EmitOMPCopy(
6461 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6462 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6463 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6464 }
6465 if (!IsInclusive) {
6466 EmitBlock(BB: ExclusiveExitBB);
6467 }
6468 }
6469 EmitBranch(Block: (OMPFirstScanLoop == IsInclusive) ? OMPBeforeScanBlock
6470 : OMPAfterScanBlock);
6471 EmitBlock(BB: OMPAfterScanBlock);
6472}
6473
6474void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
6475 const CodeGenLoopTy &CodeGenLoop,
6476 Expr *IncExpr) {
6477 // Emit the loop iteration variable.
6478 const auto *IVExpr = cast<DeclRefExpr>(Val: S.getIterationVariable());
6479 const auto *IVDecl = cast<VarDecl>(Val: IVExpr->getDecl());
6480 EmitVarDecl(D: *IVDecl);
6481
6482 // Emit the iterations count variable.
6483 // If it is not a variable, Sema decided to calculate iterations count on each
6484 // iteration (e.g., it is foldable into a constant).
6485 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
6486 EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
6487 // Emit calculation of the iterations count.
6488 EmitIgnoredExpr(E: S.getCalcLastIteration());
6489 }
6490
6491 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
6492
6493 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
6494 bool HasLastprivateClause = false;
6495 // Check pre-condition.
6496 {
6497 OMPLoopScope PreInitScope(*this, S);
6498 // Skip the entire loop if we don't meet the precondition.
6499 // If the condition constant folds and can be elided, avoid emitting the
6500 // whole loop.
6501 bool CondConstant;
6502 llvm::BasicBlock *ContBlock = nullptr;
6503 if (ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
6504 if (!CondConstant)
6505 return;
6506 } else {
6507 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "omp.precond.then");
6508 ContBlock = createBasicBlock(name: "omp.precond.end");
6509 emitPreCond(CGF&: *this, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
6510 TrueCount: getProfileCount(S: &S));
6511 EmitBlock(BB: ThenBlock);
6512 incrementProfileCounter(S: &S);
6513 }
6514
6515 emitAlignedClause(CGF&: *this, D: S);
6516 // Emit 'then' code.
6517 {
6518 // Emit helper vars inits.
6519
6520 LValue LB = EmitOMPHelperVar(
6521 CGF&: *this, Helper: cast<DeclRefExpr>(Val: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6522 ? S.getCombinedLowerBoundVariable()
6523 : S.getLowerBoundVariable()));
6524 LValue UB = EmitOMPHelperVar(
6525 CGF&: *this, Helper: cast<DeclRefExpr>(Val: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6526 ? S.getCombinedUpperBoundVariable()
6527 : S.getUpperBoundVariable()));
6528 LValue ST =
6529 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()));
6530 LValue IL =
6531 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()));
6532
6533 OMPPrivateScope LoopScope(*this);
6534 if (EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope)) {
6535 // Emit implicit barrier to synchronize threads and avoid data races
6536 // on initialization of firstprivate variables and post-update of
6537 // lastprivate variables.
6538 CGM.getOpenMPRuntime().emitBarrierCall(
6539 CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
6540 /*ForceSimpleCall=*/true);
6541 }
6542 EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
6543 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()) &&
6544 !isOpenMPParallelDirective(DKind: S.getDirectiveKind()) &&
6545 !isOpenMPTeamsDirective(DKind: S.getDirectiveKind()))
6546 EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
6547 HasLastprivateClause = EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
6548 EmitOMPPrivateLoopCounters(S, LoopScope);
6549 (void)LoopScope.Privatize();
6550 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
6551 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF&: *this, D: S);
6552
6553 // Detect the distribute schedule kind and chunk.
6554 llvm::Value *Chunk = nullptr;
6555 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
6556 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
6557 ScheduleKind = C->getDistScheduleKind();
6558 if (const Expr *Ch = C->getChunkSize()) {
6559 Chunk = EmitScalarExpr(E: Ch);
6560 Chunk = EmitScalarConversion(Src: Chunk, SrcTy: Ch->getType(),
6561 DstTy: S.getIterationVariable()->getType(),
6562 Loc: S.getBeginLoc());
6563 }
6564 } else {
6565 // Default behaviour for dist_schedule clause.
6566 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6567 CGF&: *this, S, ScheduleKind, Chunk);
6568 }
6569 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
6570 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
6571
6572 // GPU fused schedule: omit the outer distribute loop and let the inner
6573 // worksharing loop schedule the flattened team/thread iteration space.
6574 if (canEmitGPUFusedDistSchedule(CGM, S, DKind: EKind)) {
6575 JumpDest LoopExit =
6576 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
6577 CodeGenLoop(*this, S, LoopExit);
6578 EmitBlock(BB: LoopExit.getBlock());
6579 } else {
6580 // OpenMP [2.10.8, distribute Construct, Description]
6581 // If dist_schedule is specified, kind must be static. If specified,
6582 // iterations are divided into chunks of size chunk_size, chunks are
6583 // assigned to the teams of the league in a round-robin fashion in the
6584 // order of the team number. When no chunk_size is specified, the
6585 // iteration space is divided into chunks that are approximately equal
6586 // in size, and at most one chunk is distributed to each team of the
6587 // league. The size of the chunks is unspecified in this case.
6588 bool StaticChunked =
6589 RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
6590 isOpenMPLoopBoundSharingDirective(Kind: EKind);
6591 if (RT.isStaticNonchunked(ScheduleKind,
6592 /* Chunked */ Chunk != nullptr) ||
6593 StaticChunked) {
6594 CGOpenMPRuntime::StaticRTInput StaticInit(
6595 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
6596 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6597 StaticChunked ? Chunk : nullptr);
6598 RT.emitDistributeStaticInit(CGF&: *this, Loc: S.getBeginLoc(), SchedKind: ScheduleKind,
6599 Values: StaticInit);
6600 JumpDest LoopExit =
6601 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
6602 // UB = min(UB, GlobalUB);
6603 EmitIgnoredExpr(E: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6604 ? S.getCombinedEnsureUpperBound()
6605 : S.getEnsureUpperBound());
6606 // IV = LB;
6607 EmitIgnoredExpr(E: isOpenMPLoopBoundSharingDirective(Kind: EKind)
6608 ? S.getCombinedInit()
6609 : S.getInit());
6610
6611 const Expr *Cond = isOpenMPLoopBoundSharingDirective(Kind: EKind)
6612 ? S.getCombinedCond()
6613 : S.getCond();
6614
6615 if (StaticChunked)
6616 Cond = S.getCombinedDistCond();
6617
6618 // For static unchunked schedules generate:
6619 //
6620 // 1. For distribute alone, codegen
6621 // while (idx <= UB) {
6622 // BODY;
6623 // ++idx;
6624 // }
6625 //
6626 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
6627 // while (idx <= UB) {
6628 // <CodeGen rest of pragma>(LB, UB);
6629 // idx += ST;
6630 // }
6631 //
6632 // For static chunk one schedule generate:
6633 //
6634 // while (IV <= GlobalUB) {
6635 // <CodeGen rest of pragma>(LB, UB);
6636 // LB += ST;
6637 // UB += ST;
6638 // UB = min(UB, GlobalUB);
6639 // IV = LB;
6640 // }
6641 //
6642 emitCommonSimdLoop(
6643 CGF&: *this, S,
6644 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6645 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()))
6646 CGF.EmitOMPSimdInit(D: S);
6647 },
6648 BodyCodeGen: [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
6649 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
6650 CGF.EmitOMPInnerLoop(
6651 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: Cond, IncExpr,
6652 BodyGen: [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6653 CodeGenLoop(CGF, S, LoopExit);
6654 },
6655 PostIncGen: [&S, StaticChunked](CodeGenFunction &CGF) {
6656 if (StaticChunked) {
6657 CGF.EmitIgnoredExpr(E: S.getCombinedNextLowerBound());
6658 CGF.EmitIgnoredExpr(E: S.getCombinedNextUpperBound());
6659 CGF.EmitIgnoredExpr(E: S.getCombinedEnsureUpperBound());
6660 CGF.EmitIgnoredExpr(E: S.getCombinedInit());
6661 }
6662 });
6663 });
6664 EmitBlock(BB: LoopExit.getBlock());
6665 // Tell the runtime we are done.
6666 RT.emitForStaticFinish(CGF&: *this, Loc: S.getEndLoc(), DKind: OMPD_distribute);
6667 } else {
6668 // Emit the outer loop, which requests its work chunk [LB..UB] from
6669 // runtime and runs the inner loop to process it.
6670 const OMPLoopArguments LoopArguments = {
6671 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6672 IL.getAddress(), Chunk};
6673 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArgs: LoopArguments,
6674 CodeGenLoopContent: CodeGenLoop);
6675 }
6676 }
6677 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind())) {
6678 EmitOMPSimdFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
6679 return CGF.Builder.CreateIsNotNull(
6680 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
6681 });
6682 }
6683 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()) &&
6684 !isOpenMPParallelDirective(DKind: S.getDirectiveKind()) &&
6685 !isOpenMPTeamsDirective(DKind: S.getDirectiveKind())) {
6686 EmitOMPReductionClauseFinal(D: S, ReductionKind: OMPD_simd);
6687 // Emit post-update of the reduction variables if IsLastIter != 0.
6688 emitPostUpdateForReductionClause(
6689 CGF&: *this, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
6690 return CGF.Builder.CreateIsNotNull(
6691 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
6692 });
6693 }
6694 // Emit final copy of the lastprivate variables if IsLastIter != 0.
6695 if (HasLastprivateClause) {
6696 EmitOMPLastprivateClauseFinal(
6697 D: S, /*NoFinals=*/false,
6698 IsLastIterCond: Builder.CreateIsNotNull(Arg: EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
6699 }
6700 }
6701
6702 // We're now done with the loop, so jump to the continuation block.
6703 if (ContBlock) {
6704 EmitBranch(Block: ContBlock);
6705 EmitBlock(BB: ContBlock, IsFinished: true);
6706 }
6707 }
6708}
6709
6710// Pass OMPLoopDirective (instead of OMPDistributeDirective) to make this
6711// function available for "loop bind(teams)", which maps to "distribute".
6712static void emitOMPDistributeDirective(const OMPLoopDirective &S,
6713 CodeGenFunction &CGF,
6714 CodeGenModule &CGM) {
6715 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6716 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
6717 };
6718 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
6719 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute, CodeGen);
6720}
6721
6722void CodeGenFunction::EmitOMPDistributeDirective(
6723 const OMPDistributeDirective &S) {
6724 emitOMPDistributeDirective(S, CGF&: *this, CGM);
6725}
6726
6727static llvm::Function *
6728emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S,
6729 const OMPExecutableDirective &D) {
6730 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
6731 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
6732 CGF.CapturedStmtInfo = &CapStmtInfo;
6733 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(S: *S, D);
6734 Fn->setDoesNotRecurse();
6735 return Fn;
6736}
6737
6738template <typename T>
6739static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
6740 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6741 llvm::OpenMPIRBuilder &OMPBuilder) {
6742
6743 unsigned NumLoops = C->getNumLoops();
6744 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth(
6745 /*DestWidth=*/64, /*Signed=*/1);
6746 llvm::SmallVector<llvm::Value *> StoreValues;
6747 for (unsigned I = 0; I < NumLoops; I++) {
6748 const Expr *CounterVal = C->getLoopData(I);
6749 assert(CounterVal);
6750 llvm::Value *StoreValue = CGF.EmitScalarConversion(
6751 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
6752 Loc: CounterVal->getExprLoc());
6753 StoreValues.emplace_back(Args&: StoreValue);
6754 }
6755 OMPDoacrossKind<T> ODK;
6756 bool IsDependSource = ODK.isSource(C);
6757 CGF.Builder.restoreIP(
6758 IP: OMPBuilder.createOrderedDepend(Loc: CGF.Builder, AllocaIP, NumLoops,
6759 StoreValues, Name: ".cnt.addr", IsDependSource));
6760}
6761
6762void CodeGenFunction::EmitOMPOrderedStandaloneDirective(
6763 const OMPOrderedStandaloneDirective &S) {
6764 assert((S.hasClausesOfKind<OMPDependClause>() ||
6765 S.hasClausesOfKind<OMPDoacrossClause>()) &&
6766 "Standalone ordered directive should have either depend or doacross "
6767 "clause");
6768 // The ordered-standalone directive.
6769 assert(!S.hasAssociatedStmt() && "No associated statement must be in "
6770 "ordered depend|doacross construct.");
6771
6772 if (CGM.getLangOpts().OpenMPIRBuilder) {
6773 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6774 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6775
6776 InsertPointTy AllocaIP(AllocaInsertPt->getIterator());
6777 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6778 emitRestoreIP(CGF&: *this, C: DC, AllocaIP, OMPBuilder);
6779 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6780 emitRestoreIP(CGF&: *this, C: DC, AllocaIP, OMPBuilder);
6781 return;
6782 }
6783
6784 if (S.hasClausesOfKind<OMPDependClause>()) {
6785 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6786 CGM.getOpenMPRuntime().emitDoacrossOrdered(CGF&: *this, C: DC);
6787 } else if (S.hasClausesOfKind<OMPDoacrossClause>()) {
6788 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6789 CGM.getOpenMPRuntime().emitDoacrossOrdered(CGF&: *this, C: DC);
6790 }
6791}
6792
6793void CodeGenFunction::EmitOMPOrderedBlockAssocDirective(
6794 const OMPOrderedBlockAssocDirective &S) {
6795 if (CGM.getLangOpts().OpenMPIRBuilder) {
6796 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6797 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6798
6799 // The ordered directive with threads or simd clause, or without clause.
6800 // Without clause, it behaves as if the threads clause is specified.
6801 const auto *C = S.getSingleClause<OMPSIMDClause>();
6802
6803 auto FiniCB = [this](InsertPointTy IP) {
6804 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
6805 return llvm::Error::success();
6806 };
6807
6808 auto BodyGenCB = [&S, C, this](InsertPointTy AllocIP,
6809 InsertPointTy CodeGenIP,
6810 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
6811 Builder.restoreIP(IP: CodeGenIP);
6812
6813 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6814 if (C) {
6815 llvm::BasicBlock *CodeGenBB = CodeGenIP.getNodeParent();
6816 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6817 Builder, /*CreateBranch=*/false, Suffix: ".ordered.after");
6818 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
6819 GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
6820 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, S: CS, D: S);
6821 assert(S.getBeginLoc().isValid() &&
6822 "Outlined function call location must be valid.");
6823 ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: S.getBeginLoc());
6824 OMPBuilderCBHelpers::EmitCaptureStmt(CGF&: *this, CodeGenIPBB: CodeGenBB, FiniBB&: *FiniBB,
6825 Fn: OutlinedFn, Args: CapturedVars);
6826 } else {
6827 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
6828 CGF&: *this, RegionBodyStmt: CS->getCapturedStmt(), AllocaIP: AllocIP, CodeGenIP, RegionName: "ordered");
6829 }
6830 return llvm::Error::success();
6831 };
6832
6833 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6834 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6835 ValOrErr: OMPBuilder.createOrderedThreadsSimd(Loc: Builder, BodyGenCB, FiniCB, IsThreads: !C));
6836 Builder.restoreIP(IP: AfterIP);
6837 return;
6838 }
6839
6840 const auto *C = S.getSingleClause<OMPSIMDClause>();
6841 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6842 PrePostActionTy &Action) {
6843 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6844 if (C) {
6845 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
6846 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
6847 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, S: CS, D: S);
6848 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc: S.getBeginLoc(),
6849 OutlinedFn, Args: CapturedVars);
6850 } else {
6851 Action.Enter(CGF);
6852 CGF.EmitStmt(S: CS->getCapturedStmt());
6853 }
6854 };
6855 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6856 CGM.getOpenMPRuntime().emitOrderedRegion(CGF&: *this, OrderedOpGen: CodeGen, Loc: S.getBeginLoc(), IsThreads: !C);
6857}
6858
6859static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
6860 QualType SrcType, QualType DestType,
6861 SourceLocation Loc) {
6862 assert(CGF.hasScalarEvaluationKind(DestType) &&
6863 "DestType must have scalar evaluation kind.");
6864 assert(!Val.isAggregate() && "Must be a scalar or complex.");
6865 return Val.isScalar() ? CGF.EmitScalarConversion(Src: Val.getScalarVal(), SrcTy: SrcType,
6866 DstTy: DestType, Loc)
6867 : CGF.EmitComplexToScalarConversion(
6868 Src: Val.getComplexVal(), SrcTy: SrcType, DstTy: DestType, Loc);
6869}
6870
6871static CodeGenFunction::ComplexPairTy
6872convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
6873 QualType DestType, SourceLocation Loc) {
6874 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
6875 "DestType must have complex evaluation kind.");
6876 CodeGenFunction::ComplexPairTy ComplexVal;
6877 if (Val.isScalar()) {
6878 // Convert the input element to the element type of the complex.
6879 QualType DestElementType =
6880 DestType->castAs<ComplexType>()->getElementType();
6881 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
6882 Src: Val.getScalarVal(), SrcTy: SrcType, DstTy: DestElementType, Loc);
6883 ComplexVal = CodeGenFunction::ComplexPairTy(
6884 ScalarVal, llvm::Constant::getNullValue(Ty: ScalarVal->getType()));
6885 } else {
6886 assert(Val.isComplex() && "Must be a scalar or complex.");
6887 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
6888 QualType DestElementType =
6889 DestType->castAs<ComplexType>()->getElementType();
6890 ComplexVal.first = CGF.EmitScalarConversion(
6891 Src: Val.getComplexVal().first, SrcTy: SrcElementType, DstTy: DestElementType, Loc);
6892 ComplexVal.second = CGF.EmitScalarConversion(
6893 Src: Val.getComplexVal().second, SrcTy: SrcElementType, DstTy: DestElementType, Loc);
6894 }
6895 return ComplexVal;
6896}
6897
6898static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6899 LValue LVal, RValue RVal) {
6900 if (LVal.isGlobalReg())
6901 CGF.EmitStoreThroughGlobalRegLValue(Src: RVal, Dst: LVal);
6902 else
6903 CGF.EmitAtomicStore(rvalue: RVal, lvalue: LVal, AO, IsVolatile: LVal.isVolatile(), /*isInit=*/false);
6904}
6905
6906static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
6907 llvm::AtomicOrdering AO, LValue LVal,
6908 SourceLocation Loc) {
6909 if (LVal.isGlobalReg())
6910 return CGF.EmitLoadOfLValue(V: LVal, Loc);
6911 return CGF.EmitAtomicLoad(
6912 lvalue: LVal, loc: Loc, AO: llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO),
6913 IsVolatile: LVal.isVolatile());
6914}
6915
6916void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
6917 QualType RValTy, SourceLocation Loc) {
6918 switch (getEvaluationKind(T: LVal.getType())) {
6919 case TEK_Scalar:
6920 EmitStoreThroughLValue(Src: RValue::get(V: convertToScalarValue(
6921 CGF&: *this, Val: RVal, SrcType: RValTy, DestType: LVal.getType(), Loc)),
6922 Dst: LVal);
6923 break;
6924 case TEK_Complex:
6925 EmitStoreOfComplex(
6926 V: convertToComplexValue(CGF&: *this, Val: RVal, SrcType: RValTy, DestType: LVal.getType(), Loc), dest: LVal,
6927 /*isInit=*/false);
6928 break;
6929 case TEK_Aggregate:
6930 llvm_unreachable("Must be a scalar or complex.");
6931 }
6932}
6933
6934static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6935 const Expr *X, const Expr *V,
6936 SourceLocation Loc) {
6937 // v = x;
6938 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6939 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6940 LValue XLValue = CGF.EmitLValue(E: X);
6941 LValue VLValue = CGF.EmitLValue(E: V);
6942 RValue Res = emitSimpleAtomicLoad(CGF, AO, LVal: XLValue, Loc);
6943 // OpenMP, 2.17.7, atomic Construct
6944 // If the read or capture clause is specified and the acquire, acq_rel, or
6945 // seq_cst clause is specified then the strong flush on exit from the atomic
6946 // operation is also an acquire flush.
6947 switch (AO) {
6948 case llvm::AtomicOrdering::Acquire:
6949 case llvm::AtomicOrdering::AcquireRelease:
6950 case llvm::AtomicOrdering::SequentiallyConsistent:
6951 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
6952 AO: llvm::AtomicOrdering::Acquire);
6953 break;
6954 case llvm::AtomicOrdering::Monotonic:
6955 case llvm::AtomicOrdering::Release:
6956 break;
6957 case llvm::AtomicOrdering::NotAtomic:
6958 case llvm::AtomicOrdering::Unordered:
6959 llvm_unreachable("Unexpected ordering.");
6960 }
6961 CGF.emitOMPSimpleStore(LVal: VLValue, RVal: Res, RValTy: X->getType().getNonReferenceType(), Loc);
6962 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: V);
6963}
6964
6965static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,
6966 llvm::AtomicOrdering AO, const Expr *X,
6967 const Expr *E, SourceLocation Loc) {
6968 // x = expr;
6969 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
6970 emitSimpleAtomicStore(CGF, AO, LVal: CGF.EmitLValue(E: X), RVal: CGF.EmitAnyExpr(E));
6971 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
6972 // OpenMP, 2.17.7, atomic Construct
6973 // If the write, update, or capture clause is specified and the release,
6974 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6975 // the atomic operation is also a release flush.
6976 switch (AO) {
6977 case llvm::AtomicOrdering::Release:
6978 case llvm::AtomicOrdering::AcquireRelease:
6979 case llvm::AtomicOrdering::SequentiallyConsistent:
6980 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
6981 AO: llvm::AtomicOrdering::Release);
6982 break;
6983 case llvm::AtomicOrdering::Acquire:
6984 case llvm::AtomicOrdering::Monotonic:
6985 break;
6986 case llvm::AtomicOrdering::NotAtomic:
6987 case llvm::AtomicOrdering::Unordered:
6988 llvm_unreachable("Unexpected ordering.");
6989 }
6990}
6991
6992static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
6993 RValue Update,
6994 BinaryOperatorKind BO,
6995 llvm::AtomicOrdering AO,
6996 bool IsXLHSInRHSPart) {
6997 ASTContext &Context = CGF.getContext();
6998 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
6999 // expression is simple and atomic is allowed for the given type for the
7000 // target platform.
7001 if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||
7002 (!isa<llvm::ConstantInt>(Val: Update.getScalarVal()) &&
7003 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||
7004 !Context.getTargetInfo().hasBuiltinAtomic(
7005 AtomicSizeInBits: Context.getTypeSize(T: X.getType()), AlignmentInBits: Context.toBits(CharSize: X.getAlignment())))
7006 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7007
7008 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
7009 if (T->isIntegerTy())
7010 return true;
7011
7012 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
7013 return llvm::isPowerOf2_64(Value: CGF.CGM.getDataLayout().getTypeStoreSize(Ty: T));
7014
7015 return false;
7016 };
7017
7018 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
7019 !CheckAtomicSupport(X.getAddress().getElementType(), BO))
7020 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7021
7022 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();
7023 llvm::AtomicRMWInst::BinOp RMWOp;
7024 switch (BO) {
7025 case BO_Add:
7026 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
7027 break;
7028 case BO_Sub:
7029 if (!IsXLHSInRHSPart)
7030 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7031 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
7032 break;
7033 case BO_And:
7034 RMWOp = llvm::AtomicRMWInst::And;
7035 break;
7036 case BO_Or:
7037 RMWOp = llvm::AtomicRMWInst::Or;
7038 break;
7039 case BO_Xor:
7040 RMWOp = llvm::AtomicRMWInst::Xor;
7041 break;
7042 case BO_LT:
7043 if (IsInteger)
7044 RMWOp = X.getType()->hasSignedIntegerRepresentation()
7045 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
7046 : llvm::AtomicRMWInst::Max)
7047 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
7048 : llvm::AtomicRMWInst::UMax);
7049 else
7050 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
7051 : llvm::AtomicRMWInst::FMax;
7052 break;
7053 case BO_GT:
7054 if (IsInteger)
7055 RMWOp = X.getType()->hasSignedIntegerRepresentation()
7056 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
7057 : llvm::AtomicRMWInst::Min)
7058 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
7059 : llvm::AtomicRMWInst::UMin);
7060 else
7061 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
7062 : llvm::AtomicRMWInst::FMin;
7063 break;
7064 case BO_Assign:
7065 RMWOp = llvm::AtomicRMWInst::Xchg;
7066 break;
7067 case BO_Mul:
7068 case BO_Div:
7069 case BO_Rem:
7070 case BO_Shl:
7071 case BO_Shr:
7072 case BO_LAnd:
7073 case BO_LOr:
7074 return std::make_pair(x: false, y: RValue::get(V: nullptr));
7075 case BO_PtrMemD:
7076 case BO_PtrMemI:
7077 case BO_LE:
7078 case BO_GE:
7079 case BO_EQ:
7080 case BO_NE:
7081 case BO_Cmp:
7082 case BO_AddAssign:
7083 case BO_SubAssign:
7084 case BO_AndAssign:
7085 case BO_OrAssign:
7086 case BO_XorAssign:
7087 case BO_MulAssign:
7088 case BO_DivAssign:
7089 case BO_RemAssign:
7090 case BO_ShlAssign:
7091 case BO_ShrAssign:
7092 case BO_Comma:
7093 llvm_unreachable("Unsupported atomic update operation");
7094 }
7095 llvm::Value *UpdateVal = Update.getScalarVal();
7096 if (auto *IC = dyn_cast<llvm::ConstantInt>(Val: UpdateVal)) {
7097 if (IsInteger)
7098 UpdateVal = CGF.Builder.CreateIntCast(
7099 V: IC, DestTy: X.getAddress().getElementType(),
7100 isSigned: X.getType()->hasSignedIntegerRepresentation());
7101 else
7102 UpdateVal = CGF.Builder.CreateCast(Op: llvm::Instruction::CastOps::UIToFP, V: IC,
7103 DestTy: X.getAddress().getElementType());
7104 }
7105 llvm::AtomicRMWInst *Res =
7106 CGF.emitAtomicRMWInst(Op: RMWOp, Addr: X.getAddress(), Val: UpdateVal, Order: AO);
7107 return std::make_pair(x: true, y: RValue::get(V: Res));
7108}
7109
7110std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
7111 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
7112 llvm::AtomicOrdering AO, SourceLocation Loc,
7113 const llvm::function_ref<RValue(RValue)> CommonGen) {
7114 // Update expressions are allowed to have the following forms:
7115 // x binop= expr; -> xrval + expr;
7116 // x++, ++x -> xrval + 1;
7117 // x--, --x -> xrval - 1;
7118 // x = x binop expr; -> xrval binop expr
7119 // x = expr Op x; - > expr binop xrval;
7120 auto Res = emitOMPAtomicRMW(CGF&: *this, X, Update: E, BO, AO, IsXLHSInRHSPart);
7121 if (!Res.first) {
7122 if (X.isGlobalReg()) {
7123 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
7124 // 'xrval'.
7125 EmitStoreThroughLValue(Src: CommonGen(EmitLoadOfLValue(V: X, Loc)), Dst: X);
7126 } else {
7127 // Perform compare-and-swap procedure.
7128 EmitAtomicUpdate(LVal: X, AO, UpdateOp: CommonGen, IsVolatile: X.getType().isVolatileQualified());
7129 }
7130 }
7131 return Res;
7132}
7133
7134static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,
7135 llvm::AtomicOrdering AO, const Expr *X,
7136 const Expr *E, const Expr *UE,
7137 bool IsXLHSInRHSPart, SourceLocation Loc) {
7138 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
7139 "Update expr in 'atomic update' must be a binary operator.");
7140 const auto *BOUE = cast<BinaryOperator>(Val: UE->IgnoreImpCasts());
7141 // Update expressions are allowed to have the following forms:
7142 // x binop= expr; -> xrval + expr;
7143 // x++, ++x -> xrval + 1;
7144 // x--, --x -> xrval - 1;
7145 // x = x binop expr; -> xrval binop expr
7146 // x = expr Op x; - > expr binop xrval;
7147 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
7148 LValue XLValue = CGF.EmitLValue(E: X);
7149 RValue ExprRValue = CGF.EmitAnyExpr(E);
7150 const auto *LHS = cast<OpaqueValueExpr>(Val: BOUE->getLHS()->IgnoreImpCasts());
7151 const auto *RHS = cast<OpaqueValueExpr>(Val: BOUE->getRHS()->IgnoreImpCasts());
7152 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
7153 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
7154 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
7155 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7156 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
7157 return CGF.EmitAnyExpr(E: UE);
7158 };
7159 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
7160 X: XLValue, E: ExprRValue, BO: BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, CommonGen: Gen);
7161 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7162 // OpenMP, 2.17.7, atomic Construct
7163 // If the write, update, or capture clause is specified and the release,
7164 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7165 // the atomic operation is also a release flush.
7166 switch (AO) {
7167 case llvm::AtomicOrdering::Release:
7168 case llvm::AtomicOrdering::AcquireRelease:
7169 case llvm::AtomicOrdering::SequentiallyConsistent:
7170 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7171 AO: llvm::AtomicOrdering::Release);
7172 break;
7173 case llvm::AtomicOrdering::Acquire:
7174 case llvm::AtomicOrdering::Monotonic:
7175 break;
7176 case llvm::AtomicOrdering::NotAtomic:
7177 case llvm::AtomicOrdering::Unordered:
7178 llvm_unreachable("Unexpected ordering.");
7179 }
7180}
7181
7182static RValue convertToType(CodeGenFunction &CGF, RValue Value,
7183 QualType SourceType, QualType ResType,
7184 SourceLocation Loc) {
7185 switch (CGF.getEvaluationKind(T: ResType)) {
7186 case TEK_Scalar:
7187 return RValue::get(
7188 V: convertToScalarValue(CGF, Val: Value, SrcType: SourceType, DestType: ResType, Loc));
7189 case TEK_Complex: {
7190 auto Res = convertToComplexValue(CGF, Val: Value, SrcType: SourceType, DestType: ResType, Loc);
7191 return RValue::getComplex(V1: Res.first, V2: Res.second);
7192 }
7193 case TEK_Aggregate:
7194 break;
7195 }
7196 llvm_unreachable("Must be a scalar or complex.");
7197}
7198
7199static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
7200 llvm::AtomicOrdering AO,
7201 bool IsPostfixUpdate, const Expr *V,
7202 const Expr *X, const Expr *E,
7203 const Expr *UE, bool IsXLHSInRHSPart,
7204 SourceLocation Loc) {
7205 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
7206 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
7207 RValue NewVVal;
7208 LValue VLValue = CGF.EmitLValue(E: V);
7209 LValue XLValue = CGF.EmitLValue(E: X);
7210 RValue ExprRValue = CGF.EmitAnyExpr(E);
7211 QualType NewVValType;
7212 if (UE) {
7213 // 'x' is updated with some additional value.
7214 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
7215 "Update expr in 'atomic capture' must be a binary operator.");
7216 const auto *BOUE = cast<BinaryOperator>(Val: UE->IgnoreImpCasts());
7217 // Update expressions are allowed to have the following forms:
7218 // x binop= expr; -> xrval + expr;
7219 // x++, ++x -> xrval + 1;
7220 // x--, --x -> xrval - 1;
7221 // x = x binop expr; -> xrval binop expr
7222 // x = expr Op x; - > expr binop xrval;
7223 const auto *LHS = cast<OpaqueValueExpr>(Val: BOUE->getLHS()->IgnoreImpCasts());
7224 const auto *RHS = cast<OpaqueValueExpr>(Val: BOUE->getRHS()->IgnoreImpCasts());
7225 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
7226 NewVValType = XRValExpr->getType();
7227 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
7228 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
7229 IsPostfixUpdate](RValue XRValue) {
7230 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7231 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
7232 RValue Res = CGF.EmitAnyExpr(E: UE);
7233 NewVVal = IsPostfixUpdate ? XRValue : Res;
7234 return Res;
7235 };
7236 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
7237 X: XLValue, E: ExprRValue, BO: BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, CommonGen: Gen);
7238 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7239 if (Res.first) {
7240 // 'atomicrmw' instruction was generated.
7241 if (IsPostfixUpdate) {
7242 // Use old value from 'atomicrmw'.
7243 NewVVal = Res.second;
7244 } else {
7245 // 'atomicrmw' does not provide new value, so evaluate it using old
7246 // value of 'x'.
7247 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
7248 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
7249 NewVVal = CGF.EmitAnyExpr(E: UE);
7250 }
7251 }
7252 } else {
7253 // 'x' is simply rewritten with some 'expr'.
7254 NewVValType = X->getType().getNonReferenceType();
7255 ExprRValue = convertToType(CGF, Value: ExprRValue, SourceType: E->getType(),
7256 ResType: X->getType().getNonReferenceType(), Loc);
7257 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
7258 NewVVal = XRValue;
7259 return ExprRValue;
7260 };
7261 // Try to perform atomicrmw xchg, otherwise simple exchange.
7262 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
7263 X: XLValue, E: ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
7264 Loc, CommonGen: Gen);
7265 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7266 if (Res.first) {
7267 // 'atomicrmw' instruction was generated.
7268 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
7269 }
7270 }
7271 // Emit post-update store to 'v' of old/new 'x' value.
7272 CGF.emitOMPSimpleStore(LVal: VLValue, RVal: NewVVal, RValTy: NewVValType, Loc);
7273 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: V);
7274 // OpenMP 5.1 removes the required flush for capture clause.
7275 if (CGF.CGM.getLangOpts().OpenMP < 51) {
7276 // OpenMP, 2.17.7, atomic Construct
7277 // If the write, update, or capture clause is specified and the release,
7278 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7279 // the atomic operation is also a release flush.
7280 // If the read or capture clause is specified and the acquire, acq_rel, or
7281 // seq_cst clause is specified then the strong flush on exit from the atomic
7282 // operation is also an acquire flush.
7283 switch (AO) {
7284 case llvm::AtomicOrdering::Release:
7285 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7286 AO: llvm::AtomicOrdering::Release);
7287 break;
7288 case llvm::AtomicOrdering::Acquire:
7289 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7290 AO: llvm::AtomicOrdering::Acquire);
7291 break;
7292 case llvm::AtomicOrdering::AcquireRelease:
7293 case llvm::AtomicOrdering::SequentiallyConsistent:
7294 CGF.CGM.getOpenMPRuntime().emitFlush(
7295 CGF, Vars: {}, Loc, AO: llvm::AtomicOrdering::AcquireRelease);
7296 break;
7297 case llvm::AtomicOrdering::Monotonic:
7298 break;
7299 case llvm::AtomicOrdering::NotAtomic:
7300 case llvm::AtomicOrdering::Unordered:
7301 llvm_unreachable("Unexpected ordering.");
7302 }
7303 }
7304}
7305
7306static void emitOMPAtomicCompareExpr(
7307 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
7308 const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,
7309 const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,
7310 SourceLocation Loc) {
7311 llvm::OpenMPIRBuilder &OMPBuilder =
7312 CGF.CGM.getOpenMPRuntime().getOMPBuilder();
7313
7314 OMPAtomicCompareOp Op;
7315 assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");
7316 switch (cast<BinaryOperator>(Val: CE)->getOpcode()) {
7317 case BO_EQ:
7318 Op = OMPAtomicCompareOp::EQ;
7319 break;
7320 case BO_LT:
7321 Op = OMPAtomicCompareOp::MIN;
7322 break;
7323 case BO_GT:
7324 Op = OMPAtomicCompareOp::MAX;
7325 break;
7326 default:
7327 llvm_unreachable("unsupported atomic compare binary operator");
7328 }
7329
7330 LValue XLVal = CGF.EmitLValue(E: X);
7331 Address XAddr = XLVal.getAddress();
7332
7333 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
7334 if (X->getType() == E->getType())
7335 return CGF.EmitScalarExpr(E);
7336 const Expr *NewE = E->IgnoreImplicitAsWritten();
7337 llvm::Value *V = CGF.EmitScalarExpr(E: NewE);
7338 if (NewE->getType() == X->getType())
7339 return V;
7340 return CGF.EmitScalarConversion(Src: V, SrcTy: NewE->getType(), DstTy: X->getType(), Loc);
7341 };
7342
7343 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
7344 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
7345 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: EVal))
7346 EVal = CGF.Builder.CreateIntCast(
7347 V: CI, DestTy: XLVal.getAddress().getElementType(),
7348 isSigned: E->getType()->hasSignedIntegerRepresentation());
7349 if (DVal)
7350 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: DVal))
7351 DVal = CGF.Builder.CreateIntCast(
7352 V: CI, DestTy: XLVal.getAddress().getElementType(),
7353 isSigned: D->getType()->hasSignedIntegerRepresentation());
7354
7355 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
7356 .Var: XAddr.emitRawPointer(CGF), .ElemTy: XAddr.getElementType(),
7357 .IsSigned: X->getType()->hasSignedIntegerRepresentation(),
7358 .IsVolatile: X->getType().isVolatileQualified()};
7359 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
7360 if (V) {
7361 LValue LV = CGF.EmitLValue(E: V);
7362 Address Addr = LV.getAddress();
7363 VOpVal = {.Var: Addr.emitRawPointer(CGF), .ElemTy: Addr.getElementType(),
7364 .IsSigned: V->getType()->hasSignedIntegerRepresentation(),
7365 .IsVolatile: V->getType().isVolatileQualified()};
7366 }
7367 if (R) {
7368 LValue LV = CGF.EmitLValue(E: R);
7369 Address Addr = LV.getAddress();
7370 ROpVal = {.Var: Addr.emitRawPointer(CGF), .ElemTy: Addr.getElementType(),
7371 .IsSigned: R->getType()->hasSignedIntegerRepresentation(),
7372 .IsVolatile: R->getType().isVolatileQualified()};
7373 }
7374
7375 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
7376 // fail clause was not mentioned on the
7377 // "#pragma omp atomic compare" construct.
7378 CGF.Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
7379 Loc: CGF.Builder, X&: XOpVal, V&: VOpVal, R&: ROpVal, E: EVal, D: DVal, AO, Op, IsXBinopExpr,
7380 IsPostfixUpdate, IsFailOnly));
7381 } else
7382 CGF.Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
7383 Loc: CGF.Builder, X&: XOpVal, V&: VOpVal, R&: ROpVal, E: EVal, D: DVal, AO, Op, IsXBinopExpr,
7384 IsPostfixUpdate, IsFailOnly, Failure: FailAO));
7385}
7386
7387static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
7388 llvm::AtomicOrdering AO,
7389 llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,
7390 const Expr *X, const Expr *V, const Expr *R,
7391 const Expr *E, const Expr *UE, const Expr *D,
7392 const Expr *CE, bool IsXLHSInRHSPart,
7393 bool IsFailOnly, SourceLocation Loc) {
7394 switch (Kind) {
7395 case OMPC_read:
7396 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
7397 break;
7398 case OMPC_write:
7399 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
7400 break;
7401 case OMPC_unknown:
7402 case OMPC_update:
7403 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
7404 break;
7405 case OMPC_capture:
7406 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
7407 IsXLHSInRHSPart, Loc);
7408 break;
7409 case OMPC_compare: {
7410 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
7411 IsXBinopExpr: IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
7412 break;
7413 }
7414 default:
7415 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
7416 }
7417}
7418
7419void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
7420 llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7421 // Fail Memory Clause Ordering.
7422 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
7423 bool MemOrderingSpecified = false;
7424 if (S.getSingleClause<OMPSeqCstClause>()) {
7425 AO = llvm::AtomicOrdering::SequentiallyConsistent;
7426 MemOrderingSpecified = true;
7427 } else if (S.getSingleClause<OMPAcqRelClause>()) {
7428 AO = llvm::AtomicOrdering::AcquireRelease;
7429 MemOrderingSpecified = true;
7430 } else if (S.getSingleClause<OMPAcquireClause>()) {
7431 AO = llvm::AtomicOrdering::Acquire;
7432 MemOrderingSpecified = true;
7433 } else if (S.getSingleClause<OMPReleaseClause>()) {
7434 AO = llvm::AtomicOrdering::Release;
7435 MemOrderingSpecified = true;
7436 } else if (S.getSingleClause<OMPRelaxedClause>()) {
7437 AO = llvm::AtomicOrdering::Monotonic;
7438 MemOrderingSpecified = true;
7439 }
7440 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
7441 OpenMPClauseKind Kind = OMPC_unknown;
7442 for (const OMPClause *C : S.clauses()) {
7443 // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
7444 // if it is first).
7445 OpenMPClauseKind K = C->getClauseKind();
7446 // TBD
7447 if (K == OMPC_weak)
7448 return;
7449 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
7450 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
7451 continue;
7452 Kind = K;
7453 KindsEncountered.insert(V: K);
7454 }
7455 // We just need to correct Kind here. No need to set a bool saying it is
7456 // actually compare capture because we can tell from whether V and R are
7457 // nullptr.
7458 if (KindsEncountered.contains(V: OMPC_compare) &&
7459 KindsEncountered.contains(V: OMPC_capture))
7460 Kind = OMPC_compare;
7461 if (!MemOrderingSpecified) {
7462 llvm::AtomicOrdering DefaultOrder =
7463 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7464 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
7465 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
7466 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
7467 Kind == OMPC_capture)) {
7468 AO = DefaultOrder;
7469 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
7470 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
7471 AO = llvm::AtomicOrdering::Release;
7472 } else if (Kind == OMPC_read) {
7473 assert(Kind == OMPC_read && "Unexpected atomic kind.");
7474 AO = llvm::AtomicOrdering::Acquire;
7475 }
7476 }
7477 }
7478
7479 if (KindsEncountered.contains(V: OMPC_compare) &&
7480 KindsEncountered.contains(V: OMPC_fail)) {
7481 Kind = OMPC_compare;
7482 const auto *FailClause = S.getSingleClause<OMPFailClause>();
7483 if (FailClause) {
7484 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
7485 if (FailParameter == llvm::omp::OMPC_relaxed)
7486 FailAO = llvm::AtomicOrdering::Monotonic;
7487 else if (FailParameter == llvm::omp::OMPC_acquire)
7488 FailAO = llvm::AtomicOrdering::Acquire;
7489 else if (FailParameter == llvm::omp::OMPC_seq_cst)
7490 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
7491 }
7492 }
7493
7494 LexicalScope Scope(*this, S.getSourceRange());
7495 EmitStopPoint(S: S.getAssociatedStmt());
7496 emitOMPAtomicExpr(CGF&: *this, Kind, AO, FailAO, IsPostfixUpdate: S.isPostfixUpdate(), X: S.getX(),
7497 V: S.getV(), R: S.getR(), E: S.getExpr(), UE: S.getUpdateExpr(),
7498 D: S.getD(), CE: S.getCondExpr(), IsXLHSInRHSPart: S.isXLHSInRHSPart(),
7499 IsFailOnly: S.isFailOnly(), Loc: S.getBeginLoc());
7500}
7501
7502static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
7503 const OMPExecutableDirective &S,
7504 const RegionCodeGenTy &CodeGen) {
7505 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
7506 CodeGenModule &CGM = CGF.CGM;
7507
7508 // On device emit this construct as inlined code.
7509 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
7510 OMPLexicalScope Scope(CGF, S, OMPD_target);
7511 CGM.getOpenMPRuntime().emitInlinedDirective(
7512 CGF, InnermostKind: OMPD_target, CodeGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7513 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
7514 });
7515 return;
7516 }
7517
7518 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
7519 llvm::Function *Fn = nullptr;
7520 llvm::Constant *FnID = nullptr;
7521
7522 const Expr *IfCond = nullptr;
7523 // Check for the at most one if clause associated with the target region.
7524 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7525 if (C->getNameModifier() == OMPD_unknown ||
7526 C->getNameModifier() == OMPD_target) {
7527 IfCond = C->getCondition();
7528 break;
7529 }
7530 }
7531
7532 // Check if we have any device clause associated with the directive.
7533 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
7534 nullptr, OMPC_DEVICE_unknown);
7535 if (auto *C = S.getSingleClause<OMPDeviceClause>())
7536 Device.setPointerAndInt(PtrVal: C->getDevice(), IntVal: C->getModifier());
7537
7538 // Check if we have an if clause whose conditional always evaluates to false
7539 // or if we do not have any targets specified. If so the target region is not
7540 // an offload entry point.
7541 bool IsOffloadEntry = true;
7542 if (IfCond) {
7543 bool Val;
7544 if (CGF.ConstantFoldsToSimpleInteger(Cond: IfCond, Result&: Val) && !Val)
7545 IsOffloadEntry = false;
7546 }
7547 if (CGM.getLangOpts().OMPTargetTriples.empty())
7548 IsOffloadEntry = false;
7549
7550 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
7551 CGM.getDiags().Report(DiagID: diag::err_missing_mandatory_offloading);
7552 }
7553
7554 StringRef ParentName;
7555 // In case we have Ctors/Dtors we use the complete type variant to produce
7556 // the mangling of the device outlined kernel. Lambdas and blocks at
7557 // namespace scope have no parent function.
7558 if (!CGF.CurFuncDecl)
7559 ParentName = CGF.CurFn->getName();
7560 else if (const auto *D = dyn_cast<CXXConstructorDecl>(Val: CGF.CurFuncDecl))
7561 ParentName = CGM.getMangledName(GD: GlobalDecl(D, Ctor_Complete));
7562 else if (const auto *D = dyn_cast<CXXDestructorDecl>(Val: CGF.CurFuncDecl))
7563 ParentName = CGM.getMangledName(GD: GlobalDecl(D, Dtor_Complete));
7564 else
7565 ParentName =
7566 CGM.getMangledName(GD: GlobalDecl(cast<FunctionDecl>(Val: CGF.CurFuncDecl)));
7567
7568 // Emit target region as a standalone region.
7569 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: FnID,
7570 IsOffloadEntry, CodeGen);
7571 OMPLexicalScope Scope(CGF, S, OMPD_task);
7572 auto &&SizeEmitter =
7573 [IsOffloadEntry](CodeGenFunction &CGF,
7574 const OMPLoopDirective &D) -> llvm::Value * {
7575 if (IsOffloadEntry) {
7576 OMPLoopScope PreInitScope(CGF, D);
7577 // Emit calculation of the iterations count.
7578 llvm::Value *NumIterations = CGF.EmitScalarExpr(E: D.getNumIterations());
7579 NumIterations = CGF.Builder.CreateIntCast(V: NumIterations, DestTy: CGF.Int64Ty,
7580 /*isSigned=*/false);
7581 return NumIterations;
7582 }
7583 return nullptr;
7584 };
7585 CGM.getOpenMPRuntime().emitTargetCall(CGF, D: S, OutlinedFn: Fn, OutlinedFnID: FnID, IfCond, Device,
7586 SizeEmitter);
7587}
7588
7589static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
7590 PrePostActionTy &Action) {
7591 Action.Enter(CGF);
7592 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7593 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7594 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7595 (void)PrivateScope.Privatize();
7596 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
7597 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
7598
7599 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_target)->getCapturedStmt());
7600 CGF.EnsureInsertPoint();
7601}
7602
7603void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
7604 StringRef ParentName,
7605 const OMPTargetDirective &S) {
7606 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7607 emitTargetRegion(CGF, S, Action);
7608 };
7609 llvm::Function *Fn;
7610 llvm::Constant *Addr;
7611 // Emit target region as a standalone region.
7612 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7613 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7614 assert(Fn && Addr && "Target device function emission failed.");
7615}
7616
7617void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
7618 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7619 emitTargetRegion(CGF, S, Action);
7620 };
7621 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7622}
7623
7624static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
7625 const OMPExecutableDirective &S,
7626 OpenMPDirectiveKind InnermostKind,
7627 const RegionCodeGenTy &CodeGen) {
7628 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_teams);
7629 llvm::Function *OutlinedFn =
7630 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
7631 CGF, D: S, ThreadIDVar: *CS->getCapturedDecl()->param_begin(), InnermostKind,
7632 CodeGen);
7633
7634 OMPTeamsScope Scope(CGF, S);
7635 auto ParallelLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7636 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
7637 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7638 if (NT || TL) {
7639 const Expr *NumTeams = NT ? NT->getNumTeams().front() : nullptr;
7640 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7641
7642 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
7643 Loc: S.getBeginLoc());
7644 }
7645 };
7646
7647 const Expr *IfCond = nullptr;
7648 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7649 if (C->getNameModifier() == OMPD_unknown ||
7650 C->getNameModifier() == OMPD_teams) {
7651 IfCond = C->getCondition();
7652 break;
7653 }
7654 }
7655 if (IfCond && CGF.CGM.getLangOpts().OpenMP >= 52) {
7656 auto SerialLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7657 // OpenMP 5.2, 10.2, teams Construct
7658 // When an if clause is present on a teams construct and the if clause
7659 // expression evaluates to false, the number of created teams is one.
7660 const llvm::APInt One(32, 1);
7661 IntegerLiteral NumTeams(
7662 CGF.getContext(), One,
7663 CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
7664 SourceLocation());
7665 // The thread_limit clause is unaffected by the if clause.
7666 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7667 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7668 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams: &NumTeams, ThreadLimit,
7669 Loc: S.getBeginLoc());
7670 };
7671 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, Cond: IfCond, ThenGen: ParallelLeague,
7672 ElseGen: SerialLeague);
7673 } else {
7674 const RegionCodeGenTy ThenRCG(ParallelLeague);
7675 ThenRCG(CGF);
7676 }
7677
7678 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
7679 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
7680 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, D: S, Loc: S.getBeginLoc(), OutlinedFn,
7681 CapturedVars);
7682}
7683
7684void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
7685 // Emit teams region as a standalone region.
7686 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7687 Action.Enter(CGF);
7688 OMPPrivateScope PrivateScope(CGF);
7689 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7690 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7691 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7692 (void)PrivateScope.Privatize();
7693 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_teams)->getCapturedStmt());
7694 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7695 };
7696 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
7697 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7698 CondGen: [](CodeGenFunction &) { return nullptr; });
7699}
7700
7701static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
7702 const OMPTargetTeamsDirective &S) {
7703 auto *CS = S.getCapturedStmt(RegionKind: OMPD_teams);
7704 Action.Enter(CGF);
7705 // Emit teams region as a standalone region.
7706 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7707 Action.Enter(CGF);
7708 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7709 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7710 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7711 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7712 (void)PrivateScope.Privatize();
7713 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
7714 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
7715 CGF.EmitStmt(S: CS->getCapturedStmt());
7716 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7717 };
7718 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_teams, CodeGen);
7719 emitPostUpdateForReductionClause(CGF, D: S,
7720 CondGen: [](CodeGenFunction &) { return nullptr; });
7721}
7722
7723void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
7724 CodeGenModule &CGM, StringRef ParentName,
7725 const OMPTargetTeamsDirective &S) {
7726 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7727 emitTargetTeamsRegion(CGF, Action, S);
7728 };
7729 llvm::Function *Fn;
7730 llvm::Constant *Addr;
7731 // Emit target region as a standalone region.
7732 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7733 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7734 assert(Fn && Addr && "Target device function emission failed.");
7735}
7736
7737void CodeGenFunction::EmitOMPTargetTeamsDirective(
7738 const OMPTargetTeamsDirective &S) {
7739 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7740 emitTargetTeamsRegion(CGF, Action, S);
7741 };
7742 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7743}
7744
7745static void
7746emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
7747 const OMPTargetTeamsDistributeDirective &S) {
7748 Action.Enter(CGF);
7749 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7750 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7751 };
7752
7753 // Emit teams region as a standalone region.
7754 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7755 PrePostActionTy &Action) {
7756 Action.Enter(CGF);
7757 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7758 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7759 (void)PrivateScope.Privatize();
7760 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7761 CodeGen: CodeGenDistribute);
7762 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7763 };
7764 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute, CodeGen);
7765 emitPostUpdateForReductionClause(CGF, D: S,
7766 CondGen: [](CodeGenFunction &) { return nullptr; });
7767}
7768
7769void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
7770 CodeGenModule &CGM, StringRef ParentName,
7771 const OMPTargetTeamsDistributeDirective &S) {
7772 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7773 emitTargetTeamsDistributeRegion(CGF, Action, S);
7774 };
7775 llvm::Function *Fn;
7776 llvm::Constant *Addr;
7777 // Emit target region as a standalone region.
7778 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7779 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7780 assert(Fn && Addr && "Target device function emission failed.");
7781}
7782
7783void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
7784 const OMPTargetTeamsDistributeDirective &S) {
7785 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7786 emitTargetTeamsDistributeRegion(CGF, Action, S);
7787 };
7788 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7789}
7790
7791static void emitTargetTeamsDistributeSimdRegion(
7792 CodeGenFunction &CGF, PrePostActionTy &Action,
7793 const OMPTargetTeamsDistributeSimdDirective &S) {
7794 Action.Enter(CGF);
7795 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7796 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7797 };
7798
7799 // Emit teams region as a standalone region.
7800 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7801 PrePostActionTy &Action) {
7802 Action.Enter(CGF);
7803 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7804 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7805 (void)PrivateScope.Privatize();
7806 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7807 CodeGen: CodeGenDistribute);
7808 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7809 };
7810 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_simd, CodeGen);
7811 emitPostUpdateForReductionClause(CGF, D: S,
7812 CondGen: [](CodeGenFunction &) { return nullptr; });
7813}
7814
7815void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
7816 CodeGenModule &CGM, StringRef ParentName,
7817 const OMPTargetTeamsDistributeSimdDirective &S) {
7818 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7819 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7820 };
7821 llvm::Function *Fn;
7822 llvm::Constant *Addr;
7823 // Emit target region as a standalone region.
7824 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7825 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7826 assert(Fn && Addr && "Target device function emission failed.");
7827}
7828
7829void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
7830 const OMPTargetTeamsDistributeSimdDirective &S) {
7831 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7832 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7833 };
7834 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7835}
7836
7837void CodeGenFunction::EmitOMPTeamsDistributeDirective(
7838 const OMPTeamsDistributeDirective &S) {
7839
7840 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7841 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7842 };
7843
7844 // Emit teams region as a standalone region.
7845 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7846 PrePostActionTy &Action) {
7847 Action.Enter(CGF);
7848 OMPPrivateScope PrivateScope(CGF);
7849 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7850 (void)PrivateScope.Privatize();
7851 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7852 CodeGen: CodeGenDistribute);
7853 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7854 };
7855 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
7856 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7857 CondGen: [](CodeGenFunction &) { return nullptr; });
7858}
7859
7860void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
7861 const OMPTeamsDistributeSimdDirective &S) {
7862 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7863 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7864 };
7865
7866 // Emit teams region as a standalone region.
7867 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7868 PrePostActionTy &Action) {
7869 Action.Enter(CGF);
7870 OMPPrivateScope PrivateScope(CGF);
7871 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7872 (void)PrivateScope.Privatize();
7873 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd,
7874 CodeGen: CodeGenDistribute);
7875 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7876 };
7877 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_simd, CodeGen);
7878 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7879 CondGen: [](CodeGenFunction &) { return nullptr; });
7880}
7881
7882void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
7883 const OMPTeamsDistributeParallelForDirective &S) {
7884 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7885 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7886 IncExpr: S.getDistInc());
7887 };
7888
7889 // Emit teams region as a standalone region.
7890 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7891 PrePostActionTy &Action) {
7892 Action.Enter(CGF);
7893 OMPPrivateScope PrivateScope(CGF);
7894 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7895 (void)PrivateScope.Privatize();
7896 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7897 CodeGen: CodeGenDistribute);
7898 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7899 };
7900 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_parallel_for, CodeGen);
7901 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7902 CondGen: [](CodeGenFunction &) { return nullptr; });
7903}
7904
7905void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
7906 const OMPTeamsDistributeParallelForSimdDirective &S) {
7907 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7908 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7909 IncExpr: S.getDistInc());
7910 };
7911
7912 // Emit teams region as a standalone region.
7913 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7914 PrePostActionTy &Action) {
7915 Action.Enter(CGF);
7916 OMPPrivateScope PrivateScope(CGF);
7917 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7918 (void)PrivateScope.Privatize();
7919 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7920 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
7921 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7922 };
7923 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_parallel_for_simd,
7924 CodeGen);
7925 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7926 CondGen: [](CodeGenFunction &) { return nullptr; });
7927}
7928
7929void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) {
7930 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7931 llvm::Value *Device = nullptr;
7932 llvm::Value *NumDependences = nullptr;
7933 llvm::Value *DependenceList = nullptr;
7934
7935 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7936 Device = EmitScalarExpr(E: C->getDevice());
7937
7938 // Build list and emit dependences
7939 OMPTaskDataTy Data;
7940 buildDependences(S, Data);
7941 if (!Data.Dependences.empty()) {
7942 Address DependenciesArray = Address::invalid();
7943 std::tie(args&: NumDependences, args&: DependenciesArray) =
7944 CGM.getOpenMPRuntime().emitDependClause(CGF&: *this, Dependencies: Data.Dependences,
7945 Loc: S.getBeginLoc());
7946 DependenceList = DependenciesArray.emitRawPointer(CGF&: *this);
7947 }
7948 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
7949
7950 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||
7951 S.getSingleClause<OMPDestroyClause>() ||
7952 S.getSingleClause<OMPUseClause>())) &&
7953 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7954
7955 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();
7956 if (!ItOMPInitClause.empty()) {
7957 // Look at the multiple init clauses
7958 for (const OMPInitClause *C : ItOMPInitClause) {
7959 llvm::Value *InteropvarPtr =
7960 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
7961 llvm::omp::OMPInteropType InteropType =
7962 llvm::omp::OMPInteropType::Unknown;
7963 if (C->getIsTarget()) {
7964 InteropType = llvm::omp::OMPInteropType::Target;
7965 } else {
7966 assert(C->getIsTargetSync() &&
7967 "Expected interop-type target/targetsync");
7968 InteropType = llvm::omp::OMPInteropType::TargetSync;
7969 }
7970 OMPBuilder.createOMPInteropInit(Loc: Builder, InteropVar: InteropvarPtr, InteropType,
7971 Device, NumDependences, DependenceAddress: DependenceList,
7972 HaveNowaitClause: Data.HasNowaitClause);
7973 }
7974 }
7975 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();
7976 if (!ItOMPDestroyClause.empty()) {
7977 // Look at the multiple destroy clauses
7978 for (const OMPDestroyClause *C : ItOMPDestroyClause) {
7979 llvm::Value *InteropvarPtr =
7980 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
7981 OMPBuilder.createOMPInteropDestroy(Loc: Builder, InteropVar: InteropvarPtr, Device,
7982 NumDependences, DependenceAddress: DependenceList,
7983 HaveNowaitClause: Data.HasNowaitClause);
7984 }
7985 }
7986 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();
7987 if (!ItOMPUseClause.empty()) {
7988 // Look at the multiple use clauses
7989 for (const OMPUseClause *C : ItOMPUseClause) {
7990 llvm::Value *InteropvarPtr =
7991 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
7992 OMPBuilder.createOMPInteropUse(Loc: Builder, InteropVar: InteropvarPtr, Device,
7993 NumDependences, DependenceAddress: DependenceList,
7994 HaveNowaitClause: Data.HasNowaitClause);
7995 }
7996 }
7997}
7998
7999static void emitTargetTeamsDistributeParallelForRegion(
8000 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
8001 PrePostActionTy &Action) {
8002 Action.Enter(CGF);
8003 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8004 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
8005 IncExpr: S.getDistInc());
8006 };
8007
8008 // Emit teams region as a standalone region.
8009 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8010 PrePostActionTy &Action) {
8011 Action.Enter(CGF);
8012 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8013 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8014 (void)PrivateScope.Privatize();
8015 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8016 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
8017 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8018 };
8019
8020 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for,
8021 CodeGen: CodeGenTeams);
8022 emitPostUpdateForReductionClause(CGF, D: S,
8023 CondGen: [](CodeGenFunction &) { return nullptr; });
8024}
8025
8026void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
8027 CodeGenModule &CGM, StringRef ParentName,
8028 const OMPTargetTeamsDistributeParallelForDirective &S) {
8029 // Emit SPMD target teams distribute parallel for region as a standalone
8030 // region.
8031 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8032 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
8033 };
8034 llvm::Function *Fn;
8035 llvm::Constant *Addr;
8036 // Emit target region as a standalone region.
8037 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8038 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8039 assert(Fn && Addr && "Target device function emission failed.");
8040}
8041
8042void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
8043 const OMPTargetTeamsDistributeParallelForDirective &S) {
8044 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8045 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
8046 };
8047 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8048}
8049
8050static void emitTargetTeamsDistributeParallelForSimdRegion(
8051 CodeGenFunction &CGF,
8052 const OMPTargetTeamsDistributeParallelForSimdDirective &S,
8053 PrePostActionTy &Action) {
8054 Action.Enter(CGF);
8055 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8056 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
8057 IncExpr: S.getDistInc());
8058 };
8059
8060 // Emit teams region as a standalone region.
8061 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8062 PrePostActionTy &Action) {
8063 Action.Enter(CGF);
8064 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8065 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8066 (void)PrivateScope.Privatize();
8067 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8068 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
8069 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8070 };
8071
8072 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for_simd,
8073 CodeGen: CodeGenTeams);
8074 emitPostUpdateForReductionClause(CGF, D: S,
8075 CondGen: [](CodeGenFunction &) { return nullptr; });
8076}
8077
8078void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
8079 CodeGenModule &CGM, StringRef ParentName,
8080 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
8081 // Emit SPMD target teams distribute parallel for simd region as a standalone
8082 // region.
8083 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8084 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
8085 };
8086 llvm::Function *Fn;
8087 llvm::Constant *Addr;
8088 // Emit target region as a standalone region.
8089 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8090 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8091 assert(Fn && Addr && "Target device function emission failed.");
8092}
8093
8094void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
8095 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
8096 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8097 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
8098 };
8099 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8100}
8101
8102void CodeGenFunction::EmitOMPCancellationPointDirective(
8103 const OMPCancellationPointDirective &S) {
8104 CGM.getOpenMPRuntime().emitCancellationPointCall(CGF&: *this, Loc: S.getBeginLoc(),
8105 CancelRegion: S.getCancelRegion());
8106}
8107
8108void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
8109 const Expr *IfCond = nullptr;
8110 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
8111 if (C->getNameModifier() == OMPD_unknown ||
8112 C->getNameModifier() == OMPD_cancel) {
8113 IfCond = C->getCondition();
8114 break;
8115 }
8116 }
8117 if (CGM.getLangOpts().OpenMPIRBuilder) {
8118 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
8119 // TODO: This check is necessary as we only generate `omp parallel` through
8120 // the OpenMPIRBuilder for now.
8121 if (S.getCancelRegion() == OMPD_parallel ||
8122 S.getCancelRegion() == OMPD_sections ||
8123 S.getCancelRegion() == OMPD_section) {
8124 llvm::Value *IfCondition = nullptr;
8125 if (IfCond)
8126 IfCondition = EmitScalarExpr(E: IfCond,
8127 /*IgnoreResultAssign=*/true);
8128 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
8129 ValOrErr: OMPBuilder.createCancel(Loc: Builder, IfCondition, CanceledDirective: S.getCancelRegion()));
8130 return Builder.restoreIP(IP: AfterIP);
8131 }
8132 }
8133
8134 CGM.getOpenMPRuntime().emitCancelCall(CGF&: *this, Loc: S.getBeginLoc(), IfCond,
8135 CancelRegion: S.getCancelRegion());
8136}
8137
8138CodeGenFunction::JumpDest
8139CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
8140 if (Kind == OMPD_parallel || Kind == OMPD_task ||
8141 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
8142 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
8143 return ReturnBlock;
8144 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
8145 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
8146 Kind == OMPD_distribute_parallel_for ||
8147 Kind == OMPD_target_parallel_for ||
8148 Kind == OMPD_teams_distribute_parallel_for ||
8149 Kind == OMPD_target_teams_distribute_parallel_for);
8150 return OMPCancelStack.getExitBlock();
8151}
8152
8153void CodeGenFunction::EmitOMPUseDevicePtrClause(
8154 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
8155 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8156 CaptureDeviceAddrMap) {
8157 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8158 for (const Expr *OrigVarIt : C.varlist()) {
8159 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: OrigVarIt)->getDecl());
8160 if (!Processed.insert(V: OrigVD).second)
8161 continue;
8162
8163 // In order to identify the right initializer we need to match the
8164 // declaration used by the mapping logic. In some cases we may get
8165 // OMPCapturedExprDecl that refers to the original declaration.
8166 const ValueDecl *MatchingVD = OrigVD;
8167 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: MatchingVD)) {
8168 // OMPCapturedExprDecl are used to privative fields of the current
8169 // structure.
8170 const auto *ME = cast<MemberExpr>(Val: OED->getInit());
8171 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
8172 "Base should be the current struct!");
8173 MatchingVD = ME->getMemberDecl();
8174 }
8175
8176 // If we don't have information about the current list item, move on to
8177 // the next one.
8178 auto InitAddrIt = CaptureDeviceAddrMap.find(Val: MatchingVD);
8179 if (InitAddrIt == CaptureDeviceAddrMap.end())
8180 continue;
8181
8182 llvm::Type *Ty = ConvertTypeForMem(T: OrigVD->getType().getNonReferenceType());
8183
8184 // Return the address of the private variable.
8185 bool IsRegistered = PrivateScope.addPrivate(
8186 LocalVD: OrigVD,
8187 Addr: Address(InitAddrIt->second, Ty,
8188 getContext().getTypeAlignInChars(T: getContext().VoidPtrTy)));
8189 assert(IsRegistered && "firstprivate var already registered as private");
8190 // Silence the warning about unused variable.
8191 (void)IsRegistered;
8192 }
8193}
8194
8195static const VarDecl *getBaseDecl(const Expr *Ref) {
8196 const Expr *Base = Ref->IgnoreParenImpCasts();
8197 while (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Base))
8198 Base = OASE->getBase()->IgnoreParenImpCasts();
8199 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
8200 Base = ASE->getBase()->IgnoreParenImpCasts();
8201 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Base)->getDecl());
8202}
8203
8204void CodeGenFunction::EmitOMPUseDeviceAddrClause(
8205 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
8206 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
8207 CaptureDeviceAddrMap) {
8208 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
8209 for (const Expr *Ref : C.varlist()) {
8210 const VarDecl *OrigVD = getBaseDecl(Ref);
8211 if (!Processed.insert(V: OrigVD).second)
8212 continue;
8213 // In order to identify the right initializer we need to match the
8214 // declaration used by the mapping logic. In some cases we may get
8215 // OMPCapturedExprDecl that refers to the original declaration.
8216 const ValueDecl *MatchingVD = OrigVD;
8217 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: MatchingVD)) {
8218 // OMPCapturedExprDecl are used to privative fields of the current
8219 // structure.
8220 const auto *ME = cast<MemberExpr>(Val: OED->getInit());
8221 assert(isa<CXXThisExpr>(ME->getBase()) &&
8222 "Base should be the current struct!");
8223 MatchingVD = ME->getMemberDecl();
8224 }
8225
8226 // If we don't have information about the current list item, move on to
8227 // the next one.
8228 auto InitAddrIt = CaptureDeviceAddrMap.find(Val: MatchingVD);
8229 if (InitAddrIt == CaptureDeviceAddrMap.end())
8230 continue;
8231
8232 llvm::Type *Ty = ConvertTypeForMem(T: OrigVD->getType().getNonReferenceType());
8233
8234 Address PrivAddr =
8235 Address(InitAddrIt->second, Ty,
8236 getContext().getTypeAlignInChars(T: getContext().VoidPtrTy));
8237 // For declrefs and variable length array need to load the pointer for
8238 // correct mapping, since the pointer to the data was passed to the runtime.
8239 if (isa<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts()) ||
8240 MatchingVD->getType()->isArrayType()) {
8241 QualType PtrTy = getContext().getPointerType(
8242 T: OrigVD->getType().getNonReferenceType());
8243 PrivAddr =
8244 EmitLoadOfPointer(Ptr: PrivAddr.withElementType(ElemTy: ConvertTypeForMem(T: PtrTy)),
8245 PtrTy: PtrTy->castAs<PointerType>());
8246 }
8247
8248 (void)PrivateScope.addPrivate(LocalVD: OrigVD, Addr: PrivAddr);
8249 }
8250}
8251
8252// Generate the instructions for '#pragma omp target data' directive.
8253void CodeGenFunction::EmitOMPTargetDataDirective(
8254 const OMPTargetDataDirective &S) {
8255 // Emit vtable only from host for target data directive.
8256 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
8257 CGM.getOpenMPRuntime().registerVTable(D: S);
8258
8259 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,
8260 /*SeparateBeginEndCalls=*/true);
8261
8262 // Create a pre/post action to signal the privatization of the device pointer.
8263 // This action can be replaced by the OpenMP runtime code generation to
8264 // deactivate privatization.
8265 bool PrivatizeDevicePointers = false;
8266 class DevicePointerPrivActionTy : public PrePostActionTy {
8267 bool &PrivatizeDevicePointers;
8268
8269 public:
8270 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
8271 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
8272 void Enter(CodeGenFunction &CGF) override {
8273 PrivatizeDevicePointers = true;
8274 }
8275 };
8276 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
8277
8278 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8279 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8280 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
8281 };
8282
8283 // Codegen that selects whether to generate the privatization code or not.
8284 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8285 RegionCodeGenTy RCG(InnermostCodeGen);
8286 PrivatizeDevicePointers = false;
8287
8288 // Call the pre-action to change the status of PrivatizeDevicePointers if
8289 // needed.
8290 Action.Enter(CGF);
8291
8292 if (PrivatizeDevicePointers) {
8293 OMPPrivateScope PrivateScope(CGF);
8294 // Emit all instances of the use_device_ptr clause.
8295 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8296 CGF.EmitOMPUseDevicePtrClause(C: *C, PrivateScope,
8297 CaptureDeviceAddrMap: Info.CaptureDeviceAddrMap);
8298 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8299 CGF.EmitOMPUseDeviceAddrClause(C: *C, PrivateScope,
8300 CaptureDeviceAddrMap: Info.CaptureDeviceAddrMap);
8301 (void)PrivateScope.Privatize();
8302 RCG(CGF);
8303 } else {
8304 // If we don't have target devices, don't bother emitting the data
8305 // mapping code.
8306 std::optional<OpenMPDirectiveKind> CaptureRegion;
8307 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8308 // Emit helper decls of the use_device_ptr/use_device_addr clauses.
8309 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8310 for (const Expr *E : C->varlist()) {
8311 const Decl *D = cast<DeclRefExpr>(Val: E)->getDecl();
8312 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
8313 CGF.EmitVarDecl(D: *OED);
8314 }
8315 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8316 for (const Expr *E : C->varlist()) {
8317 const Decl *D = getBaseDecl(Ref: E);
8318 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
8319 CGF.EmitVarDecl(D: *OED);
8320 }
8321 } else {
8322 CaptureRegion = OMPD_unknown;
8323 }
8324
8325 OMPLexicalScope Scope(CGF, S, CaptureRegion);
8326 RCG(CGF);
8327 }
8328 };
8329
8330 // Forward the provided action to the privatization codegen.
8331 RegionCodeGenTy PrivRCG(PrivCodeGen);
8332 PrivRCG.setAction(Action);
8333
8334 // Notwithstanding the body of the region is emitted as inlined directive,
8335 // we don't use an inline scope as changes in the references inside the
8336 // region are expected to be visible outside, so we do not privative them.
8337 OMPLexicalScope Scope(CGF, S);
8338 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_target_data,
8339 CodeGen: PrivRCG);
8340 };
8341
8342 RegionCodeGenTy RCG(CodeGen);
8343
8344 // If we don't have target devices, don't bother emitting the data mapping
8345 // code.
8346 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8347 RCG(*this);
8348 return;
8349 }
8350
8351 // Check if we have any if clause associated with the directive.
8352 const Expr *IfCond = nullptr;
8353 if (const auto *C = S.getSingleClause<OMPIfClause>())
8354 IfCond = C->getCondition();
8355
8356 // Check if we have any device clause associated with the directive.
8357 const Expr *Device = nullptr;
8358 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8359 Device = C->getDevice();
8360
8361 // Set the action to signal privatization of device pointers.
8362 RCG.setAction(PrivAction);
8363
8364 // Emit region code.
8365 CGM.getOpenMPRuntime().emitTargetDataCalls(CGF&: *this, D: S, IfCond, Device, CodeGen: RCG,
8366 Info);
8367}
8368
8369void CodeGenFunction::EmitOMPTargetEnterDataDirective(
8370 const OMPTargetEnterDataDirective &S) {
8371 // If we don't have target devices, don't bother emitting the data mapping
8372 // code.
8373 if (CGM.getLangOpts().OMPTargetTriples.empty())
8374 return;
8375
8376 // Check if we have any if clause associated with the directive.
8377 const Expr *IfCond = nullptr;
8378 if (const auto *C = S.getSingleClause<OMPIfClause>())
8379 IfCond = C->getCondition();
8380
8381 // Check if we have any device clause associated with the directive.
8382 const Expr *Device = nullptr;
8383 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8384 Device = C->getDevice();
8385
8386 OMPLexicalScope Scope(*this, S, OMPD_task);
8387 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8388}
8389
8390void CodeGenFunction::EmitOMPTargetExitDataDirective(
8391 const OMPTargetExitDataDirective &S) {
8392 // If we don't have target devices, don't bother emitting the data mapping
8393 // code.
8394 if (CGM.getLangOpts().OMPTargetTriples.empty())
8395 return;
8396
8397 // Check if we have any if clause associated with the directive.
8398 const Expr *IfCond = nullptr;
8399 if (const auto *C = S.getSingleClause<OMPIfClause>())
8400 IfCond = C->getCondition();
8401
8402 // Check if we have any device clause associated with the directive.
8403 const Expr *Device = nullptr;
8404 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8405 Device = C->getDevice();
8406
8407 OMPLexicalScope Scope(*this, S, OMPD_task);
8408 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8409}
8410
8411static void emitTargetParallelRegion(CodeGenFunction &CGF,
8412 const OMPTargetParallelDirective &S,
8413 PrePostActionTy &Action) {
8414 // Get the captured statement associated with the 'parallel' region.
8415 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
8416 Action.Enter(CGF);
8417 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
8418 Action.Enter(CGF);
8419 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8420 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
8421 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
8422 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8423 (void)PrivateScope.Privatize();
8424 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
8425 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
8426 // TODO: Add support for clauses.
8427 CGF.EmitStmt(S: CS->getCapturedStmt());
8428 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
8429 };
8430 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_parallel, CodeGen,
8431 CodeGenBoundParameters: emitEmptyBoundParameters);
8432 emitPostUpdateForReductionClause(CGF, D: S,
8433 CondGen: [](CodeGenFunction &) { return nullptr; });
8434}
8435
8436void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
8437 CodeGenModule &CGM, StringRef ParentName,
8438 const OMPTargetParallelDirective &S) {
8439 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8440 emitTargetParallelRegion(CGF, S, Action);
8441 };
8442 llvm::Function *Fn;
8443 llvm::Constant *Addr;
8444 // Emit target region as a standalone region.
8445 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8446 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8447 assert(Fn && Addr && "Target device function emission failed.");
8448}
8449
8450void CodeGenFunction::EmitOMPTargetParallelDirective(
8451 const OMPTargetParallelDirective &S) {
8452 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8453 emitTargetParallelRegion(CGF, S, Action);
8454 };
8455 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8456}
8457
8458static void emitTargetParallelForRegion(CodeGenFunction &CGF,
8459 const OMPTargetParallelForDirective &S,
8460 PrePostActionTy &Action) {
8461 Action.Enter(CGF);
8462 // Emit directive as a combined directive that consists of two implicit
8463 // directives: 'parallel' with 'for' directive.
8464 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8465 Action.Enter(CGF);
8466 CodeGenFunction::OMPCancelStackRAII CancelRegion(
8467 CGF, OMPD_target_parallel_for, S.hasCancel());
8468 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
8469 CGDispatchBounds: emitDispatchForLoopBounds);
8470 };
8471 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_for, CodeGen,
8472 CodeGenBoundParameters: emitEmptyBoundParameters);
8473}
8474
8475void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
8476 CodeGenModule &CGM, StringRef ParentName,
8477 const OMPTargetParallelForDirective &S) {
8478 // Emit SPMD target parallel for region as a standalone region.
8479 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8480 emitTargetParallelForRegion(CGF, S, Action);
8481 };
8482 llvm::Function *Fn;
8483 llvm::Constant *Addr;
8484 // Emit target region as a standalone region.
8485 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8486 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8487 assert(Fn && Addr && "Target device function emission failed.");
8488}
8489
8490void CodeGenFunction::EmitOMPTargetParallelForDirective(
8491 const OMPTargetParallelForDirective &S) {
8492 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8493 emitTargetParallelForRegion(CGF, S, Action);
8494 };
8495 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8496}
8497
8498static void
8499emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
8500 const OMPTargetParallelForSimdDirective &S,
8501 PrePostActionTy &Action) {
8502 Action.Enter(CGF);
8503 // Emit directive as a combined directive that consists of two implicit
8504 // directives: 'parallel' with 'for' directive.
8505 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8506 Action.Enter(CGF);
8507 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
8508 CGDispatchBounds: emitDispatchForLoopBounds);
8509 };
8510 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_simd, CodeGen,
8511 CodeGenBoundParameters: emitEmptyBoundParameters);
8512}
8513
8514void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
8515 CodeGenModule &CGM, StringRef ParentName,
8516 const OMPTargetParallelForSimdDirective &S) {
8517 // Emit SPMD target parallel for region as a standalone region.
8518 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8519 emitTargetParallelForSimdRegion(CGF, S, Action);
8520 };
8521 llvm::Function *Fn;
8522 llvm::Constant *Addr;
8523 // Emit target region as a standalone region.
8524 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8525 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8526 assert(Fn && Addr && "Target device function emission failed.");
8527}
8528
8529void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
8530 const OMPTargetParallelForSimdDirective &S) {
8531 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8532 emitTargetParallelForSimdRegion(CGF, S, Action);
8533 };
8534 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8535}
8536
8537/// Emit a helper variable and return corresponding lvalue.
8538static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
8539 const ImplicitParamDecl *PVD,
8540 CodeGenFunction::OMPPrivateScope &Privates) {
8541 const auto *VDecl = cast<VarDecl>(Val: Helper->getDecl());
8542 Privates.addPrivate(LocalVD: VDecl, Addr: CGF.GetAddrOfLocalVar(VD: PVD));
8543}
8544
8545void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
8546 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
8547 // Emit outlined function for task construct.
8548 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_taskloop);
8549 Address CapturedStruct = Address::invalid();
8550 {
8551 OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8552 CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
8553 }
8554 CanQualType SharedsTy =
8555 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
8556 const Expr *IfCond = nullptr;
8557 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
8558 if (C->getNameModifier() == OMPD_unknown ||
8559 C->getNameModifier() == OMPD_taskloop) {
8560 IfCond = C->getCondition();
8561 break;
8562 }
8563 }
8564
8565 OMPTaskDataTy Data;
8566 // Check if taskloop must be emitted without taskgroup.
8567 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
8568 // TODO: Check if we should emit tied or untied task.
8569 Data.Tied = true;
8570 // Set scheduling for taskloop
8571 if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
8572 // grainsize clause
8573 Data.Schedule.setInt(/*IntVal=*/false);
8574 Data.Schedule.setPointer(EmitScalarExpr(E: Clause->getGrainsize()));
8575 Data.HasModifier =
8576 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;
8577 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
8578 // num_tasks clause
8579 Data.Schedule.setInt(/*IntVal=*/true);
8580 Data.Schedule.setPointer(EmitScalarExpr(E: Clause->getNumTasks()));
8581 Data.HasModifier =
8582 (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;
8583 }
8584
8585 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
8586 // if (PreCond) {
8587 // for (IV in 0..LastIteration) BODY;
8588 // <Final counter/linear vars updates>;
8589 // }
8590 //
8591
8592 // Emit: if (PreCond) - begin.
8593 // If the condition constant folds and can be elided, avoid emitting the
8594 // whole loop.
8595 bool CondConstant;
8596 llvm::BasicBlock *ContBlock = nullptr;
8597 OMPLoopScope PreInitScope(CGF, S);
8598 if (CGF.ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
8599 if (!CondConstant)
8600 return;
8601 } else {
8602 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "taskloop.if.then");
8603 ContBlock = CGF.createBasicBlock(name: "taskloop.if.end");
8604 emitPreCond(CGF, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
8605 TrueCount: CGF.getProfileCount(S: &S));
8606 CGF.EmitBlock(BB: ThenBlock);
8607 CGF.incrementProfileCounter(S: &S);
8608 }
8609
8610 (void)CGF.EmitOMPLinearClauseInit(D: S);
8611
8612 OMPPrivateScope LoopScope(CGF);
8613 // Emit helper vars inits.
8614 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8615 auto *I = CS->getCapturedDecl()->param_begin();
8616 auto *LBP = std::next(x: I, n: LowerBound);
8617 auto *UBP = std::next(x: I, n: UpperBound);
8618 auto *STP = std::next(x: I, n: Stride);
8619 auto *LIP = std::next(x: I, n: LastIter);
8620 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getLowerBoundVariable()), PVD: *LBP,
8621 Privates&: LoopScope);
8622 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getUpperBoundVariable()), PVD: *UBP,
8623 Privates&: LoopScope);
8624 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()), PVD: *STP, Privates&: LoopScope);
8625 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()), PVD: *LIP,
8626 Privates&: LoopScope);
8627 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8628 CGF.EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
8629 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
8630 (void)LoopScope.Privatize();
8631 // Emit the loop iteration variable.
8632 const Expr *IVExpr = S.getIterationVariable();
8633 const auto *IVDecl = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IVExpr)->getDecl());
8634 CGF.EmitVarDecl(D: *IVDecl);
8635 CGF.EmitIgnoredExpr(E: S.getInit());
8636
8637 // Emit the iterations count variable.
8638 // If it is not a variable, Sema decided to calculate iterations count on
8639 // each iteration (e.g., it is foldable into a constant).
8640 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
8641 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
8642 // Emit calculation of the iterations count.
8643 CGF.EmitIgnoredExpr(E: S.getCalcLastIteration());
8644 }
8645
8646 {
8647 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8648 emitCommonSimdLoop(
8649 CGF, S,
8650 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8651 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()))
8652 CGF.EmitOMPSimdInit(D: S);
8653 },
8654 BodyCodeGen: [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
8655 CGF.EmitOMPInnerLoop(
8656 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: S.getCond(), IncExpr: S.getInc(),
8657 BodyGen: [&S](CodeGenFunction &CGF) {
8658 emitOMPLoopBodyWithStopPoint(CGF, S,
8659 LoopExit: CodeGenFunction::JumpDest());
8660 },
8661 PostIncGen: [](CodeGenFunction &) {});
8662 });
8663 }
8664 // Emit: if (PreCond) - end.
8665 if (ContBlock) {
8666 CGF.EmitBranch(Block: ContBlock);
8667 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
8668 }
8669 // Emit final copy of the lastprivate variables if IsLastIter != 0.
8670 if (HasLastprivateClause) {
8671 CGF.EmitOMPLastprivateClauseFinal(
8672 D: S, NoFinals: isOpenMPSimdDirective(DKind: S.getDirectiveKind()),
8673 IsLastIterCond: CGF.Builder.CreateIsNotNull(Arg: CGF.EmitLoadOfScalar(
8674 Addr: CGF.GetAddrOfLocalVar(VD: *LIP), /*Volatile=*/false,
8675 Ty: (*LIP)->getType(), Loc: S.getBeginLoc())));
8676 }
8677 LoopScope.restoreMap();
8678 CGF.EmitOMPLinearClauseFinal(D: S, CondGen: [LIP, &S](CodeGenFunction &CGF) {
8679 return CGF.Builder.CreateIsNotNull(
8680 Arg: CGF.EmitLoadOfScalar(Addr: CGF.GetAddrOfLocalVar(VD: *LIP), /*Volatile=*/false,
8681 Ty: (*LIP)->getType(), Loc: S.getBeginLoc()));
8682 });
8683 };
8684 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8685 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8686 const OMPTaskDataTy &Data) {
8687 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8688 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
8689 OMPLoopScope PreInitScope(CGF, S);
8690 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, Loc: S.getBeginLoc(), D: S,
8691 TaskFunction: OutlinedFn, SharedsTy,
8692 Shareds: CapturedStruct, IfCond, Data);
8693 };
8694 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_taskloop,
8695 CodeGen);
8696 };
8697 if (Data.Nogroup) {
8698 EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_taskloop, BodyGen, TaskGen, Data);
8699 } else {
8700 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8701 CGF&: *this,
8702 TaskgroupOpGen: [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
8703 PrePostActionTy &Action) {
8704 Action.Enter(CGF);
8705 CGF.EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_taskloop, BodyGen, TaskGen,
8706 Data);
8707 },
8708 Loc: S.getBeginLoc());
8709 }
8710}
8711
8712void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
8713 auto LPCRegion =
8714 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8715 EmitOMPTaskLoopBasedDirective(S);
8716}
8717
8718void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
8719 const OMPTaskLoopSimdDirective &S) {
8720 auto LPCRegion =
8721 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8722 OMPLexicalScope Scope(*this, S);
8723 EmitOMPTaskLoopBasedDirective(S);
8724}
8725
8726void CodeGenFunction::EmitOMPMasterTaskLoopDirective(
8727 const OMPMasterTaskLoopDirective &S) {
8728 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8729 Action.Enter(CGF);
8730 EmitOMPTaskLoopBasedDirective(S);
8731 };
8732 auto LPCRegion =
8733 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8734 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8735 CGM.getOpenMPRuntime().emitMasterRegion(CGF&: *this, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
8736}
8737
8738void CodeGenFunction::EmitOMPMaskedTaskLoopDirective(
8739 const OMPMaskedTaskLoopDirective &S) {
8740 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8741 Action.Enter(CGF);
8742 EmitOMPTaskLoopBasedDirective(S);
8743 };
8744 auto LPCRegion =
8745 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8746 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8747 CGM.getOpenMPRuntime().emitMaskedRegion(CGF&: *this, MaskedOpGen: CodeGen, Loc: S.getBeginLoc());
8748}
8749
8750void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(
8751 const OMPMasterTaskLoopSimdDirective &S) {
8752 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8753 Action.Enter(CGF);
8754 EmitOMPTaskLoopBasedDirective(S);
8755 };
8756 auto LPCRegion =
8757 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8758 OMPLexicalScope Scope(*this, S);
8759 CGM.getOpenMPRuntime().emitMasterRegion(CGF&: *this, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
8760}
8761
8762void CodeGenFunction::EmitOMPMaskedTaskLoopSimdDirective(
8763 const OMPMaskedTaskLoopSimdDirective &S) {
8764 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8765 Action.Enter(CGF);
8766 EmitOMPTaskLoopBasedDirective(S);
8767 };
8768 auto LPCRegion =
8769 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8770 OMPLexicalScope Scope(*this, S);
8771 CGM.getOpenMPRuntime().emitMaskedRegion(CGF&: *this, MaskedOpGen: CodeGen, Loc: S.getBeginLoc());
8772}
8773
8774void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(
8775 const OMPParallelMasterTaskLoopDirective &S) {
8776 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8777 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8778 PrePostActionTy &Action) {
8779 Action.Enter(CGF);
8780 CGF.EmitOMPTaskLoopBasedDirective(S);
8781 };
8782 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8783 CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: TaskLoopCodeGen,
8784 Loc: S.getBeginLoc());
8785 };
8786 auto LPCRegion =
8787 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8788 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master_taskloop, CodeGen,
8789 CodeGenBoundParameters: emitEmptyBoundParameters);
8790}
8791
8792void CodeGenFunction::EmitOMPParallelMaskedTaskLoopDirective(
8793 const OMPParallelMaskedTaskLoopDirective &S) {
8794 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8795 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8796 PrePostActionTy &Action) {
8797 Action.Enter(CGF);
8798 CGF.EmitOMPTaskLoopBasedDirective(S);
8799 };
8800 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8801 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: TaskLoopCodeGen,
8802 Loc: S.getBeginLoc());
8803 };
8804 auto LPCRegion =
8805 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8806 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked_taskloop, CodeGen,
8807 CodeGenBoundParameters: emitEmptyBoundParameters);
8808}
8809
8810void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(
8811 const OMPParallelMasterTaskLoopSimdDirective &S) {
8812 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8813 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8814 PrePostActionTy &Action) {
8815 Action.Enter(CGF);
8816 CGF.EmitOMPTaskLoopBasedDirective(S);
8817 };
8818 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8819 CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: TaskLoopCodeGen,
8820 Loc: S.getBeginLoc());
8821 };
8822 auto LPCRegion =
8823 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8824 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master_taskloop_simd, CodeGen,
8825 CodeGenBoundParameters: emitEmptyBoundParameters);
8826}
8827
8828void CodeGenFunction::EmitOMPParallelMaskedTaskLoopSimdDirective(
8829 const OMPParallelMaskedTaskLoopSimdDirective &S) {
8830 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8831 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8832 PrePostActionTy &Action) {
8833 Action.Enter(CGF);
8834 CGF.EmitOMPTaskLoopBasedDirective(S);
8835 };
8836 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8837 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: TaskLoopCodeGen,
8838 Loc: S.getBeginLoc());
8839 };
8840 auto LPCRegion =
8841 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8842 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked_taskloop_simd, CodeGen,
8843 CodeGenBoundParameters: emitEmptyBoundParameters);
8844}
8845
8846// Generate the instructions for '#pragma omp target update' directive.
8847void CodeGenFunction::EmitOMPTargetUpdateDirective(
8848 const OMPTargetUpdateDirective &S) {
8849 // If we don't have target devices, don't bother emitting the data mapping
8850 // code.
8851 if (CGM.getLangOpts().OMPTargetTriples.empty())
8852 return;
8853
8854 // Check if we have any if clause associated with the directive.
8855 const Expr *IfCond = nullptr;
8856 if (const auto *C = S.getSingleClause<OMPIfClause>())
8857 IfCond = C->getCondition();
8858
8859 // Check if we have any device clause associated with the directive.
8860 const Expr *Device = nullptr;
8861 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8862 Device = C->getDevice();
8863
8864 OMPLexicalScope Scope(*this, S, OMPD_task);
8865 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8866}
8867
8868void CodeGenFunction::EmitOMPGenericLoopDirective(
8869 const OMPGenericLoopDirective &S) {
8870 // Always expect a bind clause on the loop directive. It it wasn't
8871 // in the source, it should have been added in sema.
8872
8873 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;
8874 if (const auto *C = S.getSingleClause<OMPBindClause>())
8875 BindKind = C->getBindKind();
8876
8877 switch (BindKind) {
8878 case OMPC_BIND_parallel: // for
8879 return emitOMPForDirective(S, CGF&: *this, CGM, /*HasCancel=*/false);
8880 case OMPC_BIND_teams: // distribute
8881 return emitOMPDistributeDirective(S, CGF&: *this, CGM);
8882 case OMPC_BIND_thread: // simd
8883 return emitOMPSimdDirective(S, CGF&: *this, CGM);
8884 case OMPC_BIND_unknown:
8885 break;
8886 }
8887
8888 // Unimplemented, just inline the underlying statement for now.
8889 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8890 // Emit the loop iteration variable.
8891 const Stmt *CS =
8892 cast<CapturedStmt>(Val: S.getAssociatedStmt())->getCapturedStmt();
8893 const auto *ForS = dyn_cast<ForStmt>(Val: CS);
8894 if (ForS && !isa<DeclStmt>(Val: ForS->getInit())) {
8895 OMPPrivateScope LoopScope(CGF);
8896 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8897 (void)LoopScope.Privatize();
8898 CGF.EmitStmt(S: CS);
8899 LoopScope.restoreMap();
8900 } else {
8901 CGF.EmitStmt(S: CS);
8902 }
8903 };
8904 OMPLexicalScope Scope(*this, S, OMPD_unknown);
8905 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_loop, CodeGen);
8906}
8907
8908void CodeGenFunction::EmitOMPParallelGenericLoopDirective(
8909 const OMPLoopDirective &S) {
8910 // Emit combined directive as if its constituent constructs are 'parallel'
8911 // and 'for'.
8912 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8913 Action.Enter(CGF);
8914 emitOMPCopyinClause(CGF, S);
8915 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
8916 };
8917 {
8918 auto LPCRegion =
8919 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8920 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for, CodeGen,
8921 CodeGenBoundParameters: emitEmptyBoundParameters);
8922 }
8923 // Check for outer lastprivate conditional update.
8924 checkForLastprivateConditionalUpdate(CGF&: *this, S);
8925}
8926
8927void CodeGenFunction::EmitOMPTeamsGenericLoopDirective(
8928 const OMPTeamsGenericLoopDirective &S) {
8929 // To be consistent with current behavior of 'target teams loop', emit
8930 // 'teams loop' as if its constituent constructs are 'teams' and 'distribute'.
8931 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8932 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
8933 };
8934
8935 // Emit teams region as a standalone region.
8936 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8937 PrePostActionTy &Action) {
8938 Action.Enter(CGF);
8939 OMPPrivateScope PrivateScope(CGF);
8940 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8941 (void)PrivateScope.Privatize();
8942 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
8943 CodeGen: CodeGenDistribute);
8944 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8945 };
8946 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
8947 emitPostUpdateForReductionClause(CGF&: *this, D: S,
8948 CondGen: [](CodeGenFunction &) { return nullptr; });
8949}
8950
8951#ifndef NDEBUG
8952static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF,
8953 std::string StatusMsg,
8954 const OMPExecutableDirective &D) {
8955 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
8956 if (IsDevice)
8957 StatusMsg += ": DEVICE";
8958 else
8959 StatusMsg += ": HOST";
8960 SourceLocation L = D.getBeginLoc();
8961 auto &SM = CGF.getContext().getSourceManager();
8962 PresumedLoc PLoc = SM.getPresumedLoc(L);
8963 const char *FileName = PLoc.isValid() ? PLoc.getFilename() : nullptr;
8964 unsigned LineNo =
8965 PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L);
8966 llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n";
8967}
8968#endif
8969
8970static void emitTargetTeamsGenericLoopRegionAsParallel(
8971 CodeGenFunction &CGF, PrePostActionTy &Action,
8972 const OMPTargetTeamsGenericLoopDirective &S) {
8973 Action.Enter(CGF);
8974 // Emit 'teams loop' as if its constituent constructs are 'distribute,
8975 // 'parallel, and 'for'.
8976 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8977 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
8978 IncExpr: S.getDistInc());
8979 };
8980
8981 // Emit teams region as a standalone region.
8982 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8983 PrePostActionTy &Action) {
8984 Action.Enter(CGF);
8985 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8986 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8987 (void)PrivateScope.Privatize();
8988 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8989 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
8990 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8991 };
8992 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
8993 emitTargetTeamsLoopCodegenStatus(
8994 CGF, TTL_CODEGEN_TYPE " as parallel for", S));
8995 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for,
8996 CodeGen: CodeGenTeams);
8997 emitPostUpdateForReductionClause(CGF, D: S,
8998 CondGen: [](CodeGenFunction &) { return nullptr; });
8999}
9000
9001static void emitTargetTeamsGenericLoopRegionAsDistribute(
9002 CodeGenFunction &CGF, PrePostActionTy &Action,
9003 const OMPTargetTeamsGenericLoopDirective &S) {
9004 Action.Enter(CGF);
9005 // Emit 'teams loop' as if its constituent construct is 'distribute'.
9006 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
9007 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
9008 };
9009
9010 // Emit teams region as a standalone region.
9011 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
9012 PrePostActionTy &Action) {
9013 Action.Enter(CGF);
9014 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
9015 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
9016 (void)PrivateScope.Privatize();
9017 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
9018 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
9019 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
9020 };
9021 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
9022 emitTargetTeamsLoopCodegenStatus(
9023 CGF, TTL_CODEGEN_TYPE " as distribute", S));
9024 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute, CodeGen);
9025 emitPostUpdateForReductionClause(CGF, D: S,
9026 CondGen: [](CodeGenFunction &) { return nullptr; });
9027}
9028
9029void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDirective(
9030 const OMPTargetTeamsGenericLoopDirective &S) {
9031 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9032 if (S.canBeParallelFor())
9033 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
9034 else
9035 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
9036 };
9037 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
9038}
9039
9040void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
9041 CodeGenModule &CGM, StringRef ParentName,
9042 const OMPTargetTeamsGenericLoopDirective &S) {
9043 // Emit SPMD target parallel loop region as a standalone region.
9044 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9045 if (S.canBeParallelFor())
9046 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
9047 else
9048 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
9049 };
9050 llvm::Function *Fn;
9051 llvm::Constant *Addr;
9052 // Emit target region as a standalone region.
9053 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9054 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
9055 assert(Fn && Addr &&
9056 "Target device function emission failed for 'target teams loop'.");
9057}
9058
9059static void emitTargetParallelGenericLoopRegion(
9060 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S,
9061 PrePostActionTy &Action) {
9062 Action.Enter(CGF);
9063 // Emit as 'parallel for'.
9064 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9065 Action.Enter(CGF);
9066 CodeGenFunction::OMPCancelStackRAII CancelRegion(
9067 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
9068 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
9069 CGDispatchBounds: emitDispatchForLoopBounds);
9070 };
9071 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_for, CodeGen,
9072 CodeGenBoundParameters: emitEmptyBoundParameters);
9073}
9074
9075void CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
9076 CodeGenModule &CGM, StringRef ParentName,
9077 const OMPTargetParallelGenericLoopDirective &S) {
9078 // Emit target parallel loop region as a standalone region.
9079 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9080 emitTargetParallelGenericLoopRegion(CGF, S, Action);
9081 };
9082 llvm::Function *Fn;
9083 llvm::Constant *Addr;
9084 // Emit target region as a standalone region.
9085 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
9086 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
9087 assert(Fn && Addr && "Target device function emission failed.");
9088}
9089
9090/// Emit combined directive 'target parallel loop' as if its constituent
9091/// constructs are 'target', 'parallel', and 'for'.
9092void CodeGenFunction::EmitOMPTargetParallelGenericLoopDirective(
9093 const OMPTargetParallelGenericLoopDirective &S) {
9094 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
9095 emitTargetParallelGenericLoopRegion(CGF, S, Action);
9096 };
9097 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
9098}
9099
9100void CodeGenFunction::EmitSimpleOMPExecutableDirective(
9101 const OMPExecutableDirective &D) {
9102 if (const auto *SD = dyn_cast<OMPScanDirective>(Val: &D)) {
9103 EmitOMPScanDirective(S: *SD);
9104 return;
9105 }
9106 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
9107 return;
9108 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
9109 OMPPrivateScope GlobalsScope(CGF);
9110 if (isOpenMPTaskingDirective(Kind: D.getDirectiveKind())) {
9111 // Capture global firstprivates to avoid crash.
9112 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
9113 for (const Expr *Ref : C->varlist()) {
9114 const auto *DRE = cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
9115 if (!DRE)
9116 continue;
9117 const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl());
9118 if (!VD || VD->hasLocalStorage())
9119 continue;
9120 if (!CGF.LocalDeclMap.count(Val: VD)) {
9121 LValue GlobLVal = CGF.EmitLValue(E: Ref);
9122 GlobalsScope.addPrivate(LocalVD: VD, Addr: GlobLVal.getAddress());
9123 }
9124 }
9125 }
9126 }
9127 if (isOpenMPSimdDirective(DKind: D.getDirectiveKind())) {
9128 (void)GlobalsScope.Privatize();
9129 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
9130 emitOMPSimdRegion(CGF, S: cast<OMPLoopDirective>(Val: D), Action);
9131 } else {
9132 if (const auto *LD = dyn_cast<OMPLoopDirective>(Val: &D)) {
9133 for (const Expr *E : LD->counters()) {
9134 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
9135 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(Val: VD)) {
9136 LValue GlobLVal = CGF.EmitLValue(E);
9137 GlobalsScope.addPrivate(LocalVD: VD, Addr: GlobLVal.getAddress());
9138 }
9139 if (isa<OMPCapturedExprDecl>(Val: VD)) {
9140 // Emit only those that were not explicitly referenced in clauses.
9141 if (!CGF.LocalDeclMap.count(Val: VD))
9142 CGF.EmitVarDecl(D: *VD);
9143 }
9144 }
9145 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
9146 if (!C->getNumForLoops())
9147 continue;
9148 for (unsigned I = LD->getLoopsNumber(),
9149 E = C->getLoopNumIterations().size();
9150 I < E; ++I) {
9151 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
9152 Val: cast<DeclRefExpr>(Val: C->getLoopCounter(NumLoop: I))->getDecl())) {
9153 // Emit only those that were not explicitly referenced in clauses.
9154 if (!CGF.LocalDeclMap.count(Val: VD))
9155 CGF.EmitVarDecl(D: *VD);
9156 }
9157 }
9158 }
9159 }
9160 (void)GlobalsScope.Privatize();
9161 CGF.EmitStmt(S: D.getInnermostCapturedStmt()->getCapturedStmt());
9162 }
9163 };
9164 if (D.getDirectiveKind() == OMPD_atomic ||
9165 D.getDirectiveKind() == OMPD_critical ||
9166 D.getDirectiveKind() == OMPD_section ||
9167 D.getDirectiveKind() == OMPD_master ||
9168 D.getDirectiveKind() == OMPD_masked ||
9169 D.getDirectiveKind() == OMPD_unroll ||
9170 D.getDirectiveKind() == OMPD_assume) {
9171 EmitStmt(S: D.getAssociatedStmt());
9172 } else {
9173 auto LPCRegion =
9174 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S: D);
9175 OMPSimdLexicalScope Scope(*this, D);
9176 CGM.getOpenMPRuntime().emitInlinedDirective(
9177 CGF&: *this,
9178 InnermostKind: isOpenMPSimdDirective(DKind: D.getDirectiveKind()) ? OMPD_simd
9179 : D.getDirectiveKind(),
9180 CodeGen);
9181 }
9182 // Check for outer lastprivate conditional update.
9183 checkForLastprivateConditionalUpdate(CGF&: *this, S: D);
9184}
9185
9186void CodeGenFunction::EmitOMPAssumeDirective(const OMPAssumeDirective &S) {
9187 for (const auto *C : S.getClausesOfKind<OMPHoldsClause>()) {
9188 const Expr *E = C->getExpr();
9189 assert(E && "holds clause requires an expression");
9190 if (!E->HasSideEffects(Ctx: getContext()))
9191 Builder.CreateAssumption(Cond: EvaluateExprAsBool(E));
9192 }
9193 EmitStmt(S: S.getAssociatedStmt());
9194}
9195