1//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCleanup.h"
14#include "CGDebugInfo.h"
15#include "CGOpenMPRuntime.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
18#include "CodeGenPGO.h"
19#include "TargetInfo.h"
20#include "clang/AST/ASTContext.h"
21#include "clang/AST/Attr.h"
22#include "clang/AST/DeclOpenMP.h"
23#include "clang/AST/OpenMPClause.h"
24#include "clang/AST/Stmt.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/PrettyStackTrace.h"
30#include "clang/Basic/SourceManager.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/BinaryFormat/Dwarf.h"
33#include "llvm/Frontend/OpenMP/OMPConstants.h"
34#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugInfoMetadata.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/IntrinsicInst.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/Support/AtomicOrdering.h"
41#include "llvm/Support/Debug.h"
42#include <optional>
43using namespace clang;
44using namespace CodeGen;
45using namespace llvm::omp;
46
47#define TTL_CODEGEN_TYPE "target-teams-loop-codegen"
48
49static const VarDecl *getBaseDecl(const Expr *Ref);
50static OpenMPDirectiveKind
51getEffectiveDirectiveKind(const OMPExecutableDirective &S);
52
53/// Whether a combined `distribute parallel for` may use the fused
54/// distr_static_chunk + static_chunkone schedule (enum 93): one
55/// for_static_init, no surrounding distribute_static_init.
56static bool canEmitGPUFusedDistSchedule(const CodeGenModule &CGM,
57 const OMPLoopDirective &S,
58 OpenMPDirectiveKind DKind) {
59 // Reduction-only for now. Non-reduction cases might follow in the future, but
60 // need more analysis for maximum profit.
61 return CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU() &&
62 isOpenMPLoopBoundSharingDirective(Kind: DKind) &&
63 S.hasClausesOfKind<OMPReductionClause>() &&
64 !S.getSingleClause<OMPDistScheduleClause>() &&
65 !S.getSingleClause<OMPScheduleClause>() &&
66 !S.getSingleClause<OMPOrderedClause>();
67}
68
69namespace {
70/// Lexical scope for OpenMP executable constructs, that handles correct codegen
71/// for captured expressions.
72class OMPLexicalScope : public CodeGenFunction::LexicalScope {
73 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
74 for (const auto *C : S.clauses()) {
75 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
76 if (const auto *PreInit =
77 cast_or_null<DeclStmt>(Val: CPI->getPreInitStmt())) {
78 for (const auto *I : PreInit->decls()) {
79 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
80 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
81 } else {
82 CodeGenFunction::AutoVarEmission Emission =
83 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
84 CGF.EmitAutoVarCleanups(emission: Emission);
85 }
86 }
87 }
88 }
89 }
90 }
91 CodeGenFunction::OMPPrivateScope InlinedShareds;
92
93 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
94 return CGF.LambdaCaptureFields.lookup(Val: VD) ||
95 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
96 (isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl) &&
97 cast<BlockDecl>(Val: CGF.CurCodeDecl)->capturesVariable(var: VD));
98 }
99
100public:
101 OMPLexicalScope(
102 CodeGenFunction &CGF, const OMPExecutableDirective &S,
103 const std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,
104 const bool EmitPreInitStmt = true)
105 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
106 InlinedShareds(CGF) {
107 if (EmitPreInitStmt)
108 emitPreInitStmt(CGF, S);
109 if (!CapturedRegion)
110 return;
111 assert(S.hasAssociatedStmt() &&
112 "Expected associated statement for inlined directive.");
113 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: *CapturedRegion);
114 for (const auto &C : CS->captures()) {
115 if (C.capturesVariable() || C.capturesVariableByCopy()) {
116 auto *VD = C.getCapturedVar();
117 assert(VD == VD->getCanonicalDecl() &&
118 "Canonical decl must be captured.");
119 DeclRefExpr DRE(
120 CGF.getContext(), const_cast<VarDecl *>(VD),
121 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
122 InlinedShareds.isGlobalVarCaptured(VD)),
123 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
124 InlinedShareds.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
125 }
126 }
127 (void)InlinedShareds.Privatize();
128 }
129};
130
131/// Lexical scope for OpenMP parallel construct, that handles correct codegen
132/// for captured expressions.
133class OMPParallelScope final : public OMPLexicalScope {
134 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
135 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
136 return !(isOpenMPTargetExecutionDirective(DKind: EKind) ||
137 isOpenMPLoopBoundSharingDirective(Kind: EKind)) &&
138 isOpenMPParallelDirective(DKind: EKind);
139 }
140
141public:
142 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
143 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
144 EmitPreInitStmt(S)) {}
145};
146
147/// Lexical scope for OpenMP teams construct, that handles correct codegen
148/// for captured expressions.
149class OMPTeamsScope final : public OMPLexicalScope {
150 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
151 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
152 return !isOpenMPTargetExecutionDirective(DKind: EKind) &&
153 isOpenMPTeamsDirective(DKind: EKind);
154 }
155
156public:
157 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
158 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/std::nullopt,
159 EmitPreInitStmt(S)) {}
160};
161
162/// Private scope for OpenMP loop-based directives, that supports capturing
163/// of used expression from loop statement.
164class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
165 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
166 const Stmt *PreInits;
167 CodeGenFunction::OMPMapVars PreCondVars;
168 if (auto *LD = dyn_cast<OMPLoopDirective>(Val: &S)) {
169 // Emit init, __range, __begin and __end variables for C++ range loops.
170 (void)OMPLoopBasedDirective::doForAllLoops(
171 CurStmt: LD->getInnermostCapturedStmt()->getCapturedStmt(),
172 /*TryImperfectlyNestedLoops=*/true, NumLoops: LD->getLoopsNumber(),
173 Callback: [&CGF](unsigned Cnt, const Stmt *CurStmt) {
174 if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(Val: CurStmt)) {
175 if (const Stmt *Init = CXXFor->getInit())
176 CGF.EmitStmt(S: Init);
177 CGF.EmitStmt(S: CXXFor->getRangeStmt());
178 CGF.EmitStmt(S: CXXFor->getBeginStmt());
179 CGF.EmitStmt(S: CXXFor->getEndStmt());
180 }
181 return false;
182 });
183 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
184 for (const auto *E : LD->counters()) {
185 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
186 EmittedAsPrivate.insert(V: VD->getCanonicalDecl());
187 (void)PreCondVars.setVarAddr(
188 CGF, LocalVD: VD, TempAddr: CGF.CreateMemTemp(T: VD->getType().getNonReferenceType()));
189 }
190 // Mark private vars as undefs.
191 for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
192 for (const Expr *IRef : C->varlist()) {
193 const auto *OrigVD =
194 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IRef)->getDecl());
195 if (EmittedAsPrivate.insert(V: OrigVD->getCanonicalDecl()).second) {
196 QualType OrigVDTy = OrigVD->getType().getNonReferenceType();
197 (void)PreCondVars.setVarAddr(
198 CGF, LocalVD: OrigVD,
199 TempAddr: Address(llvm::UndefValue::get(T: CGF.ConvertTypeForMem(
200 T: CGF.getContext().getPointerType(T: OrigVDTy))),
201 CGF.ConvertTypeForMem(T: OrigVDTy),
202 CGF.getContext().getDeclAlign(D: OrigVD)));
203 }
204 }
205 }
206 (void)PreCondVars.apply(CGF);
207 PreInits = LD->getPreInits();
208 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(Val: &S)) {
209 PreInits = Tile->getPreInits();
210 } else if (const auto *Stripe = dyn_cast<OMPStripeDirective>(Val: &S)) {
211 PreInits = Stripe->getPreInits();
212 } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(Val: &S)) {
213 PreInits = Unroll->getPreInits();
214 } else if (const auto *Reverse = dyn_cast<OMPReverseDirective>(Val: &S)) {
215 PreInits = Reverse->getPreInits();
216 } else if (const auto *Split = dyn_cast<OMPSplitDirective>(Val: &S)) {
217 PreInits = Split->getPreInits();
218 } else if (const auto *Interchange =
219 dyn_cast<OMPInterchangeDirective>(Val: &S)) {
220 PreInits = Interchange->getPreInits();
221 } else {
222 llvm_unreachable("Unknown loop-based directive kind.");
223 }
224 doEmitPreinits(PreInits);
225 PreCondVars.restore(CGF);
226 }
227
228 void
229 emitPreInitStmt(CodeGenFunction &CGF,
230 const OMPCanonicalLoopSequenceTransformationDirective &S) {
231 const Stmt *PreInits;
232 if (const auto *Fuse = dyn_cast<OMPFuseDirective>(Val: &S)) {
233 PreInits = Fuse->getPreInits();
234 } else {
235 llvm_unreachable(
236 "Unknown canonical loop sequence transform directive kind.");
237 }
238 doEmitPreinits(PreInits);
239 }
240
241 void doEmitPreinits(const Stmt *PreInits) {
242 if (PreInits) {
243 // CompoundStmts and DeclStmts are used as lists of PreInit statements and
244 // declarations. Since declarations must be visible in the the following
245 // that they initialize, unpack the CompoundStmt they are nested in.
246 SmallVector<const Stmt *> PreInitStmts;
247 if (auto *PreInitCompound = dyn_cast<CompoundStmt>(Val: PreInits))
248 llvm::append_range(C&: PreInitStmts, R: PreInitCompound->body());
249 else
250 PreInitStmts.push_back(Elt: PreInits);
251
252 for (const Stmt *S : PreInitStmts) {
253 // EmitStmt skips any OMPCapturedExprDecls, but needs to be emitted
254 // here.
255 if (auto *PreInitDecl = dyn_cast<DeclStmt>(Val: S)) {
256 for (Decl *I : PreInitDecl->decls())
257 CGF.EmitVarDecl(D: cast<VarDecl>(Val&: *I));
258 continue;
259 }
260 CGF.EmitStmt(S);
261 }
262 }
263 }
264
265public:
266 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
267 : CodeGenFunction::RunCleanupsScope(CGF) {
268 emitPreInitStmt(CGF, S);
269 }
270 OMPLoopScope(CodeGenFunction &CGF,
271 const OMPCanonicalLoopSequenceTransformationDirective &S)
272 : CodeGenFunction::RunCleanupsScope(CGF) {
273 emitPreInitStmt(CGF, S);
274 }
275};
276
277class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
278 CodeGenFunction::OMPPrivateScope InlinedShareds;
279
280 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
281 return CGF.LambdaCaptureFields.lookup(Val: VD) ||
282 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
283 (isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl) &&
284 cast<BlockDecl>(Val: CGF.CurCodeDecl)->capturesVariable(var: VD));
285 }
286
287public:
288 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
289 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
290 InlinedShareds(CGF) {
291 for (const auto *C : S.clauses()) {
292 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
293 if (const auto *PreInit =
294 cast_or_null<DeclStmt>(Val: CPI->getPreInitStmt())) {
295 for (const auto *I : PreInit->decls()) {
296 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
297 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
298 } else {
299 CodeGenFunction::AutoVarEmission Emission =
300 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
301 CGF.EmitAutoVarCleanups(emission: Emission);
302 }
303 }
304 }
305 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(Val: C)) {
306 for (const Expr *E : UDP->varlist()) {
307 const Decl *D = cast<DeclRefExpr>(Val: E)->getDecl();
308 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
309 CGF.EmitVarDecl(D: *OED);
310 }
311 } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(Val: C)) {
312 for (const Expr *E : UDP->varlist()) {
313 const Decl *D = getBaseDecl(Ref: E);
314 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
315 CGF.EmitVarDecl(D: *OED);
316 }
317 }
318 }
319 if (!isOpenMPSimdDirective(DKind: getEffectiveDirectiveKind(S)))
320 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: InlinedShareds);
321 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(Val: &S)) {
322 if (const Expr *E = TG->getReductionRef())
323 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl()));
324 }
325 // Temp copy arrays for inscan reductions should not be emitted as they are
326 // not used in simd only mode.
327 llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
328 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
329 if (C->getModifier() != OMPC_REDUCTION_inscan)
330 continue;
331 for (const Expr *E : C->copy_array_temps())
332 CopyArrayTemps.insert(V: cast<DeclRefExpr>(Val: E)->getDecl());
333 }
334 const auto *CS = cast_or_null<CapturedStmt>(Val: S.getAssociatedStmt());
335 while (CS) {
336 for (auto &C : CS->captures()) {
337 if (C.capturesVariable() || C.capturesVariableByCopy()) {
338 auto *VD = C.getCapturedVar();
339 if (CopyArrayTemps.contains(V: VD))
340 continue;
341 assert(VD == VD->getCanonicalDecl() &&
342 "Canonical decl must be captured.");
343 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
344 isCapturedVar(CGF, VD) ||
345 (CGF.CapturedStmtInfo &&
346 InlinedShareds.isGlobalVarCaptured(VD)),
347 VD->getType().getNonReferenceType(), VK_LValue,
348 C.getLocation());
349 InlinedShareds.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
350 }
351 }
352 CS = dyn_cast<CapturedStmt>(Val: CS->getCapturedStmt());
353 }
354 (void)InlinedShareds.Privatize();
355 }
356};
357
358} // namespace
359
360// The loop directive with a bind clause will be mapped to a different
361// directive with corresponding semantics.
362static OpenMPDirectiveKind
363getEffectiveDirectiveKind(const OMPExecutableDirective &S) {
364 OpenMPDirectiveKind Kind = S.getDirectiveKind();
365 if (Kind != OMPD_loop)
366 return Kind;
367
368 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;
369 if (const auto *C = S.getSingleClause<OMPBindClause>())
370 BindKind = C->getBindKind();
371
372 switch (BindKind) {
373 case OMPC_BIND_parallel:
374 return OMPD_for;
375 case OMPC_BIND_teams:
376 return OMPD_distribute;
377 case OMPC_BIND_thread:
378 return OMPD_simd;
379 default:
380 return OMPD_loop;
381 }
382}
383
384static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
385 const OMPExecutableDirective &S,
386 const RegionCodeGenTy &CodeGen);
387
388LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
389 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(Val: E)) {
390 if (const auto *OrigVD = dyn_cast<VarDecl>(Val: OrigDRE->getDecl())) {
391 OrigVD = OrigVD->getCanonicalDecl();
392 bool IsCaptured =
393 LambdaCaptureFields.lookup(Val: OrigVD) ||
394 (CapturedStmtInfo && CapturedStmtInfo->lookup(VD: OrigVD)) ||
395 (isa_and_nonnull<BlockDecl>(Val: CurCodeDecl));
396 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
397 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
398 return EmitLValue(E: &DRE);
399 }
400 }
401 return EmitLValue(E);
402}
403
404llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
405 ASTContext &C = getContext();
406 llvm::Value *Size = nullptr;
407 auto SizeInChars = C.getTypeSizeInChars(T: Ty);
408 if (SizeInChars.isZero()) {
409 // getTypeSizeInChars() returns 0 for a VLA.
410 while (const VariableArrayType *VAT = C.getAsVariableArrayType(T: Ty)) {
411 VlaSizePair VlaSize = getVLASize(vla: VAT);
412 Ty = VlaSize.Type;
413 Size =
414 Size ? Builder.CreateNUWMul(LHS: Size, RHS: VlaSize.NumElts) : VlaSize.NumElts;
415 }
416 SizeInChars = C.getTypeSizeInChars(T: Ty);
417 if (SizeInChars.isZero())
418 return llvm::ConstantInt::get(Ty: SizeTy, /*V=*/0);
419 return Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: SizeInChars));
420 }
421 return CGM.getSize(numChars: SizeInChars);
422}
423
424void CodeGenFunction::GenerateOpenMPCapturedVars(
425 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
426 const RecordDecl *RD = S.getCapturedRecordDecl();
427 auto CurField = RD->field_begin();
428 auto CurCap = S.captures().begin();
429 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
430 E = S.capture_init_end();
431 I != E; ++I, ++CurField, ++CurCap) {
432 if (CurField->hasCapturedVLAType()) {
433 const VariableArrayType *VAT = CurField->getCapturedVLAType();
434 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
435 CapturedVars.push_back(Elt: Val);
436 } else if (CurCap->capturesThis()) {
437 CapturedVars.push_back(Elt: CXXThisValue);
438 } else if (CurCap->capturesVariableByCopy()) {
439 llvm::Value *CV = EmitLoadOfScalar(lvalue: EmitLValue(E: *I), Loc: CurCap->getLocation());
440
441 // If the field is not a pointer, we need to save the actual value
442 // and load it as a void pointer.
443 if (!CurField->getType()->isAnyPointerType()) {
444 ASTContext &Ctx = getContext();
445 Address DstAddr = CreateMemTempWithoutCast(
446 T: Ctx.getUIntPtrType(),
447 Name: Twine(CurCap->getCapturedVar()->getName(), ".casted"));
448 LValue DstLV = MakeAddrLValue(Addr: DstAddr, T: Ctx.getUIntPtrType());
449
450 llvm::Value *SrcAddrVal = EmitScalarConversion(
451 Src: DstAddr.emitRawPointer(CGF&: *this),
452 SrcTy: Ctx.getPointerType(T: Ctx.getUIntPtrType()),
453 DstTy: Ctx.getPointerType(T: CurField->getType()), Loc: CurCap->getLocation());
454 LValue SrcLV =
455 MakeNaturalAlignAddrLValue(V: SrcAddrVal, T: CurField->getType());
456
457 // Store the value using the source type pointer.
458 EmitStoreThroughLValue(Src: RValue::get(V: CV), Dst: SrcLV);
459
460 // Load the value using the destination type pointer.
461 CV = EmitLoadOfScalar(lvalue: DstLV, Loc: CurCap->getLocation());
462 }
463 CapturedVars.push_back(Elt: CV);
464 } else {
465 assert(CurCap->capturesVariable() && "Expected capture by reference.");
466 CapturedVars.push_back(Elt: EmitLValue(E: *I).getAddress().emitRawPointer(CGF&: *this));
467 }
468 }
469}
470
471static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
472 QualType DstType, StringRef Name,
473 LValue AddrLV) {
474 ASTContext &Ctx = CGF.getContext();
475
476 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
477 Src: AddrLV.getAddress().emitRawPointer(CGF), SrcTy: Ctx.getUIntPtrType(),
478 DstTy: Ctx.getPointerType(T: DstType), Loc);
479 // FIXME: should the pointee type (DstType) be passed?
480 Address TmpAddr =
481 CGF.MakeNaturalAlignAddrLValue(V: CastedPtr, T: DstType).getAddress();
482 return TmpAddr;
483}
484
485static QualType getCanonicalParamType(ASTContext &C, QualType T) {
486 if (T->isLValueReferenceType())
487 return C.getLValueReferenceType(
488 T: getCanonicalParamType(C, T: T.getNonReferenceType()),
489 /*SpelledAsLValue=*/false);
490 if (T->isPointerType())
491 return C.getPointerType(T: getCanonicalParamType(C, T: T->getPointeeType()));
492 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
493 if (const auto *VLA = dyn_cast<VariableArrayType>(Val: A))
494 return getCanonicalParamType(C, T: VLA->getElementType());
495 if (!A->isVariablyModifiedType())
496 return C.getCanonicalType(T);
497 }
498 return C.getCanonicalParamType(T);
499}
500
501namespace {
502/// Contains required data for proper outlined function codegen.
503struct FunctionOptions {
504 /// Captured statement for which the function is generated.
505 const CapturedStmt *S = nullptr;
506 /// true if cast to/from UIntPtr is required for variables captured by
507 /// value.
508 const bool UIntPtrCastRequired = true;
509 /// true if only casted arguments must be registered as local args or VLA
510 /// sizes.
511 const bool RegisterCastedArgsOnly = false;
512 /// Name of the generated function.
513 const StringRef FunctionName;
514 /// Location of the non-debug version of the outlined function.
515 SourceLocation Loc;
516 const bool IsDeviceKernel = false;
517 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
518 bool RegisterCastedArgsOnly, StringRef FunctionName,
519 SourceLocation Loc, bool IsDeviceKernel)
520 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
521 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
522 FunctionName(FunctionName), Loc(Loc), IsDeviceKernel(IsDeviceKernel) {}
523};
524} // namespace
525
526static llvm::Function *emitOutlinedFunctionPrologue(
527 CodeGenFunction &CGF, FunctionArgList &Args,
528 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
529 &LocalAddrs,
530 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
531 &VLASizes,
532 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
533 const CapturedDecl *CD = FO.S->getCapturedDecl();
534 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
535 assert(CD->hasBody() && "missing CapturedDecl body");
536
537 CXXThisValue = nullptr;
538 // Build the argument list.
539 CodeGenModule &CGM = CGF.CGM;
540 ASTContext &Ctx = CGM.getContext();
541 FunctionArgList TargetArgs;
542 Args.append(in_start: CD->param_begin(),
543 in_end: std::next(x: CD->param_begin(), n: CD->getContextParamPosition()));
544 TargetArgs.append(
545 in_start: CD->param_begin(),
546 in_end: std::next(x: CD->param_begin(), n: CD->getContextParamPosition()));
547 auto I = FO.S->captures().begin();
548 FunctionDecl *DebugFunctionDecl = nullptr;
549 if (!FO.UIntPtrCastRequired) {
550 FunctionProtoType::ExtProtoInfo EPI;
551 QualType FunctionTy = Ctx.getFunctionType(ResultTy: Ctx.VoidTy, Args: {}, EPI);
552 DebugFunctionDecl = FunctionDecl::Create(
553 C&: Ctx, DC: Ctx.getTranslationUnitDecl(), StartLoc: FO.S->getBeginLoc(),
554 NLoc: SourceLocation(), N: DeclarationName(), T: FunctionTy,
555 TInfo: Ctx.getTrivialTypeSourceInfo(T: FunctionTy), SC: SC_Static,
556 /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
557 /*hasWrittenPrototype=*/false);
558 }
559 for (const FieldDecl *FD : RD->fields()) {
560 QualType ArgType = FD->getType();
561 IdentifierInfo *II = nullptr;
562 VarDecl *CapVar = nullptr;
563
564 // If this is a capture by copy and the type is not a pointer, the outlined
565 // function argument type should be uintptr and the value properly casted to
566 // uintptr. This is necessary given that the runtime library is only able to
567 // deal with pointers. We can pass in the same way the VLA type sizes to the
568 // outlined function.
569 if (FO.UIntPtrCastRequired &&
570 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
571 I->capturesVariableArrayType()))
572 ArgType = Ctx.getUIntPtrType();
573
574 if (I->capturesVariable() || I->capturesVariableByCopy()) {
575 CapVar = I->getCapturedVar();
576 II = CapVar->getIdentifier();
577 } else if (I->capturesThis()) {
578 II = &Ctx.Idents.get(Name: "this");
579 } else {
580 assert(I->capturesVariableArrayType());
581 II = &Ctx.Idents.get(Name: "vla");
582 }
583 if (ArgType->isVariablyModifiedType())
584 ArgType = getCanonicalParamType(C&: Ctx, T: ArgType);
585 VarDecl *Arg;
586 if (CapVar && (CapVar->getTLSKind() != clang::VarDecl::TLS_None)) {
587 Arg = ImplicitParamDecl::Create(C&: Ctx, /*DC=*/nullptr, IdLoc: FD->getLocation(),
588 Id: II, T: ArgType,
589 ParamKind: ImplicitParamKind::ThreadPrivateVar);
590 } else if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
591 Arg = ParmVarDecl::Create(
592 C&: Ctx, DC: DebugFunctionDecl,
593 StartLoc: CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
594 IdLoc: CapVar ? CapVar->getLocation() : FD->getLocation(), Id: II, T: ArgType,
595 /*TInfo=*/nullptr, S: SC_None, /*DefArg=*/nullptr);
596 } else {
597 Arg = ImplicitParamDecl::Create(C&: Ctx, /*DC=*/nullptr, IdLoc: FD->getLocation(),
598 Id: II, T: ArgType, ParamKind: ImplicitParamKind::Other);
599 }
600 Args.emplace_back(Args&: Arg);
601 // Do not cast arguments if we emit function with non-original types.
602 TargetArgs.emplace_back(
603 Args: FO.UIntPtrCastRequired
604 ? Arg
605 : CGM.getOpenMPRuntime().translateParameter(FD, NativeParam: Arg));
606 ++I;
607 }
608 Args.append(in_start: std::next(x: CD->param_begin(), n: CD->getContextParamPosition() + 1),
609 in_end: CD->param_end());
610 TargetArgs.append(
611 in_start: std::next(x: CD->param_begin(), n: CD->getContextParamPosition() + 1),
612 in_end: CD->param_end());
613
614 // Create the function declaration.
615 const CGFunctionInfo &FuncInfo =
616 FO.IsDeviceKernel
617 ? CGM.getTypes().arrangeDeviceKernelCallerDeclaration(resultType: Ctx.VoidTy,
618 args: TargetArgs)
619 : CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy,
620 args: TargetArgs);
621 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
622
623 auto *F =
624 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
625 N: FO.FunctionName, M: &CGM.getModule());
626 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
627
628 // Adjust the calling convention for SPIR-V targets to avoid mismatches
629 // between callee and caller.
630 if (CGM.getTriple().isSPIRV() && !FO.IsDeviceKernel)
631 F->setCallingConv(llvm::CallingConv::SPIR_FUNC);
632
633 if (CD->isNothrow())
634 F->setDoesNotThrow();
635
636 // Always inline the outlined function if optimizations are enabled.
637 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
638 F->removeFnAttr(Kind: llvm::Attribute::NoInline);
639 F->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
640 }
641 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
642 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
643
644 // Generate the function.
645 CGF.StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args: TargetArgs,
646 Loc: FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
647 StartLoc: FO.UIntPtrCastRequired ? FO.Loc
648 : CD->getBody()->getBeginLoc());
649 unsigned Cnt = CD->getContextParamPosition();
650 I = FO.S->captures().begin();
651 for (const FieldDecl *FD : RD->fields()) {
652 // Do not map arguments if we emit function with non-original types.
653 Address LocalAddr(Address::invalid());
654 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
655 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, NativeParam: Args[Cnt],
656 TargetParam: TargetArgs[Cnt]);
657 } else {
658 LocalAddr = CGF.GetAddrOfLocalVar(VD: Args[Cnt]);
659 }
660 // If we are capturing a pointer by copy we don't need to do anything, just
661 // use the value that we get from the arguments.
662 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
663 const VarDecl *CurVD = I->getCapturedVar();
664 if (!FO.RegisterCastedArgsOnly)
665 LocalAddrs.insert(KV: {Args[Cnt], {CurVD, LocalAddr}});
666 ++Cnt;
667 ++I;
668 continue;
669 }
670
671 LValue ArgLVal = CGF.MakeAddrLValue(Addr: LocalAddr, T: Args[Cnt]->getType(),
672 Source: AlignmentSource::Decl);
673 if (FD->hasCapturedVLAType()) {
674 if (FO.UIntPtrCastRequired) {
675 ArgLVal = CGF.MakeAddrLValue(
676 Addr: castValueFromUintptr(CGF, Loc: I->getLocation(), DstType: FD->getType(),
677 Name: Args[Cnt]->getName(), AddrLV: ArgLVal),
678 T: FD->getType(), Source: AlignmentSource::Decl);
679 }
680 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: I->getLocation());
681 const VariableArrayType *VAT = FD->getCapturedVLAType();
682 VLASizes.try_emplace(Key: Args[Cnt], Args: VAT->getSizeExpr(), Args&: ExprArg);
683 } else if (I->capturesVariable()) {
684 const VarDecl *Var = I->getCapturedVar();
685 QualType VarTy = Var->getType();
686 Address ArgAddr = ArgLVal.getAddress();
687 if (ArgLVal.getType()->isLValueReferenceType()) {
688 ArgAddr = CGF.EmitLoadOfReference(RefLVal: ArgLVal);
689 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
690 assert(ArgLVal.getType()->isPointerType());
691 ArgAddr = CGF.EmitLoadOfPointer(
692 Ptr: ArgAddr, PtrTy: ArgLVal.getType()->castAs<PointerType>());
693 }
694 if (!FO.RegisterCastedArgsOnly) {
695 LocalAddrs.insert(
696 KV: {Args[Cnt], {Var, ArgAddr.withAlignment(NewAlignment: Ctx.getDeclAlign(D: Var))}});
697 }
698 } else if (I->capturesVariableByCopy()) {
699 assert(!FD->getType()->isAnyPointerType() &&
700 "Not expecting a captured pointer.");
701 const VarDecl *Var = I->getCapturedVar();
702 LocalAddrs.insert(KV: {Args[Cnt],
703 {Var, FO.UIntPtrCastRequired
704 ? castValueFromUintptr(
705 CGF, Loc: I->getLocation(), DstType: FD->getType(),
706 Name: Args[Cnt]->getName(), AddrLV: ArgLVal)
707 : ArgLVal.getAddress()}});
708 } else {
709 // If 'this' is captured, load it into CXXThisValue.
710 assert(I->capturesThis());
711 CXXThisValue = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: I->getLocation());
712 LocalAddrs.insert(KV: {Args[Cnt], {nullptr, ArgLVal.getAddress()}});
713 }
714 ++Cnt;
715 ++I;
716 }
717
718 return F;
719}
720
721static llvm::Function *emitOutlinedFunctionPrologueAggregate(
722 CodeGenFunction &CGF, FunctionArgList &Args,
723 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
724 &LocalAddrs,
725 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
726 &VLASizes,
727 llvm::Value *&CXXThisValue, llvm::Value *&ContextV, const CapturedStmt &CS,
728 SourceLocation Loc, StringRef FunctionName) {
729 const CapturedDecl *CD = CS.getCapturedDecl();
730 const RecordDecl *RD = CS.getCapturedRecordDecl();
731
732 CXXThisValue = nullptr;
733 CodeGenModule &CGM = CGF.CGM;
734 ASTContext &Ctx = CGM.getContext();
735 Args.push_back(Elt: CD->getContextParam());
736
737 const CGFunctionInfo &FuncInfo =
738 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
739 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
740
741 auto *F =
742 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
743 N: FunctionName, M: &CGM.getModule());
744 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
745 if (CD->isNothrow())
746 F->setDoesNotThrow();
747
748 CGF.StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc, StartLoc: Loc);
749 Address ContextAddr = CGF.GetAddrOfLocalVar(VD: CD->getContextParam());
750 ContextV = CGF.Builder.CreateLoad(Addr: ContextAddr);
751
752 // The runtime passes arguments as an array of pointers.
753 llvm::Type *PtrTy = CGF.Builder.getPtrTy();
754 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
755 CharUnits SlotAlign = CharUnits::fromQuantity(Quantity: PtrAlign.value());
756
757 for (auto [FD, C, FieldIdx] :
758 llvm::zip(t: RD->fields(), u: CS.captures(),
759 args: llvm::seq<unsigned>(Size: RD->getNumFields()))) {
760 llvm::Value *SlotPtr =
761 CGF.Builder.CreateConstInBoundsGEP1_32(Ty: PtrTy, Ptr: ContextV, Idx0: FieldIdx);
762 llvm::Value *Slot = CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Ptr: SlotPtr, Align: PtrAlign);
763
764 // Generate the appropriate load from the per-argument storage. This
765 // includes all of the user arguments as well as the implicit kernel
766 // argument pointer.
767 if (C.capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
768 const VarDecl *CurVD = C.getCapturedVar();
769 Slot->setName(CurVD->getName());
770 Address SlotAddr(Slot, PtrTy, SlotAlign);
771 LocalAddrs.insert(KV: {FD, {CurVD, SlotAddr}});
772 } else if (FD->hasCapturedVLAType()) {
773 // VLA size is stored as intptr_t directly in the slot.
774 Address SlotAddr(Slot, CGF.ConvertTypeForMem(T: FD->getType()), SlotAlign);
775 LValue ArgLVal =
776 CGF.MakeAddrLValue(Addr: SlotAddr, T: FD->getType(), Source: AlignmentSource::Decl);
777 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(lvalue: ArgLVal, Loc: C.getLocation());
778 const VariableArrayType *VAT = FD->getCapturedVLAType();
779 VLASizes.try_emplace(Key: FD, Args: VAT->getSizeExpr(), Args&: ExprArg);
780 } else if (C.capturesVariable()) {
781 const VarDecl *Var = C.getCapturedVar();
782 QualType VarTy = Var->getType();
783
784 if (VarTy->isVariablyModifiedType() && VarTy->isPointerType()) {
785 Slot->setName(Var->getName() + ".addr");
786 Address SlotAddr(Slot, PtrTy, SlotAlign);
787 LocalAddrs.insert(KV: {FD, {Var, SlotAddr}});
788 } else {
789 llvm::Value *VarAddr = CGF.Builder.CreateAlignedLoad(
790 Ty: PtrTy, Ptr: Slot, Align: PtrAlign, Name: Var->getName());
791 LocalAddrs.insert(KV: {FD,
792 {Var, Address(VarAddr, CGF.ConvertTypeForMem(T: VarTy),
793 Ctx.getDeclAlign(D: Var))}});
794 }
795 } else if (C.capturesVariableByCopy()) {
796 assert(!FD->getType()->isAnyPointerType() &&
797 "Not expecting a captured pointer.");
798 const VarDecl *Var = C.getCapturedVar();
799 QualType FieldTy = FD->getType();
800
801 // Scalar values are promoted and stored directly in the slot.
802 Address SlotAddr(Slot, CGF.ConvertTypeForMem(T: FieldTy), SlotAlign);
803 Address CopyAddr =
804 CGF.CreateMemTemp(T: FieldTy, Align: Ctx.getDeclAlign(D: FD), Name: Var->getName());
805 LValue SrcLVal =
806 CGF.MakeAddrLValue(Addr: SlotAddr, T: FieldTy, Source: AlignmentSource::Decl);
807 LValue CopyLVal =
808 CGF.MakeAddrLValue(Addr: CopyAddr, T: FieldTy, Source: AlignmentSource::Decl);
809
810 RValue ArgRVal = CGF.EmitLoadOfLValue(V: SrcLVal, Loc: C.getLocation());
811 CGF.EmitStoreThroughLValue(Src: ArgRVal, Dst: CopyLVal);
812
813 LocalAddrs.insert(KV: {FD, {Var, CopyAddr}});
814 } else {
815 assert(C.capturesThis() && "Default case expected to be CXX 'this'");
816 CXXThisValue =
817 CGF.Builder.CreateAlignedLoad(Ty: PtrTy, Ptr: Slot, Align: PtrAlign, Name: "this");
818 Address SlotAddr(Slot, PtrTy, SlotAlign);
819 LocalAddrs.insert(KV: {FD, {nullptr, SlotAddr}});
820 }
821 }
822
823 return F;
824}
825
826llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunction(
827 const CapturedStmt &S, const OMPExecutableDirective &D) {
828 SourceLocation Loc = D.getBeginLoc();
829 assert(
830 CapturedStmtInfo &&
831 "CapturedStmtInfo should be set when generating the captured function");
832 const CapturedDecl *CD = S.getCapturedDecl();
833 // Build the argument list.
834 bool NeedWrapperFunction =
835 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
836 FunctionArgList Args, WrapperArgs;
837 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,
838 WrapperLocalAddrs;
839 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,
840 WrapperVLASizes;
841 SmallString<256> Buffer;
842 llvm::raw_svector_ostream Out(Buffer);
843 Out << CapturedStmtInfo->getHelperName();
844 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
845 bool IsDeviceKernel = CGM.getOpenMPRuntime().isGPU() &&
846 isOpenMPTargetExecutionDirective(DKind: EKind) &&
847 D.getCapturedStmt(RegionKind: OMPD_target) == &S;
848 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
849 llvm::Function *WrapperF = nullptr;
850 if (NeedWrapperFunction) {
851 // Emit the final kernel early to allow attributes to be added by the
852 // OpenMPI-IR-Builder.
853 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
854 /*RegisterCastedArgsOnly=*/true,
855 CapturedStmtInfo->getHelperName(), Loc,
856 IsDeviceKernel);
857 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
858 WrapperF =
859 emitOutlinedFunctionPrologue(CGF&: WrapperCGF, Args, LocalAddrs, VLASizes,
860 CXXThisValue&: WrapperCGF.CXXThisValue, FO: WrapperFO);
861 Out << "_debug__";
862 }
863 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
864 Out.str(), Loc, !NeedWrapperFunction && IsDeviceKernel);
865 llvm::Function *F = emitOutlinedFunctionPrologue(
866 CGF&: *this, Args&: WrapperArgs, LocalAddrs&: WrapperLocalAddrs, VLASizes&: WrapperVLASizes, CXXThisValue, FO);
867 CodeGenFunction::OMPPrivateScope LocalScope(*this);
868 for (const auto &LocalAddrPair : WrapperLocalAddrs) {
869 if (LocalAddrPair.second.first) {
870 LocalScope.addPrivate(LocalVD: LocalAddrPair.second.first,
871 Addr: LocalAddrPair.second.second);
872 }
873 }
874 (void)LocalScope.Privatize();
875 for (const auto &VLASizePair : WrapperVLASizes)
876 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
877 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
878 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
879 LocalScope.ForceCleanup();
880 FinishFunction(EndLoc: CD->getBodyRBrace());
881 if (!NeedWrapperFunction)
882 return F;
883
884 // Reverse the order.
885 WrapperF->removeFromParent();
886 F->getParent()->getFunctionList().insertAfter(where: F->getIterator(), New: WrapperF);
887
888 llvm::SmallVector<llvm::Value *, 4> CallArgs;
889 auto *PI = F->arg_begin();
890 for (const auto *Arg : Args) {
891 llvm::Value *CallArg;
892 auto I = LocalAddrs.find(Key: Arg);
893 if (I != LocalAddrs.end()) {
894 LValue LV = WrapperCGF.MakeAddrLValue(
895 Addr: I->second.second,
896 T: I->second.first ? I->second.first->getType() : Arg->getType(),
897 Source: AlignmentSource::Decl);
898 if (LV.getType()->isAnyComplexType())
899 LV.setAddress(LV.getAddress().withElementType(ElemTy: PI->getType()));
900 CallArg = WrapperCGF.EmitLoadOfScalar(lvalue: LV, Loc: S.getBeginLoc());
901 } else {
902 auto EI = VLASizes.find(Val: Arg);
903 if (EI != VLASizes.end()) {
904 CallArg = EI->second.second;
905 } else {
906 LValue LV =
907 WrapperCGF.MakeAddrLValue(Addr: WrapperCGF.GetAddrOfLocalVar(VD: Arg),
908 T: Arg->getType(), Source: AlignmentSource::Decl);
909 CallArg = WrapperCGF.EmitLoadOfScalar(lvalue: LV, Loc: S.getBeginLoc());
910 }
911 }
912 CallArgs.emplace_back(Args: WrapperCGF.EmitFromMemory(Value: CallArg, Ty: Arg->getType()));
913 ++PI;
914 }
915 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF&: WrapperCGF, Loc, OutlinedFn: F, Args: CallArgs);
916 WrapperCGF.FinishFunction();
917 return WrapperF;
918}
919
920llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunctionAggregate(
921 const CapturedStmt &S, const OMPExecutableDirective &D) {
922 SourceLocation Loc = D.getBeginLoc();
923 assert(
924 CapturedStmtInfo &&
925 "CapturedStmtInfo should be set when generating the captured function");
926 const CapturedDecl *CD = S.getCapturedDecl();
927 const RecordDecl *RD = S.getCapturedRecordDecl();
928 StringRef FunctionName = CapturedStmtInfo->getHelperName();
929 bool NeedWrapperFunction =
930 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
931
932 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
933 llvm::Function *WrapperF = nullptr;
934 llvm::Value *WrapperContextV = nullptr;
935 if (NeedWrapperFunction) {
936 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
937 FunctionArgList WrapperArgs;
938 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
939 WrapperLocalAddrs;
940 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
941 WrapperVLASizes;
942 WrapperF = emitOutlinedFunctionPrologueAggregate(
943 CGF&: WrapperCGF, Args&: WrapperArgs, LocalAddrs&: WrapperLocalAddrs, VLASizes&: WrapperVLASizes,
944 CXXThisValue&: WrapperCGF.CXXThisValue, ContextV&: WrapperContextV, CS: S, Loc, FunctionName);
945 }
946
947 FunctionArgList Args;
948 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
949 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
950 llvm::Function *F;
951
952 if (NeedWrapperFunction) {
953 SmallString<256> Buffer;
954 llvm::raw_svector_ostream Out(Buffer);
955 Out << FunctionName << "_debug__";
956
957 FunctionOptions FO(&S, /*UIntPtrCastRequired=*/false,
958 /*RegisterCastedArgsOnly=*/false, Out.str(), Loc,
959 /*IsDeviceKernel=*/false);
960 F = emitOutlinedFunctionPrologue(CGF&: *this, Args, LocalAddrs, VLASizes,
961 CXXThisValue, FO);
962 } else {
963 llvm::Value *ContextV = nullptr;
964 F = emitOutlinedFunctionPrologueAggregate(CGF&: *this, Args, LocalAddrs, VLASizes,
965 CXXThisValue, ContextV, CS: S, Loc,
966 FunctionName);
967
968 const RecordDecl *RD = S.getCapturedRecordDecl();
969 unsigned FieldIdx = RD->getNumFields();
970 for (unsigned I = 0; I < CD->getNumParams(); ++I) {
971 const ImplicitParamDecl *Param = CD->getParam(i: I);
972 if (Param == CD->getContextParam())
973 continue;
974 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
975 llvm::Value *SlotPtr = Builder.CreateConstInBoundsGEP1_32(
976 Ty: Builder.getPtrTy(), Ptr: ContextV, Idx0: FieldIdx,
977 Name: Twine(Param->getName()) + ".addr");
978 llvm::Value *ParamAddr =
979 Builder.CreateAlignedLoad(Ty: Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
980 llvm::Value *ParamVal = Builder.CreateAlignedLoad(
981 Ty: Builder.getPtrTy(), Ptr: ParamAddr, Align: PtrAlign, Name: Param->getName());
982 Address ParamLocalAddr =
983 CreateMemTemp(T: Param->getType(), Name: Param->getName());
984 Builder.CreateStore(Val: ParamVal, Addr: ParamLocalAddr);
985 LocalAddrs.insert(KV: {Param, {Param, ParamLocalAddr}});
986 ++FieldIdx;
987 }
988 }
989
990 CodeGenFunction::OMPPrivateScope LocalScope(*this);
991 for (const auto &LocalAddrPair : LocalAddrs) {
992 if (LocalAddrPair.second.first)
993 LocalScope.addPrivate(LocalVD: LocalAddrPair.second.first,
994 Addr: LocalAddrPair.second.second);
995 }
996 (void)LocalScope.Privatize();
997 for (const auto &VLASizePair : VLASizes)
998 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
999 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
1000 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
1001 (void)LocalScope.ForceCleanup();
1002 FinishFunction(EndLoc: CD->getBodyRBrace());
1003
1004 if (!NeedWrapperFunction)
1005 return F;
1006
1007 // Reverse the order.
1008 WrapperF->removeFromParent();
1009 F->getParent()->getFunctionList().insertAfter(where: F->getIterator(), New: WrapperF);
1010
1011 llvm::Align PtrAlign = CGM.getDataLayout().getPointerABIAlignment(AS: 0);
1012 llvm::SmallVector<llvm::Value *, 16> CallArgs;
1013 assert(CD->getContextParamPosition() == 0 &&
1014 "Expected context param at position 0 for target regions");
1015 assert(RD->getNumFields() + 1 == F->getNumOperands() &&
1016 "Argument count mismatch");
1017
1018 for (auto [FD, InnerParam, SlotIdx] : llvm::zip(
1019 t: RD->fields(), u: F->args(), args: llvm::seq<unsigned>(Size: RD->getNumFields()))) {
1020 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1021 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: WrapperContextV, Idx0: SlotIdx);
1022 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1023 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1024 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1025 Ty: InnerParam.getType(), Ptr: Slot, Align: PtrAlign, Name: InnerParam.getName());
1026 CallArgs.push_back(Elt: Val);
1027 }
1028
1029 // Handle the load from the implicit dyn_ptr at the end of the __context.
1030 unsigned SlotIdx = RD->getNumFields();
1031 auto InnerParam = F->arg_begin() + SlotIdx;
1032 llvm::Value *SlotPtr = WrapperCGF.Builder.CreateConstInBoundsGEP1_32(
1033 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: WrapperContextV, Idx0: SlotIdx);
1034 llvm::Value *Slot = WrapperCGF.Builder.CreateAlignedLoad(
1035 Ty: WrapperCGF.Builder.getPtrTy(), Ptr: SlotPtr, Align: PtrAlign);
1036 llvm::Value *Val = WrapperCGF.Builder.CreateAlignedLoad(
1037 Ty: InnerParam->getType(), Ptr: Slot, Align: PtrAlign, Name: InnerParam->getName());
1038 CallArgs.push_back(Elt: Val);
1039
1040 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF&: WrapperCGF, Loc, OutlinedFn: F, Args: CallArgs);
1041 WrapperCGF.FinishFunction();
1042 return WrapperF;
1043}
1044
1045//===----------------------------------------------------------------------===//
1046// OpenMP Directive Emission
1047//===----------------------------------------------------------------------===//
1048void CodeGenFunction::EmitOMPAggregateAssign(
1049 Address DestAddr, Address SrcAddr, QualType OriginalType,
1050 const llvm::function_ref<void(Address, Address)> CopyGen) {
1051 // Perform element-by-element initialization.
1052 QualType ElementTy;
1053
1054 // Drill down to the base element type on both arrays.
1055 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
1056 llvm::Value *NumElements = emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
1057 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
1058
1059 llvm::Value *SrcBegin = SrcAddr.emitRawPointer(CGF&: *this);
1060 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF&: *this);
1061 // Cast from pointer to array type to pointer to single element.
1062 llvm::Value *DestEnd = Builder.CreateInBoundsGEP(Ty: DestAddr.getElementType(),
1063 Ptr: DestBegin, IdxList: NumElements);
1064
1065 // The basic structure here is a while-do loop.
1066 llvm::BasicBlock *BodyBB = createBasicBlock(name: "omp.arraycpy.body");
1067 llvm::BasicBlock *DoneBB = createBasicBlock(name: "omp.arraycpy.done");
1068 llvm::Value *IsEmpty =
1069 Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arraycpy.isempty");
1070 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
1071
1072 // Enter the loop body, making that address the current address.
1073 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
1074 EmitBlock(BB: BodyBB);
1075
1076 CharUnits ElementSize = getContext().getTypeSizeInChars(T: ElementTy);
1077
1078 llvm::PHINode *SrcElementPHI =
1079 Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
1080 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
1081 Address SrcElementCurrent =
1082 Address(SrcElementPHI, SrcAddr.getElementType(),
1083 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
1084
1085 llvm::PHINode *DestElementPHI = Builder.CreatePHI(
1086 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
1087 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
1088 Address DestElementCurrent =
1089 Address(DestElementPHI, DestAddr.getElementType(),
1090 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
1091
1092 // Emit copy.
1093 CopyGen(DestElementCurrent, SrcElementCurrent);
1094
1095 // Shift the address forward by one element.
1096 llvm::Value *DestElementNext =
1097 Builder.CreateConstGEP1_32(Ty: DestAddr.getElementType(), Ptr: DestElementPHI,
1098 /*Idx0=*/1, Name: "omp.arraycpy.dest.element");
1099 llvm::Value *SrcElementNext =
1100 Builder.CreateConstGEP1_32(Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI,
1101 /*Idx0=*/1, Name: "omp.arraycpy.src.element");
1102 // Check whether we've reached the end.
1103 llvm::Value *Done =
1104 Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
1105 Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
1106 DestElementPHI->addIncoming(V: DestElementNext, BB: Builder.GetInsertBlock());
1107 SrcElementPHI->addIncoming(V: SrcElementNext, BB: Builder.GetInsertBlock());
1108
1109 // Done.
1110 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
1111}
1112
1113void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
1114 Address SrcAddr, const VarDecl *DestVD,
1115 const VarDecl *SrcVD, const Expr *Copy) {
1116 if (OriginalType->isArrayType()) {
1117 const auto *BO = dyn_cast<BinaryOperator>(Val: Copy);
1118 if (BO && BO->getOpcode() == BO_Assign) {
1119 // Perform simple memcpy for simple copying.
1120 LValue Dest = MakeAddrLValue(Addr: DestAddr, T: OriginalType);
1121 LValue Src = MakeAddrLValue(Addr: SrcAddr, T: OriginalType);
1122 EmitAggregateAssign(Dest, Src, EltTy: OriginalType);
1123 } else {
1124 // For arrays with complex element types perform element by element
1125 // copying.
1126 EmitOMPAggregateAssign(
1127 DestAddr, SrcAddr, OriginalType,
1128 CopyGen: [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
1129 // Working with the single array element, so have to remap
1130 // destination and source variables to corresponding array
1131 // elements.
1132 CodeGenFunction::OMPPrivateScope Remap(*this);
1133 Remap.addPrivate(LocalVD: DestVD, Addr: DestElement);
1134 Remap.addPrivate(LocalVD: SrcVD, Addr: SrcElement);
1135 (void)Remap.Privatize();
1136 EmitIgnoredExpr(E: Copy);
1137 });
1138 }
1139 } else {
1140 // Remap pseudo source variable to private copy.
1141 CodeGenFunction::OMPPrivateScope Remap(*this);
1142 Remap.addPrivate(LocalVD: SrcVD, Addr: SrcAddr);
1143 Remap.addPrivate(LocalVD: DestVD, Addr: DestAddr);
1144 (void)Remap.Privatize();
1145 // Emit copying of the whole variable.
1146 EmitIgnoredExpr(E: Copy);
1147 }
1148}
1149
1150bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
1151 OMPPrivateScope &PrivateScope) {
1152 if (!HaveInsertPoint())
1153 return false;
1154 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1155 bool DeviceConstTarget = getLangOpts().OpenMPIsTargetDevice &&
1156 isOpenMPTargetExecutionDirective(DKind: EKind);
1157 bool FirstprivateIsLastprivate = false;
1158 llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
1159 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1160 for (const auto *D : C->varlist())
1161 Lastprivates.try_emplace(
1162 Key: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D)->getDecl())->getCanonicalDecl(),
1163 Args: C->getKind());
1164 }
1165 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
1166 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
1167 getOpenMPCaptureRegions(CaptureRegions, DKind: EKind);
1168 // Force emission of the firstprivate copy if the directive does not emit
1169 // outlined function, like omp for, omp simd, omp distribute etc.
1170 bool MustEmitFirstprivateCopy =
1171 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
1172 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
1173 const auto *IRef = C->varlist_begin();
1174 const auto *InitsRef = C->inits().begin();
1175 for (const Expr *IInit : C->private_copies()) {
1176 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
1177 bool ThisFirstprivateIsLastprivate =
1178 Lastprivates.count(Val: OrigVD->getCanonicalDecl()) > 0;
1179 const FieldDecl *FD = CapturedStmtInfo->lookup(VD: OrigVD);
1180 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1181 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
1182 !FD->getType()->isReferenceType() &&
1183 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1184 EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl());
1185 ++IRef;
1186 ++InitsRef;
1187 continue;
1188 }
1189 // Do not emit copy for firstprivate constant variables in target regions,
1190 // captured by reference.
1191 if (DeviceConstTarget && OrigVD->getType().isConstant(Ctx: getContext()) &&
1192 FD && FD->getType()->isReferenceType() &&
1193 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
1194 EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl());
1195 ++IRef;
1196 ++InitsRef;
1197 continue;
1198 }
1199 FirstprivateIsLastprivate =
1200 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
1201 if (EmittedAsFirstprivate.insert(V: OrigVD->getCanonicalDecl()).second) {
1202 const auto *VDInit =
1203 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *InitsRef)->getDecl());
1204 bool IsRegistered;
1205 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1206 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
1207 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1208 LValue OriginalLVal;
1209 if (!FD) {
1210 // Check if the firstprivate variable is just a constant value.
1211 ConstantEmission CE = tryEmitAsConstant(RefExpr: &DRE);
1212 if (CE && !CE.isReference()) {
1213 // Constant value, no need to create a copy.
1214 ++IRef;
1215 ++InitsRef;
1216 continue;
1217 }
1218 if (CE && CE.isReference()) {
1219 OriginalLVal = CE.getReferenceLValue(CGF&: *this, RefExpr: &DRE);
1220 } else {
1221 assert(!CE && "Expected non-constant firstprivate.");
1222 OriginalLVal = EmitLValue(E: &DRE);
1223 }
1224 } else {
1225 OriginalLVal = EmitLValue(E: &DRE);
1226 }
1227 QualType Type = VD->getType();
1228 if (Type->isArrayType()) {
1229 // Emit VarDecl with copy init for arrays.
1230 // Get the address of the original variable captured in current
1231 // captured region.
1232 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
1233 const Expr *Init = VD->getInit();
1234 if (!isa<CXXConstructExpr>(Val: Init) || isTrivialInitializer(Init)) {
1235 // Perform simple memcpy.
1236 LValue Dest = MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: Type);
1237 EmitAggregateAssign(Dest, Src: OriginalLVal, EltTy: Type);
1238 } else {
1239 EmitOMPAggregateAssign(
1240 DestAddr: Emission.getAllocatedAddress(), SrcAddr: OriginalLVal.getAddress(), OriginalType: Type,
1241 CopyGen: [this, VDInit, Init](Address DestElement, Address SrcElement) {
1242 // Clean up any temporaries needed by the
1243 // initialization.
1244 RunCleanupsScope InitScope(*this);
1245 // Emit initialization for single element.
1246 setAddrOfLocalVar(VD: VDInit, Addr: SrcElement);
1247 EmitAnyExprToMem(E: Init, Location: DestElement,
1248 Quals: Init->getType().getQualifiers(),
1249 /*IsInitializer*/ false);
1250 LocalDeclMap.erase(Val: VDInit);
1251 });
1252 }
1253 EmitAutoVarCleanups(emission: Emission);
1254 IsRegistered =
1255 PrivateScope.addPrivate(LocalVD: OrigVD, Addr: Emission.getAllocatedAddress());
1256 } else {
1257 Address OriginalAddr = OriginalLVal.getAddress();
1258 // Emit private VarDecl with copy init.
1259 // Remap temp VDInit variable to the address of the original
1260 // variable (for proper handling of captured global variables).
1261 setAddrOfLocalVar(VD: VDInit, Addr: OriginalAddr);
1262 EmitDecl(D: *VD);
1263 LocalDeclMap.erase(Val: VDInit);
1264 Address VDAddr = GetAddrOfLocalVar(VD);
1265 if (ThisFirstprivateIsLastprivate &&
1266 Lastprivates[OrigVD->getCanonicalDecl()] ==
1267 OMPC_LASTPRIVATE_conditional) {
1268 // Create/init special variable for lastprivate conditionals.
1269 llvm::Value *V =
1270 EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: VDAddr, T: (*IRef)->getType(),
1271 Source: AlignmentSource::Decl),
1272 Loc: (*IRef)->getExprLoc());
1273 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1274 CGF&: *this, VD: OrigVD);
1275 EmitStoreOfScalar(value: V, lvalue: MakeAddrLValue(Addr: VDAddr, T: (*IRef)->getType(),
1276 Source: AlignmentSource::Decl));
1277 LocalDeclMap.erase(Val: VD);
1278 setAddrOfLocalVar(VD, Addr: VDAddr);
1279 }
1280 IsRegistered = PrivateScope.addPrivate(LocalVD: OrigVD, Addr: VDAddr);
1281 }
1282 assert(IsRegistered &&
1283 "firstprivate var already registered as private");
1284 // Silence the warning about unused variable.
1285 (void)IsRegistered;
1286 }
1287 ++IRef;
1288 ++InitsRef;
1289 }
1290 }
1291 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
1292}
1293
1294void CodeGenFunction::EmitOMPPrivateClause(
1295 const OMPExecutableDirective &D,
1296 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1297 if (!HaveInsertPoint())
1298 return;
1299 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
1300 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
1301 auto IRef = C->varlist_begin();
1302 for (const Expr *IInit : C->private_copies()) {
1303 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
1304 if (EmittedAsPrivate.insert(V: OrigVD->getCanonicalDecl()).second) {
1305 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1306 EmitDecl(D: *VD);
1307 // Emit private VarDecl with copy init.
1308 bool IsRegistered =
1309 PrivateScope.addPrivate(LocalVD: OrigVD, Addr: GetAddrOfLocalVar(VD));
1310 assert(IsRegistered && "private var already registered as private");
1311 // Silence the warning about unused variable.
1312 (void)IsRegistered;
1313 }
1314 ++IRef;
1315 }
1316 }
1317}
1318
1319bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
1320 if (!HaveInsertPoint())
1321 return false;
1322 // threadprivate_var1 = master_threadprivate_var1;
1323 // operator=(threadprivate_var2, master_threadprivate_var2);
1324 // ...
1325 // __kmpc_barrier(&loc, global_tid);
1326 llvm::DenseSet<const VarDecl *> CopiedVars;
1327 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
1328 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
1329 auto IRef = C->varlist_begin();
1330 auto ISrcRef = C->source_exprs().begin();
1331 auto IDestRef = C->destination_exprs().begin();
1332 for (const Expr *AssignOp : C->assignment_ops()) {
1333 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
1334 QualType Type = VD->getType();
1335 if (CopiedVars.insert(V: VD->getCanonicalDecl()).second) {
1336 // Get the address of the master variable. If we are emitting code with
1337 // TLS support, the address is passed from the master as field in the
1338 // captured declaration.
1339 Address MasterAddr = Address::invalid();
1340 if (getLangOpts().OpenMPUseTLS &&
1341 getContext().getTargetInfo().isTLSSupported()) {
1342 assert(CapturedStmtInfo->lookup(VD) &&
1343 "Copyin threadprivates should have been captured!");
1344 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
1345 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1346 MasterAddr = EmitLValue(E: &DRE).getAddress();
1347 LocalDeclMap.erase(Val: VD);
1348 } else {
1349 MasterAddr =
1350 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(D: VD)
1351 : CGM.GetAddrOfGlobal(GD: VD),
1352 CGM.getTypes().ConvertTypeForMem(T: VD->getType()),
1353 getContext().getDeclAlign(D: VD));
1354 }
1355 // Get the address of the threadprivate variable.
1356 Address PrivateAddr = EmitLValue(E: *IRef).getAddress();
1357 if (CopiedVars.size() == 1) {
1358 // At first check if current thread is a master thread. If it is, no
1359 // need to copy data.
1360 CopyBegin = createBasicBlock(name: "copyin.not.master");
1361 CopyEnd = createBasicBlock(name: "copyin.not.master.end");
1362 // TODO: Avoid ptrtoint conversion.
1363 auto *MasterAddrInt = Builder.CreatePtrToInt(
1364 V: MasterAddr.emitRawPointer(CGF&: *this), DestTy: CGM.IntPtrTy);
1365 auto *PrivateAddrInt = Builder.CreatePtrToInt(
1366 V: PrivateAddr.emitRawPointer(CGF&: *this), DestTy: CGM.IntPtrTy);
1367 Builder.CreateCondBr(
1368 Cond: Builder.CreateICmpNE(LHS: MasterAddrInt, RHS: PrivateAddrInt), True: CopyBegin,
1369 False: CopyEnd);
1370 EmitBlock(BB: CopyBegin);
1371 }
1372 const auto *SrcVD =
1373 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1374 const auto *DestVD =
1375 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1376 EmitOMPCopy(OriginalType: Type, DestAddr: PrivateAddr, SrcAddr: MasterAddr, DestVD, SrcVD, Copy: AssignOp);
1377 }
1378 ++IRef;
1379 ++ISrcRef;
1380 ++IDestRef;
1381 }
1382 }
1383 if (CopyEnd) {
1384 // Exit out of copying procedure for non-master thread.
1385 EmitBlock(BB: CopyEnd, /*IsFinished=*/true);
1386 return true;
1387 }
1388 return false;
1389}
1390
1391bool CodeGenFunction::EmitOMPLastprivateClauseInit(
1392 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
1393 if (!HaveInsertPoint())
1394 return false;
1395 bool HasAtLeastOneLastprivate = false;
1396 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1397 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1398 if (isOpenMPSimdDirective(DKind: EKind)) {
1399 const auto *LoopDirective = cast<OMPLoopDirective>(Val: &D);
1400 for (const Expr *C : LoopDirective->counters()) {
1401 SIMDLCVs.insert(
1402 V: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: C)->getDecl())->getCanonicalDecl());
1403 }
1404 }
1405 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1406 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1407 HasAtLeastOneLastprivate = true;
1408 if (isOpenMPTaskLoopDirective(DKind: EKind) && !getLangOpts().OpenMPSimd)
1409 break;
1410 const auto *IRef = C->varlist_begin();
1411 const auto *IDestRef = C->destination_exprs().begin();
1412 for (const Expr *IInit : C->private_copies()) {
1413 // Keep the address of the original variable for future update at the end
1414 // of the loop.
1415 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
1416 // Taskloops do not require additional initialization, it is done in
1417 // runtime support library.
1418 if (AlreadyEmittedVars.insert(V: OrigVD->getCanonicalDecl()).second) {
1419 const auto *DestVD =
1420 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1421 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1422 /*RefersToEnclosingVariableOrCapture=*/
1423 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
1424 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1425 PrivateScope.addPrivate(LocalVD: DestVD, Addr: EmitLValue(E: &DRE).getAddress());
1426 // Check if the variable is also a firstprivate: in this case IInit is
1427 // not generated. Initialization of this variable will happen in codegen
1428 // for 'firstprivate' clause.
1429 if (IInit && !SIMDLCVs.count(V: OrigVD->getCanonicalDecl())) {
1430 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IInit)->getDecl());
1431 Address VDAddr = Address::invalid();
1432 if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
1433 VDAddr = CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
1434 CGF&: *this, VD: OrigVD);
1435 setAddrOfLocalVar(VD, Addr: VDAddr);
1436 } else {
1437 // Emit private VarDecl with copy init.
1438 EmitDecl(D: *VD);
1439 VDAddr = GetAddrOfLocalVar(VD);
1440 }
1441 bool IsRegistered = PrivateScope.addPrivate(LocalVD: OrigVD, Addr: VDAddr);
1442 assert(IsRegistered &&
1443 "lastprivate var already registered as private");
1444 (void)IsRegistered;
1445 }
1446 }
1447 ++IRef;
1448 ++IDestRef;
1449 }
1450 }
1451 return HasAtLeastOneLastprivate;
1452}
1453
1454void CodeGenFunction::EmitOMPLastprivateClauseFinal(
1455 const OMPExecutableDirective &D, bool NoFinals,
1456 llvm::Value *IsLastIterCond) {
1457 if (!HaveInsertPoint())
1458 return;
1459 // Emit following code:
1460 // if (<IsLastIterCond>) {
1461 // orig_var1 = private_orig_var1;
1462 // ...
1463 // orig_varn = private_orig_varn;
1464 // }
1465 llvm::BasicBlock *ThenBB = nullptr;
1466 llvm::BasicBlock *DoneBB = nullptr;
1467 if (IsLastIterCond) {
1468 // Emit implicit barrier if at least one lastprivate conditional is found
1469 // and this is not a simd mode.
1470 if (!getLangOpts().OpenMPSimd &&
1471 llvm::any_of(Range: D.getClausesOfKind<OMPLastprivateClause>(),
1472 P: [](const OMPLastprivateClause *C) {
1473 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1474 })) {
1475 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: D.getBeginLoc(),
1476 Kind: OMPD_unknown,
1477 /*EmitChecks=*/false,
1478 /*ForceSimpleCall=*/true);
1479 }
1480 ThenBB = createBasicBlock(name: ".omp.lastprivate.then");
1481 DoneBB = createBasicBlock(name: ".omp.lastprivate.done");
1482 Builder.CreateCondBr(Cond: IsLastIterCond, True: ThenBB, False: DoneBB);
1483 EmitBlock(BB: ThenBB);
1484 }
1485 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1486 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1487 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(Val: &D)) {
1488 auto IC = LoopDirective->counters().begin();
1489 for (const Expr *F : LoopDirective->finals()) {
1490 const auto *D =
1491 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IC)->getDecl())->getCanonicalDecl();
1492 if (NoFinals)
1493 AlreadyEmittedVars.insert(V: D);
1494 else
1495 LoopCountersAndUpdates[D] = F;
1496 ++IC;
1497 }
1498 }
1499 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1500 auto IRef = C->varlist_begin();
1501 auto ISrcRef = C->source_exprs().begin();
1502 auto IDestRef = C->destination_exprs().begin();
1503 for (const Expr *AssignOp : C->assignment_ops()) {
1504 const auto *PrivateVD =
1505 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
1506 QualType Type = PrivateVD->getType();
1507 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1508 if (AlreadyEmittedVars.insert(V: CanonicalVD).second) {
1509 // If lastprivate variable is a loop control variable for loop-based
1510 // directive, update its value before copyin back to original
1511 // variable.
1512 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(Val: CanonicalVD))
1513 EmitIgnoredExpr(E: FinalExpr);
1514 const auto *SrcVD =
1515 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ISrcRef)->getDecl());
1516 const auto *DestVD =
1517 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IDestRef)->getDecl());
1518 // Get the address of the private variable.
1519 Address PrivateAddr = GetAddrOfLocalVar(VD: PrivateVD);
1520 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1521 PrivateAddr = Address(
1522 Builder.CreateLoad(Addr: PrivateAddr),
1523 CGM.getTypes().ConvertTypeForMem(T: RefTy->getPointeeType()),
1524 CGM.getNaturalTypeAlignment(T: RefTy->getPointeeType()));
1525 // Store the last value to the private copy in the last iteration.
1526 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1527 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1528 CGF&: *this, PrivLVal: MakeAddrLValue(Addr: PrivateAddr, T: (*IRef)->getType()), VD: PrivateVD,
1529 Loc: (*IRef)->getExprLoc());
1530 // Get the address of the original variable.
1531 Address OriginalAddr = GetAddrOfLocalVar(VD: DestVD);
1532 EmitOMPCopy(OriginalType: Type, DestAddr: OriginalAddr, SrcAddr: PrivateAddr, DestVD, SrcVD, Copy: AssignOp);
1533 }
1534 ++IRef;
1535 ++ISrcRef;
1536 ++IDestRef;
1537 }
1538 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1539 EmitIgnoredExpr(E: PostUpdate);
1540 }
1541 if (IsLastIterCond)
1542 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
1543}
1544
1545void CodeGenFunction::EmitOMPReductionClauseInit(
1546 const OMPExecutableDirective &D,
1547 CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
1548 if (!HaveInsertPoint())
1549 return;
1550 SmallVector<const Expr *, 4> Shareds;
1551 SmallVector<const Expr *, 4> Privates;
1552 SmallVector<const Expr *, 4> ReductionOps;
1553 SmallVector<const Expr *, 4> LHSs;
1554 SmallVector<const Expr *, 4> RHSs;
1555 OMPTaskDataTy Data;
1556 SmallVector<const Expr *, 4> TaskLHSs;
1557 SmallVector<const Expr *, 4> TaskRHSs;
1558 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1559 if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
1560 continue;
1561 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
1562 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
1563 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
1564 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1565 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1566 if (C->getModifier() == OMPC_REDUCTION_task) {
1567 Data.ReductionVars.append(in_start: C->privates().begin(), in_end: C->privates().end());
1568 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
1569 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
1570 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
1571 in_end: C->reduction_ops().end());
1572 TaskLHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1573 TaskRHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1574 }
1575 }
1576 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
1577 unsigned Count = 0;
1578 auto *ILHS = LHSs.begin();
1579 auto *IRHS = RHSs.begin();
1580 auto *IPriv = Privates.begin();
1581 for (const Expr *IRef : Shareds) {
1582 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IPriv)->getDecl());
1583 // Emit private VarDecl with reduction init.
1584 RedCG.emitSharedOrigLValue(CGF&: *this, N: Count);
1585 RedCG.emitAggregateType(CGF&: *this, N: Count);
1586 AutoVarEmission Emission = EmitAutoVarAlloca(var: *PrivateVD);
1587 RedCG.emitInitialization(CGF&: *this, N: Count, PrivateAddr: Emission.getAllocatedAddress(),
1588 SharedAddr: RedCG.getSharedLValue(N: Count).getAddress(),
1589 DefaultInit: [&Emission](CodeGenFunction &CGF) {
1590 CGF.EmitAutoVarInit(emission: Emission);
1591 return true;
1592 });
1593 EmitAutoVarCleanups(emission: Emission);
1594 Address BaseAddr = RedCG.adjustPrivateAddress(
1595 CGF&: *this, N: Count, PrivateAddr: Emission.getAllocatedAddress());
1596 bool IsRegistered =
1597 PrivateScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Count), Addr: BaseAddr);
1598 assert(IsRegistered && "private var already registered as private");
1599 // Silence the warning about unused variable.
1600 (void)IsRegistered;
1601
1602 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
1603 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
1604 QualType Type = PrivateVD->getType();
1605 bool isaOMPArraySectionExpr = isa<ArraySectionExpr>(Val: IRef);
1606 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1607 // Store the address of the original variable associated with the LHS
1608 // implicit variable.
1609 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: RedCG.getSharedLValue(N: Count).getAddress());
1610 PrivateScope.addPrivate(LocalVD: RHSVD, Addr: GetAddrOfLocalVar(VD: PrivateVD));
1611 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1612 isa<ArraySubscriptExpr>(Val: IRef)) {
1613 // Store the address of the original variable associated with the LHS
1614 // implicit variable.
1615 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: RedCG.getSharedLValue(N: Count).getAddress());
1616 PrivateScope.addPrivate(LocalVD: RHSVD,
1617 Addr: GetAddrOfLocalVar(VD: PrivateVD).withElementType(
1618 ElemTy: ConvertTypeForMem(T: RHSVD->getType())));
1619 } else {
1620 QualType Type = PrivateVD->getType();
1621 bool IsArray = getContext().getAsArrayType(T: Type) != nullptr;
1622 Address OriginalAddr = RedCG.getSharedLValue(N: Count).getAddress();
1623 // Store the address of the original variable associated with the LHS
1624 // implicit variable.
1625 if (IsArray) {
1626 OriginalAddr =
1627 OriginalAddr.withElementType(ElemTy: ConvertTypeForMem(T: LHSVD->getType()));
1628 }
1629 PrivateScope.addPrivate(LocalVD: LHSVD, Addr: OriginalAddr);
1630 PrivateScope.addPrivate(
1631 LocalVD: RHSVD, Addr: IsArray ? GetAddrOfLocalVar(VD: PrivateVD).withElementType(
1632 ElemTy: ConvertTypeForMem(T: RHSVD->getType()))
1633 : GetAddrOfLocalVar(VD: PrivateVD));
1634 }
1635 ++ILHS;
1636 ++IRHS;
1637 ++IPriv;
1638 ++Count;
1639 }
1640 if (!Data.ReductionVars.empty()) {
1641 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1642 Data.IsReductionWithTaskMod = true;
1643 Data.IsWorksharingReduction = isOpenMPWorksharingDirective(DKind: EKind);
1644 llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
1645 CGF&: *this, Loc: D.getBeginLoc(), LHSExprs: TaskLHSs, RHSExprs: TaskRHSs, Data);
1646 const Expr *TaskRedRef = nullptr;
1647 switch (EKind) {
1648 case OMPD_parallel:
1649 TaskRedRef = cast<OMPParallelDirective>(Val: D).getTaskReductionRefExpr();
1650 break;
1651 case OMPD_for:
1652 TaskRedRef = cast<OMPForDirective>(Val: D).getTaskReductionRefExpr();
1653 break;
1654 case OMPD_sections:
1655 TaskRedRef = cast<OMPSectionsDirective>(Val: D).getTaskReductionRefExpr();
1656 break;
1657 case OMPD_parallel_for:
1658 TaskRedRef = cast<OMPParallelForDirective>(Val: D).getTaskReductionRefExpr();
1659 break;
1660 case OMPD_parallel_master:
1661 TaskRedRef =
1662 cast<OMPParallelMasterDirective>(Val: D).getTaskReductionRefExpr();
1663 break;
1664 case OMPD_parallel_sections:
1665 TaskRedRef =
1666 cast<OMPParallelSectionsDirective>(Val: D).getTaskReductionRefExpr();
1667 break;
1668 case OMPD_target_parallel:
1669 TaskRedRef =
1670 cast<OMPTargetParallelDirective>(Val: D).getTaskReductionRefExpr();
1671 break;
1672 case OMPD_target_parallel_for:
1673 TaskRedRef =
1674 cast<OMPTargetParallelForDirective>(Val: D).getTaskReductionRefExpr();
1675 break;
1676 case OMPD_distribute_parallel_for:
1677 TaskRedRef =
1678 cast<OMPDistributeParallelForDirective>(Val: D).getTaskReductionRefExpr();
1679 break;
1680 case OMPD_teams_distribute_parallel_for:
1681 TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(Val: D)
1682 .getTaskReductionRefExpr();
1683 break;
1684 case OMPD_target_teams_distribute_parallel_for:
1685 TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(Val: D)
1686 .getTaskReductionRefExpr();
1687 break;
1688 case OMPD_simd:
1689 case OMPD_for_simd:
1690 case OMPD_section:
1691 case OMPD_single:
1692 case OMPD_master:
1693 case OMPD_critical:
1694 case OMPD_parallel_for_simd:
1695 case OMPD_task:
1696 case OMPD_taskyield:
1697 case OMPD_error:
1698 case OMPD_barrier:
1699 case OMPD_taskwait:
1700 case OMPD_taskgroup:
1701 case OMPD_flush:
1702 case OMPD_depobj:
1703 case OMPD_scan:
1704 case OMPD_ordered_standalone:
1705 case OMPD_ordered_blockassoc:
1706 case OMPD_atomic:
1707 case OMPD_teams:
1708 case OMPD_target:
1709 case OMPD_cancellation_point:
1710 case OMPD_cancel:
1711 case OMPD_target_data:
1712 case OMPD_target_enter_data:
1713 case OMPD_target_exit_data:
1714 case OMPD_taskloop:
1715 case OMPD_taskloop_simd:
1716 case OMPD_master_taskloop:
1717 case OMPD_master_taskloop_simd:
1718 case OMPD_parallel_master_taskloop:
1719 case OMPD_parallel_master_taskloop_simd:
1720 case OMPD_distribute:
1721 case OMPD_target_update:
1722 case OMPD_distribute_parallel_for_simd:
1723 case OMPD_distribute_simd:
1724 case OMPD_target_parallel_for_simd:
1725 case OMPD_target_simd:
1726 case OMPD_teams_distribute:
1727 case OMPD_teams_distribute_simd:
1728 case OMPD_teams_distribute_parallel_for_simd:
1729 case OMPD_target_teams:
1730 case OMPD_target_teams_distribute:
1731 case OMPD_target_teams_distribute_parallel_for_simd:
1732 case OMPD_target_teams_distribute_simd:
1733 case OMPD_declare_target:
1734 case OMPD_end_declare_target:
1735 case OMPD_threadprivate:
1736 case OMPD_allocate:
1737 case OMPD_declare_reduction:
1738 case OMPD_declare_mapper:
1739 case OMPD_declare_simd:
1740 case OMPD_requires:
1741 case OMPD_declare_variant:
1742 case OMPD_begin_declare_variant:
1743 case OMPD_end_declare_variant:
1744 case OMPD_unknown:
1745 default:
1746 llvm_unreachable("Unexpected directive with task reductions.");
1747 }
1748
1749 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: TaskRedRef)->getDecl());
1750 EmitVarDecl(D: *VD);
1751 EmitStoreOfScalar(Value: ReductionDesc, Addr: GetAddrOfLocalVar(VD),
1752 /*Volatile=*/false, Ty: TaskRedRef->getType());
1753 }
1754}
1755
1756void CodeGenFunction::EmitOMPReductionClauseFinal(
1757 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1758 if (!HaveInsertPoint())
1759 return;
1760 llvm::SmallVector<const Expr *, 8> Privates;
1761 llvm::SmallVector<const Expr *, 8> LHSExprs;
1762 llvm::SmallVector<const Expr *, 8> RHSExprs;
1763 llvm::SmallVector<const Expr *, 8> ReductionOps;
1764 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1765 bool HasAtLeastOneReduction = false;
1766 bool IsReductionWithTaskMod = false;
1767 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1768 // Do not emit for inscan reductions.
1769 if (C->getModifier() == OMPC_REDUCTION_inscan)
1770 continue;
1771 HasAtLeastOneReduction = true;
1772 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
1773 LHSExprs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
1774 RHSExprs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
1775 IsPrivateVarReduction.append(in_start: C->private_var_reduction_flags().begin(),
1776 in_end: C->private_var_reduction_flags().end());
1777 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
1778 IsReductionWithTaskMod =
1779 IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
1780 }
1781 if (HasAtLeastOneReduction) {
1782 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
1783 if (IsReductionWithTaskMod) {
1784 CGM.getOpenMPRuntime().emitTaskReductionFini(
1785 CGF&: *this, Loc: D.getBeginLoc(), IsWorksharingReduction: isOpenMPWorksharingDirective(DKind: EKind));
1786 }
1787 bool TeamsLoopCanBeParallel = false;
1788 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(Val: &D))
1789 TeamsLoopCanBeParallel = TTLD->canBeParallelFor();
1790 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1791 isOpenMPParallelDirective(DKind: EKind) ||
1792 TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;
1793 bool SimpleReduction = ReductionKind == OMPD_simd;
1794 // Emit nowait reduction if nowait clause is present or directive is a
1795 // parallel directive (it always has implicit barrier).
1796 CGM.getOpenMPRuntime().emitReduction(
1797 CGF&: *this, Loc: D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1798 Options: {.WithNowait: WithNowait, .SimpleReduction: SimpleReduction, .IsPrivateVarReduction: IsPrivateVarReduction, .ReductionKind: ReductionKind});
1799 }
1800}
1801
1802static void emitPostUpdateForReductionClause(
1803 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1804 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1805 if (!CGF.HaveInsertPoint())
1806 return;
1807 llvm::BasicBlock *DoneBB = nullptr;
1808 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1809 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1810 if (!DoneBB) {
1811 if (llvm::Value *Cond = CondGen(CGF)) {
1812 // If the first post-update expression is found, emit conditional
1813 // block if it was requested.
1814 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: ".omp.reduction.pu");
1815 DoneBB = CGF.createBasicBlock(name: ".omp.reduction.pu.done");
1816 CGF.Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
1817 CGF.EmitBlock(BB: ThenBB);
1818 }
1819 }
1820 CGF.EmitIgnoredExpr(E: PostUpdate);
1821 }
1822 }
1823 if (DoneBB)
1824 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
1825}
1826
1827namespace {
1828/// Codegen lambda for appending distribute lower and upper bounds to outlined
1829/// parallel function. This is necessary for combined constructs such as
1830/// 'distribute parallel for'
1831typedef llvm::function_ref<void(CodeGenFunction &,
1832 const OMPExecutableDirective &,
1833 llvm::SmallVectorImpl<llvm::Value *> &)>
1834 CodeGenBoundParametersTy;
1835} // anonymous namespace
1836
1837static void
1838checkForLastprivateConditionalUpdate(CodeGenFunction &CGF,
1839 const OMPExecutableDirective &S) {
1840 if (CGF.getLangOpts().OpenMP < 50)
1841 return;
1842 llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
1843 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
1844 for (const Expr *Ref : C->varlist()) {
1845 if (!Ref->getType()->isScalarType())
1846 continue;
1847 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
1848 if (!DRE)
1849 continue;
1850 PrivateDecls.insert(V: cast<VarDecl>(Val: DRE->getDecl()));
1851 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
1852 }
1853 }
1854 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
1855 for (const Expr *Ref : C->varlist()) {
1856 if (!Ref->getType()->isScalarType())
1857 continue;
1858 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
1859 if (!DRE)
1860 continue;
1861 PrivateDecls.insert(V: cast<VarDecl>(Val: DRE->getDecl()));
1862 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
1863 }
1864 }
1865 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
1866 for (const Expr *Ref : C->varlist()) {
1867 if (!Ref->getType()->isScalarType())
1868 continue;
1869 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
1870 if (!DRE)
1871 continue;
1872 PrivateDecls.insert(V: cast<VarDecl>(Val: DRE->getDecl()));
1873 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: Ref);
1874 }
1875 }
1876 // Privates should ne analyzed since they are not captured at all.
1877 // Task reductions may be skipped - tasks are ignored.
1878 // Firstprivates do not return value but may be passed by reference - no need
1879 // to check for updated lastprivate conditional.
1880 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
1881 for (const Expr *Ref : C->varlist()) {
1882 if (!Ref->getType()->isScalarType())
1883 continue;
1884 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
1885 if (!DRE)
1886 continue;
1887 PrivateDecls.insert(V: cast<VarDecl>(Val: DRE->getDecl()));
1888 }
1889 }
1890 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional(
1891 CGF, D: S, IgnoredDecls: PrivateDecls);
1892}
1893
1894static void emitCommonOMPParallelDirective(
1895 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1896 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1897 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1898 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
1899 llvm::Value *NumThreads = nullptr;
1900 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown;
1901 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is as
1902 // if sev-level is fatal."
1903 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal;
1904 clang::Expr *Message = nullptr;
1905 SourceLocation SeverityLoc = SourceLocation();
1906 SourceLocation MessageLoc = SourceLocation();
1907
1908 llvm::Function *OutlinedFn =
1909 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1910 CGF, D: S, ThreadIDVar: *CS->getCapturedDecl()->param_begin(), InnermostKind,
1911 CodeGen);
1912
1913 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1914 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1915 NumThreads = CGF.EmitScalarExpr(E: NumThreadsClause->getNumThreads().front(),
1916 /*IgnoreResultAssign=*/true);
1917 Modifier = NumThreadsClause->getPrescriptivenessModifier();
1918 if (const auto *MessageClause = S.getSingleClause<OMPMessageClause>()) {
1919 Message = MessageClause->getMessageString();
1920 MessageLoc = MessageClause->getBeginLoc();
1921 }
1922 if (const auto *SeverityClause = S.getSingleClause<OMPSeverityClause>()) {
1923 Severity = SeverityClause->getSeverityKind();
1924 SeverityLoc = SeverityClause->getBeginLoc();
1925 }
1926 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1927 CGF, NumThreads, Loc: NumThreadsClause->getBeginLoc(), Modifier, Severity,
1928 SeverityLoc, Message, MessageLoc);
1929 }
1930 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1931 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1932 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1933 CGF, ProcBind: ProcBindClause->getProcBindKind(), Loc: ProcBindClause->getBeginLoc());
1934 }
1935 const Expr *IfCond = nullptr;
1936 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1937 if (C->getNameModifier() == OMPD_unknown ||
1938 C->getNameModifier() == OMPD_parallel) {
1939 IfCond = C->getCondition();
1940 break;
1941 }
1942 }
1943
1944 OMPParallelScope Scope(CGF, S);
1945 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
1946 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1947 // lower and upper bounds with the pragma 'for' chunking mechanism.
1948 // The following lambda takes care of appending the lower and upper bound
1949 // parameters when necessary
1950 CodeGenBoundParameters(CGF, S, CapturedVars);
1951 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
1952 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, Loc: S.getBeginLoc(), OutlinedFn,
1953 CapturedVars, IfCond, NumThreads,
1954 NumThreadsModifier: Modifier, Severity, Message);
1955}
1956
1957static bool isAllocatableDecl(const VarDecl *VD) {
1958 const VarDecl *CVD = VD->getCanonicalDecl();
1959 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
1960 return false;
1961 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
1962 // Use the default allocation.
1963 return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
1964 AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
1965 !AA->getAllocator());
1966}
1967
1968static void emitEmptyBoundParameters(CodeGenFunction &,
1969 const OMPExecutableDirective &,
1970 llvm::SmallVectorImpl<llvm::Value *> &) {}
1971
1972static void emitOMPCopyinClause(CodeGenFunction &CGF,
1973 const OMPExecutableDirective &S) {
1974 bool Copyins = CGF.EmitOMPCopyinClause(D: S);
1975 if (Copyins) {
1976 // Emit implicit barrier to synchronize threads and avoid data races on
1977 // propagation master's thread values of threadprivate variables to local
1978 // instances of that variables of all other implicit threads.
1979 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1980 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
1981 /*ForceSimpleCall=*/true);
1982 }
1983}
1984
1985Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable(
1986 CodeGenFunction &CGF, const VarDecl *VD) {
1987 CodeGenModule &CGM = CGF.CGM;
1988 auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1989
1990 if (!VD)
1991 return Address::invalid();
1992 const VarDecl *CVD = VD->getCanonicalDecl();
1993 if (!isAllocatableDecl(VD: CVD))
1994 return Address::invalid();
1995 llvm::Value *Size;
1996 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
1997 if (CVD->getType()->isVariablyModifiedType()) {
1998 Size = CGF.getTypeSize(Ty: CVD->getType());
1999 // Align the size: ((size + align - 1) / align) * align
2000 Size = CGF.Builder.CreateNUWAdd(
2001 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
2002 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
2003 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
2004 } else {
2005 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
2006 Size = CGM.getSize(numChars: Sz.alignTo(Align));
2007 }
2008
2009 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
2010 assert(AA->getAllocator() &&
2011 "Expected allocator expression for non-default allocator.");
2012 llvm::Value *Allocator = CGF.EmitScalarExpr(E: AA->getAllocator());
2013 // According to the standard, the original allocator type is a enum (integer).
2014 // Convert to pointer type, if required.
2015 if (Allocator->getType()->isIntegerTy())
2016 Allocator = CGF.Builder.CreateIntToPtr(V: Allocator, DestTy: CGM.VoidPtrTy);
2017 else if (Allocator->getType()->isPointerTy())
2018 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: Allocator,
2019 DestTy: CGM.VoidPtrTy);
2020
2021 llvm::Value *Addr = OMPBuilder.createOMPAlloc(
2022 Loc: CGF.Builder, Size, Allocator,
2023 Name: getNameWithSeparators(Parts: {CVD->getName(), ".void.addr"}, FirstSeparator: ".", Separator: "."));
2024 llvm::CallInst *FreeCI =
2025 OMPBuilder.createOMPFree(Loc: CGF.Builder, Addr, Allocator);
2026
2027 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(Kind: NormalAndEHCleanup, A: FreeCI);
2028 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2029 V: Addr,
2030 DestTy: CGF.ConvertTypeForMem(T: CGM.getContext().getPointerType(T: CVD->getType())),
2031 Name: getNameWithSeparators(Parts: {CVD->getName(), ".addr"}, FirstSeparator: ".", Separator: "."));
2032 return Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
2033}
2034
2035Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2036 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
2037 SourceLocation Loc) {
2038 CodeGenModule &CGM = CGF.CGM;
2039 if (CGM.getLangOpts().OpenMPUseTLS &&
2040 CGM.getContext().getTargetInfo().isTLSSupported())
2041 return VDAddr;
2042
2043 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2044
2045 llvm::Type *VarTy = VDAddr.getElementType();
2046 llvm::Value *Data =
2047 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy);
2048 llvm::ConstantInt *Size = CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy));
2049 std::string Suffix = getNameWithSeparators(Parts: {"cache", ""});
2050 llvm::Twine CacheName = Twine(CGM.getMangledName(GD: VD)).concat(Suffix);
2051
2052 llvm::CallInst *ThreadPrivateCacheCall =
2053 OMPBuilder.createCachedThreadPrivate(Loc: CGF.Builder, Pointer: Data, Size, Name: CacheName);
2054
2055 return Address(ThreadPrivateCacheCall, CGM.Int8Ty, VDAddr.getAlignment());
2056}
2057
2058std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(
2059 ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
2060 SmallString<128> Buffer;
2061 llvm::raw_svector_ostream OS(Buffer);
2062 StringRef Sep = FirstSeparator;
2063 for (StringRef Part : Parts) {
2064 OS << Sep << Part;
2065 Sep = Separator;
2066 }
2067 return OS.str().str();
2068}
2069
2070void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
2071 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2072 InsertPointTy CodeGenIP, Twine RegionName) {
2073 CGBuilderTy &Builder = CGF.Builder;
2074 Builder.restoreIP(IP: CodeGenIP);
2075 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2076 Suffix: "." + RegionName + ".after");
2077
2078 {
2079 OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2080 CGF.EmitStmt(S: RegionBodyStmt);
2081 }
2082
2083 if (Builder.saveIP().isSet())
2084 Builder.CreateBr(Dest: FiniBB);
2085}
2086
2087void CodeGenFunction::OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
2088 CodeGenFunction &CGF, const Stmt *RegionBodyStmt, InsertPointTy AllocaIP,
2089 InsertPointTy CodeGenIP, Twine RegionName) {
2090 CGBuilderTy &Builder = CGF.Builder;
2091 Builder.restoreIP(IP: CodeGenIP);
2092 llvm::BasicBlock *FiniBB = splitBBWithSuffix(Builder, /*CreateBranch=*/false,
2093 Suffix: "." + RegionName + ".after");
2094
2095 {
2096 OMPBuilderCBHelpers::OutlinedRegionBodyRAII IRB(CGF, AllocaIP, *FiniBB);
2097 CGF.EmitStmt(S: RegionBodyStmt);
2098 }
2099
2100 if (Builder.saveIP().isSet())
2101 Builder.CreateBr(Dest: FiniBB);
2102}
2103
2104void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
2105 if (CGM.getLangOpts().OpenMPIRBuilder) {
2106 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2107 // Check if we have any if clause associated with the directive.
2108 llvm::Value *IfCond = nullptr;
2109 if (const auto *C = S.getSingleClause<OMPIfClause>())
2110 IfCond = EmitScalarExpr(E: C->getCondition(),
2111 /*IgnoreResultAssign=*/true);
2112
2113 llvm::Value *NumThreads = nullptr;
2114 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
2115 NumThreads = EmitScalarExpr(E: NumThreadsClause->getNumThreads().front(),
2116 /*IgnoreResultAssign=*/true);
2117
2118 ProcBindKind ProcBind = OMP_PROC_BIND_default;
2119 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
2120 ProcBind = ProcBindClause->getProcBindKind();
2121
2122 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2123
2124 // The cleanup callback that finalizes all variables at the given location,
2125 // thus calls destructors etc.
2126 auto FiniCB = [this](InsertPointTy IP) {
2127 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
2128 return llvm::Error::success();
2129 };
2130
2131 // Privatization callback that performs appropriate action for
2132 // shared/private/firstprivate/lastprivate/copyin/... variables.
2133 //
2134 // TODO: This defaults to shared right now.
2135 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
2136 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
2137 // The next line is appropriate only for variables (Val) with the
2138 // data-sharing attribute "shared".
2139 ReplVal = &Val;
2140
2141 return CodeGenIP;
2142 };
2143
2144 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
2145 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
2146
2147 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
2148 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
2149 OMPBuilderCBHelpers::EmitOMPOutlinedRegionBody(
2150 CGF&: *this, RegionBodyStmt: ParallelRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "parallel");
2151 return llvm::Error::success();
2152 };
2153
2154 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
2155 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
2156 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
2157 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
2158 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2159 cantFail(ValOrErr: OMPBuilder.createParallel(
2160 Loc: Builder, AllocaIP, /*DeallocBlocks=*/{}, BodyGenCB, PrivCB, FiniCB,
2161 IfCondition: IfCond, NumThreads, ProcBind, IsCancellable: S.hasCancel()));
2162 Builder.restoreIP(IP: AfterIP);
2163 return;
2164 }
2165
2166 // Emit parallel region as a standalone region.
2167 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2168 Action.Enter(CGF);
2169 OMPPrivateScope PrivateScope(CGF);
2170 emitOMPCopyinClause(CGF, S);
2171 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
2172 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
2173 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
2174 (void)PrivateScope.Privatize();
2175 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_parallel)->getCapturedStmt());
2176 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
2177 };
2178 {
2179 auto LPCRegion =
2180 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
2181 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_parallel, CodeGen,
2182 CodeGenBoundParameters: emitEmptyBoundParameters);
2183 emitPostUpdateForReductionClause(CGF&: *this, D: S,
2184 CondGen: [](CodeGenFunction &) { return nullptr; });
2185 }
2186 // Check for outer lastprivate conditional update.
2187 checkForLastprivateConditionalUpdate(CGF&: *this, S);
2188}
2189
2190void CodeGenFunction::EmitOMPMetaDirective(const OMPMetaDirective &S) {
2191 EmitStmt(S: S.getIfStmt());
2192}
2193
2194namespace {
2195/// RAII to handle scopes for loop transformation directives.
2196class OMPTransformDirectiveScopeRAII {
2197 OMPLoopScope *Scope = nullptr;
2198 CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
2199 CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
2200
2201 OMPTransformDirectiveScopeRAII(const OMPTransformDirectiveScopeRAII &) =
2202 delete;
2203 OMPTransformDirectiveScopeRAII &
2204 operator=(const OMPTransformDirectiveScopeRAII &) = delete;
2205
2206public:
2207 OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
2208 if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(Val: S)) {
2209 Scope = new OMPLoopScope(CGF, *Dir);
2210 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
2211 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2212 } else if (const auto *Dir =
2213 dyn_cast<OMPCanonicalLoopSequenceTransformationDirective>(
2214 Val: S)) {
2215 // For simplicity we reuse the loop scope similarly to what we do with
2216 // OMPCanonicalLoopNestTransformationDirective do by being a subclass
2217 // of OMPLoopBasedDirective.
2218 Scope = new OMPLoopScope(CGF, *Dir);
2219 CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
2220 CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
2221 }
2222 }
2223 ~OMPTransformDirectiveScopeRAII() {
2224 if (!Scope)
2225 return;
2226 delete CapInfoRAII;
2227 delete CGSI;
2228 delete Scope;
2229 }
2230};
2231} // namespace
2232
2233static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
2234 int MaxLevel, int Level = 0) {
2235 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
2236 const Stmt *SimplifiedS = S->IgnoreContainers();
2237 if (const auto *CS = dyn_cast<CompoundStmt>(Val: SimplifiedS)) {
2238 PrettyStackTraceLoc CrashInfo(
2239 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
2240 "LLVM IR generation of compound statement ('{}')");
2241
2242 // Keep track of the current cleanup stack depth, including debug scopes.
2243 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
2244 for (const Stmt *CurStmt : CS->body())
2245 emitBody(CGF, S: CurStmt, NextLoop, MaxLevel, Level);
2246 return;
2247 }
2248
2249 // `tryToFindNextInnerLoop` keeps the intra-tile hint wrapper around, so match
2250 // against the loop it annotates. The tile overshoot guard is emitted
2251 // separately via EmitOMPLoopBody's finals-conditions handling.
2252 if (SimplifiedS == OMPLoopBasedDirective::ignoreIntraTileHint(S: NextLoop)) {
2253 if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(Val: SimplifiedS))
2254 SimplifiedS = Dir->getTransformedStmt();
2255 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Val: SimplifiedS))
2256 SimplifiedS = CanonLoop->getLoopStmt();
2257 if (const auto *For = dyn_cast<ForStmt>(Val: SimplifiedS)) {
2258 S = For->getBody();
2259 } else {
2260 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
2261 "Expected canonical for loop or range-based for loop.");
2262 const auto *CXXFor = cast<CXXForRangeStmt>(Val: SimplifiedS);
2263 CGF.EmitStmt(S: CXXFor->getLoopVarStmt());
2264 S = CXXFor->getBody();
2265 }
2266 if (Level + 1 < MaxLevel) {
2267 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
2268 CurStmt: S, /*TryImperfectlyNestedLoops=*/true);
2269 emitBody(CGF, S, NextLoop, MaxLevel, Level: Level + 1);
2270 return;
2271 }
2272 }
2273 CGF.EmitStmt(S);
2274}
2275
2276void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
2277 JumpDest LoopExit) {
2278 RunCleanupsScope BodyScope(*this);
2279 // Update counters values on current iteration.
2280 for (const Expr *UE : D.updates())
2281 EmitIgnoredExpr(E: UE);
2282 // Update the linear variables.
2283 // In distribute directives only loop counters may be marked as linear, no
2284 // need to generate the code for them.
2285 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2286 if (!isOpenMPDistributeDirective(DKind: EKind)) {
2287 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2288 for (const Expr *UE : C->updates())
2289 EmitIgnoredExpr(E: UE);
2290 }
2291 }
2292
2293 // On a continue in the body, jump to the end.
2294 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.body.continue");
2295 BreakContinueStack.push_back(Elt: BreakContinue(D, LoopExit, Continue));
2296 for (const Expr *E : D.finals_conditions()) {
2297 if (!E)
2298 continue;
2299 // Check that loop counter in non-rectangular nest fits into the iteration
2300 // space.
2301 llvm::BasicBlock *NextBB = createBasicBlock(name: "omp.body.next");
2302 EmitBranchOnBoolExpr(Cond: E, TrueBlock: NextBB, FalseBlock: Continue.getBlock(),
2303 TrueCount: getProfileCount(S: D.getBody()));
2304 EmitBlock(BB: NextBB);
2305 }
2306
2307 OMPPrivateScope InscanScope(*this);
2308 EmitOMPReductionClauseInit(D, PrivateScope&: InscanScope, /*ForInscan=*/true);
2309 bool IsInscanRegion = InscanScope.Privatize();
2310 if (IsInscanRegion) {
2311 // Need to remember the block before and after scan directive
2312 // to dispatch them correctly depending on the clause used in
2313 // this directive, inclusive or exclusive. For inclusive scan the natural
2314 // order of the blocks is used, for exclusive clause the blocks must be
2315 // executed in reverse order.
2316 OMPBeforeScanBlock = createBasicBlock(name: "omp.before.scan.bb");
2317 OMPAfterScanBlock = createBasicBlock(name: "omp.after.scan.bb");
2318 // No need to allocate inscan exit block, in simd mode it is selected in the
2319 // codegen for the scan directive.
2320 if (EKind != OMPD_simd && !getLangOpts().OpenMPSimd)
2321 OMPScanExitBlock = createBasicBlock(name: "omp.exit.inscan.bb");
2322 OMPScanDispatch = createBasicBlock(name: "omp.inscan.dispatch");
2323 EmitBranch(Block: OMPScanDispatch);
2324 EmitBlock(BB: OMPBeforeScanBlock);
2325 }
2326
2327 // Emit loop variables for C++ range loops.
2328 const Stmt *Body =
2329 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
2330 // Emit loop body.
2331 emitBody(CGF&: *this, S: Body,
2332 NextLoop: OMPLoopBasedDirective::tryToFindNextInnerLoop(
2333 CurStmt: Body, /*TryImperfectlyNestedLoops=*/true),
2334 MaxLevel: D.getLoopsNumber());
2335
2336 // Jump to the dispatcher at the end of the loop body.
2337 if (IsInscanRegion)
2338 EmitBranch(Block: OMPScanExitBlock);
2339
2340 // The end (updates/cleanups).
2341 EmitBlock(BB: Continue.getBlock());
2342 BreakContinueStack.pop_back();
2343}
2344
2345using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
2346
2347/// Emit a captured statement and return the function as well as its captured
2348/// closure context.
2349static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF,
2350 const CapturedStmt *S) {
2351 LValue CapStruct = ParentCGF.InitCapturedStruct(S: *S);
2352 CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
2353 std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
2354 std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(args: *S);
2355 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
2356 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S: *S);
2357
2358 return {F, CapStruct.getPointer(CGF&: ParentCGF)};
2359}
2360
2361/// Emit a call to a previously captured closure.
2362static llvm::CallInst *
2363emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap,
2364 llvm::ArrayRef<llvm::Value *> Args) {
2365 // Append the closure context to the argument.
2366 SmallVector<llvm::Value *> EffectiveArgs;
2367 EffectiveArgs.reserve(N: Args.size() + 1);
2368 llvm::append_range(C&: EffectiveArgs, R&: Args);
2369 EffectiveArgs.push_back(Elt: Cap.second);
2370
2371 return ParentCGF.Builder.CreateCall(Callee: Cap.first, Args: EffectiveArgs);
2372}
2373
2374llvm::CanonicalLoopInfo *
2375CodeGenFunction::EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth) {
2376 assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
2377
2378 // The caller is processing the loop-associated directive processing the \p
2379 // Depth loops nested in \p S. Put the previous pending loop-associated
2380 // directive to the stack. If the current loop-associated directive is a loop
2381 // transformation directive, it will push its generated loops onto the stack
2382 // such that together with the loops left here they form the combined loop
2383 // nest for the parent loop-associated directive.
2384 int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
2385 ExpectedOMPLoopDepth = Depth;
2386
2387 EmitStmt(S);
2388 assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
2389
2390 // The last added loop is the outermost one.
2391 llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
2392
2393 // Pop the \p Depth loops requested by the call from that stack and restore
2394 // the previous context.
2395 OMPLoopNestStack.pop_back_n(NumItems: Depth);
2396 ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
2397
2398 return Result;
2399}
2400
2401void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
2402 const Stmt *SyntacticalLoop = S->getLoopStmt();
2403 if (!getLangOpts().OpenMPIRBuilder) {
2404 // Ignore if OpenMPIRBuilder is not enabled.
2405 EmitStmt(S: SyntacticalLoop);
2406 return;
2407 }
2408
2409 LexicalScope ForScope(*this, S->getSourceRange());
2410
2411 // Emit init statements. The Distance/LoopVar funcs may reference variable
2412 // declarations they contain.
2413 const Stmt *BodyStmt;
2414 if (const auto *For = dyn_cast<ForStmt>(Val: SyntacticalLoop)) {
2415 if (const Stmt *InitStmt = For->getInit())
2416 EmitStmt(S: InitStmt);
2417 BodyStmt = For->getBody();
2418 } else if (const auto *RangeFor =
2419 dyn_cast<CXXForRangeStmt>(Val: SyntacticalLoop)) {
2420 if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
2421 EmitStmt(S: RangeStmt);
2422 if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
2423 EmitStmt(S: BeginStmt);
2424 if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
2425 EmitStmt(S: EndStmt);
2426 if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
2427 EmitStmt(S: LoopVarStmt);
2428 BodyStmt = RangeFor->getBody();
2429 } else
2430 llvm_unreachable("Expected for-stmt or range-based for-stmt");
2431
2432 // Emit closure for later use. By-value captures will be captured here.
2433 const CapturedStmt *DistanceFunc = S->getDistanceFunc();
2434 EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(ParentCGF&: *this, S: DistanceFunc);
2435 const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
2436 EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(ParentCGF&: *this, S: LoopVarFunc);
2437
2438 // Call the distance function to get the number of iterations of the loop to
2439 // come.
2440 QualType LogicalTy = DistanceFunc->getCapturedDecl()
2441 ->getParam(i: 0)
2442 ->getType()
2443 .getNonReferenceType();
2444 RawAddress CountAddr = CreateMemTemp(T: LogicalTy, Name: ".count.addr");
2445 emitCapturedStmtCall(ParentCGF&: *this, Cap: DistanceClosure, Args: {CountAddr.getPointer()});
2446 llvm::Value *DistVal = Builder.CreateLoad(Addr: CountAddr, Name: ".count");
2447
2448 // Emit the loop structure.
2449 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
2450 auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
2451 llvm::Value *IndVar) {
2452 Builder.restoreIP(IP: CodeGenIP);
2453
2454 // Emit the loop body: Convert the logical iteration number to the loop
2455 // variable and emit the body.
2456 const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
2457 LValue LCVal = EmitLValue(E: LoopVarRef);
2458 Address LoopVarAddress = LCVal.getAddress();
2459 emitCapturedStmtCall(ParentCGF&: *this, Cap: LoopVarClosure,
2460 Args: {LoopVarAddress.emitRawPointer(CGF&: *this), IndVar});
2461
2462 RunCleanupsScope BodyScope(*this);
2463 EmitStmt(S: BodyStmt);
2464 return llvm::Error::success();
2465 };
2466
2467 llvm::CanonicalLoopInfo *CL =
2468 cantFail(ValOrErr: OMPBuilder.createCanonicalLoop(Loc: Builder, BodyGenCB: BodyGen, TripCount: DistVal));
2469
2470 // Finish up the loop.
2471 Builder.restoreIP(IP: CL->getAfterIP());
2472 ForScope.ForceCleanup();
2473
2474 // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
2475 OMPLoopNestStack.push_back(Elt: CL);
2476}
2477
2478void CodeGenFunction::EmitOMPInnerLoop(
2479 const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
2480 const Expr *IncExpr,
2481 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
2482 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
2483 auto LoopExit = getJumpDestInCurrentScope(Name: "omp.inner.for.end");
2484
2485 // Start the loop with a block that tests the condition.
2486 auto CondBlock = createBasicBlock(name: "omp.inner.for.cond");
2487 EmitBlock(BB: CondBlock);
2488 const SourceRange R = S.getSourceRange();
2489
2490 // If attributes are attached, push to the basic block with them.
2491 const auto &OMPED = cast<OMPExecutableDirective>(Val: S);
2492 const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
2493 const Stmt *SS = ICS->getCapturedStmt();
2494 const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(Val: SS);
2495 OMPLoopNestStack.clear();
2496 if (AS)
2497 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
2498 Attrs: AS->getAttrs(), StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
2499 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
2500 else
2501 LoopStack.push(Header: CondBlock, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
2502 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
2503
2504 // If there are any cleanups between here and the loop-exit scope,
2505 // create a block to stage a loop exit along.
2506 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2507 if (RequiresCleanup)
2508 ExitBlock = createBasicBlock(name: "omp.inner.for.cond.cleanup");
2509
2510 llvm::BasicBlock *LoopBody = createBasicBlock(name: "omp.inner.for.body");
2511
2512 // Emit condition.
2513 EmitBranchOnBoolExpr(Cond: LoopCond, TrueBlock: LoopBody, FalseBlock: ExitBlock, TrueCount: getProfileCount(S: &S));
2514 if (ExitBlock != LoopExit.getBlock()) {
2515 EmitBlock(BB: ExitBlock);
2516 EmitBranchThroughCleanup(Dest: LoopExit);
2517 }
2518
2519 EmitBlock(BB: LoopBody);
2520 incrementProfileCounter(S: &S);
2521
2522 // Create a block for the increment.
2523 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.inner.for.inc");
2524 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
2525
2526 BodyGen(*this);
2527
2528 // Emit "IV = IV + 1" and a back-edge to the condition block.
2529 EmitBlock(BB: Continue.getBlock());
2530 EmitIgnoredExpr(E: IncExpr);
2531 PostIncGen(*this);
2532 BreakContinueStack.pop_back();
2533 EmitBranch(Block: CondBlock);
2534 LoopStack.pop();
2535 // Emit the fall-through block.
2536 EmitBlock(BB: LoopExit.getBlock());
2537}
2538
2539bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
2540 if (!HaveInsertPoint())
2541 return false;
2542 // Emit inits for the linear variables.
2543 bool HasLinears = false;
2544 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2545 for (const Expr *Init : C->inits()) {
2546 HasLinears = true;
2547 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Init)->getDecl());
2548 if (const auto *Ref =
2549 dyn_cast<DeclRefExpr>(Val: VD->getInit()->IgnoreImpCasts())) {
2550 AutoVarEmission Emission = EmitAutoVarAlloca(var: *VD);
2551 const auto *OrigVD = cast<VarDecl>(Val: Ref->getDecl());
2552 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2553 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
2554 VD->getInit()->getType(), VK_LValue,
2555 VD->getInit()->getExprLoc());
2556 EmitExprAsInit(
2557 init: &DRE, D: VD,
2558 lvalue: MakeAddrLValue(Addr: Emission.getAllocatedAddress(), T: VD->getType()),
2559 /*capturedByInit=*/false);
2560 EmitAutoVarCleanups(emission: Emission);
2561 } else {
2562 EmitVarDecl(D: *VD);
2563 }
2564 }
2565 // Emit the linear steps for the linear clauses.
2566 // If a step is not constant, it is pre-calculated before the loop.
2567 if (const auto *CS = cast_or_null<BinaryOperator>(Val: C->getCalcStep()))
2568 if (const auto *SaveRef = cast<DeclRefExpr>(Val: CS->getLHS())) {
2569 EmitVarDecl(D: *cast<VarDecl>(Val: SaveRef->getDecl()));
2570 // Emit calculation of the linear step.
2571 EmitIgnoredExpr(E: CS);
2572 }
2573 }
2574 return HasLinears;
2575}
2576
2577void CodeGenFunction::EmitOMPLinearClauseFinal(
2578 const OMPLoopDirective &D,
2579 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2580 if (!HaveInsertPoint())
2581 return;
2582 llvm::BasicBlock *DoneBB = nullptr;
2583 // Emit the final values of the linear variables.
2584 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2585 auto IC = C->varlist_begin();
2586 for (const Expr *F : C->finals()) {
2587 if (!DoneBB) {
2588 if (llvm::Value *Cond = CondGen(*this)) {
2589 // If the first post-update expression is found, emit conditional
2590 // block if it was requested.
2591 llvm::BasicBlock *ThenBB = createBasicBlock(name: ".omp.linear.pu");
2592 DoneBB = createBasicBlock(name: ".omp.linear.pu.done");
2593 Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
2594 EmitBlock(BB: ThenBB);
2595 }
2596 }
2597 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IC)->getDecl());
2598 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
2599 CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
2600 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
2601 Address OrigAddr = EmitLValue(E: &DRE).getAddress();
2602 CodeGenFunction::OMPPrivateScope VarScope(*this);
2603 VarScope.addPrivate(LocalVD: OrigVD, Addr: OrigAddr);
2604 (void)VarScope.Privatize();
2605 EmitIgnoredExpr(E: F);
2606 ++IC;
2607 }
2608 if (const Expr *PostUpdate = C->getPostUpdateExpr())
2609 EmitIgnoredExpr(E: PostUpdate);
2610 }
2611 if (DoneBB)
2612 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
2613}
2614
2615static void emitAlignedClause(CodeGenFunction &CGF,
2616 const OMPExecutableDirective &D) {
2617 if (!CGF.HaveInsertPoint())
2618 return;
2619 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
2620 llvm::APInt ClauseAlignment(64, 0);
2621 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
2622 auto *AlignmentCI =
2623 cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AlignmentExpr));
2624 ClauseAlignment = AlignmentCI->getValue();
2625 }
2626 for (const Expr *E : Clause->varlist()) {
2627 llvm::APInt Alignment(ClauseAlignment);
2628 if (Alignment == 0) {
2629 // OpenMP [2.8.1, Description]
2630 // If no optional parameter is specified, implementation-defined default
2631 // alignments for SIMD instructions on the target platforms are assumed.
2632 Alignment =
2633 CGF.getContext()
2634 .toCharUnitsFromBits(BitSize: CGF.getContext().getOpenMPDefaultSimdAlign(
2635 T: E->getType()->getPointeeType()))
2636 .getQuantity();
2637 }
2638 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
2639 "alignment is not power of 2");
2640 if (Alignment != 0) {
2641 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
2642 CGF.emitAlignmentAssumption(
2643 PtrValue, E, /*No second loc needed*/ AssumptionLoc: SourceLocation(),
2644 Alignment: llvm::ConstantInt::get(Context&: CGF.getLLVMContext(), V: Alignment));
2645 }
2646 }
2647 }
2648}
2649
2650void CodeGenFunction::EmitOMPPrivateLoopCounters(
2651 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
2652 if (!HaveInsertPoint())
2653 return;
2654 auto I = S.private_counters().begin();
2655 for (const Expr *E : S.counters()) {
2656 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
2657 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
2658 // Emit var without initialization.
2659 AutoVarEmission VarEmission = EmitAutoVarAlloca(var: *PrivateVD);
2660 EmitAutoVarCleanups(emission: VarEmission);
2661 LocalDeclMap.erase(Val: PrivateVD);
2662 (void)LoopScope.addPrivate(LocalVD: VD, Addr: VarEmission.getAllocatedAddress());
2663 if (LocalDeclMap.count(Val: VD) || CapturedStmtInfo->lookup(VD) ||
2664 VD->hasGlobalStorage()) {
2665 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
2666 LocalDeclMap.count(Val: VD) || CapturedStmtInfo->lookup(VD),
2667 E->getType(), VK_LValue, E->getExprLoc());
2668 (void)LoopScope.addPrivate(LocalVD: PrivateVD, Addr: EmitLValue(E: &DRE).getAddress());
2669 } else {
2670 (void)LoopScope.addPrivate(LocalVD: PrivateVD, Addr: VarEmission.getAllocatedAddress());
2671 }
2672 ++I;
2673 }
2674 // Privatize extra loop counters used in loops for ordered(n) clauses.
2675 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
2676 if (!C->getNumForLoops())
2677 continue;
2678 for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
2679 I < E; ++I) {
2680 const auto *DRE = cast<DeclRefExpr>(Val: C->getLoopCounter(NumLoop: I));
2681 const auto *VD = cast<VarDecl>(Val: DRE->getDecl());
2682 // Override only those variables that can be captured to avoid re-emission
2683 // of the variables declared within the loops.
2684 if (DRE->refersToEnclosingVariableOrCapture()) {
2685 (void)LoopScope.addPrivate(
2686 LocalVD: VD, Addr: CreateMemTemp(T: DRE->getType(), Name: VD->getName()));
2687 }
2688 }
2689 }
2690}
2691
2692static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
2693 const Expr *Cond, llvm::BasicBlock *TrueBlock,
2694 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
2695 if (!CGF.HaveInsertPoint())
2696 return;
2697 {
2698 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
2699 CGF.EmitOMPPrivateLoopCounters(S, LoopScope&: PreCondScope);
2700 (void)PreCondScope.Privatize();
2701 // Get initial values of real counters.
2702 for (const Expr *I : S.inits()) {
2703 CGF.EmitIgnoredExpr(E: I);
2704 }
2705 }
2706 // Create temp loop control variables with their init values to support
2707 // non-rectangular loops.
2708 CodeGenFunction::OMPMapVars PreCondVars;
2709 for (const Expr *E : S.dependent_counters()) {
2710 if (!E)
2711 continue;
2712 assert(!E->getType().getNonReferenceType()->isRecordType() &&
2713 "dependent counter must not be an iterator.");
2714 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
2715 Address CounterAddr =
2716 CGF.CreateMemTemp(T: VD->getType().getNonReferenceType());
2717 (void)PreCondVars.setVarAddr(CGF, LocalVD: VD, TempAddr: CounterAddr);
2718 }
2719 (void)PreCondVars.apply(CGF);
2720 for (const Expr *E : S.dependent_inits()) {
2721 if (!E)
2722 continue;
2723 CGF.EmitIgnoredExpr(E);
2724 }
2725 // Check that loop is executed at least one time.
2726 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
2727 PreCondVars.restore(CGF);
2728}
2729
2730void CodeGenFunction::EmitOMPLinearClause(
2731 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
2732 if (!HaveInsertPoint())
2733 return;
2734 llvm::DenseSet<const VarDecl *> SIMDLCVs;
2735 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2736 if (isOpenMPSimdDirective(DKind: EKind)) {
2737 const auto *LoopDirective = cast<OMPLoopDirective>(Val: &D);
2738 for (const Expr *C : LoopDirective->counters()) {
2739 SIMDLCVs.insert(
2740 V: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: C)->getDecl())->getCanonicalDecl());
2741 }
2742 }
2743 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
2744 auto CurPrivate = C->privates().begin();
2745 for (const Expr *E : C->varlist()) {
2746 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
2747 const auto *PrivateVD =
2748 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *CurPrivate)->getDecl());
2749 if (!SIMDLCVs.count(V: VD->getCanonicalDecl())) {
2750 // Emit private VarDecl with copy init.
2751 EmitVarDecl(D: *PrivateVD);
2752 bool IsRegistered =
2753 PrivateScope.addPrivate(LocalVD: VD, Addr: GetAddrOfLocalVar(VD: PrivateVD));
2754 assert(IsRegistered && "linear var already registered as private");
2755 // Silence the warning about unused variable.
2756 (void)IsRegistered;
2757 } else {
2758 EmitVarDecl(D: *PrivateVD);
2759 }
2760 ++CurPrivate;
2761 }
2762 }
2763}
2764
2765static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
2766 const OMPExecutableDirective &D) {
2767 if (!CGF.HaveInsertPoint())
2768 return;
2769 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
2770 RValue Len = CGF.EmitAnyExpr(E: C->getSimdlen(), aggSlot: AggValueSlot::ignored(),
2771 /*ignoreResult=*/true);
2772 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
2773 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2774 // In presence of finite 'safelen', it may be unsafe to mark all
2775 // the memory instructions parallel, because loop-carried
2776 // dependences of 'safelen' iterations are possible.
2777 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
2778 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
2779 RValue Len = CGF.EmitAnyExpr(E: C->getSafelen(), aggSlot: AggValueSlot::ignored(),
2780 /*ignoreResult=*/true);
2781 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
2782 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
2783 // In presence of finite 'safelen', it may be unsafe to mark all
2784 // the memory instructions parallel, because loop-carried
2785 // dependences of 'safelen' iterations are possible.
2786 CGF.LoopStack.setParallel(/*Enable=*/false);
2787 }
2788}
2789
2790// Check for the presence of an `OMPOrderedBlockAssocDirective`,
2791// i.e., `ordered` in `#pragma omp ordered simd`.
2792//
2793// Consider the following source code:
2794// ```
2795// __attribute__((noinline)) void omp_simd_loop(float X[ARRAY_SIZE][ARRAY_SIZE])
2796// {
2797// for (int r = 1; r < ARRAY_SIZE; ++r) {
2798// for (int c = 1; c < ARRAY_SIZE; ++c) {
2799// #pragma omp simd
2800// for (int k = 2; k < ARRAY_SIZE; ++k) {
2801// #pragma omp ordered simd
2802// X[r][k] = X[r][k - 2] + sinf((float)(r / c));
2803// }
2804// }
2805// }
2806// }
2807// ```
2808//
2809// Suppose we are in `CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective
2810// &D)`. By examining `D.dump()` we have the following AST containing
2811// `OMPOrderedBlockAssocDirective`:
2812//
2813// ```
2814// OMPSimdDirective 0x1c32950
2815// `-CapturedStmt 0x1c32028
2816// |-CapturedDecl 0x1c310e8
2817// | |-ForStmt 0x1c31e30
2818// | | |-DeclStmt 0x1c31298
2819// | | | `-VarDecl 0x1c31208 used k 'int' cinit
2820// | | | `-IntegerLiteral 0x1c31278 'int' 2
2821// | | |-<<<NULL>>>
2822// | | |-BinaryOperator 0x1c31308 'int' '<'
2823// | | | |-ImplicitCastExpr 0x1c312f0 'int' <LValueToRValue>
2824// | | | | `-DeclRefExpr 0x1c312b0 'int' lvalue Var 0x1c31208 'k' 'int'
2825// | | | `-IntegerLiteral 0x1c312d0 'int' 256
2826// | | |-UnaryOperator 0x1c31348 'int' prefix '++'
2827// | | | `-DeclRefExpr 0x1c31328 'int' lvalue Var 0x1c31208 'k' 'int'
2828// | | `-CompoundStmt 0x1c31e18
2829// | | `-OMPOrderedBlockAssocDirective 0x1c31dd8
2830// | | |-OMPSimdClause 0x1c31380
2831// | | `-CapturedStmt 0x1c31cd0
2832// ```
2833//
2834// Note the presence of `OMPOrderedBlockAssocDirective` above:
2835// It's (transitively) nested in a `CapturedStmt` representing the pragma
2836// annotated compound statement. Thus, we need to consider this nesting and
2837// include checking the `getCapturedStmt` in this case.
2838static bool hasOrderedBlockAssocDirective(const Stmt *S) {
2839 if (isa<OMPOrderedBlockAssocDirective>(Val: S))
2840 return true;
2841
2842 if (const auto *CS = dyn_cast<CapturedStmt>(Val: S))
2843 return hasOrderedBlockAssocDirective(S: CS->getCapturedStmt());
2844
2845 for (const Stmt *Child : S->children()) {
2846 if (Child && hasOrderedBlockAssocDirective(S: Child))
2847 return true;
2848 }
2849
2850 return false;
2851}
2852
2853static void applyConservativeSimdOrderedDirective(const Stmt &AssociatedStmt,
2854 LoopInfoStack &LoopStack) {
2855 // Check for the presence of an `OMPOrderedBlockAssocDirective`
2856 // i.e., `ordered` in `#pragma omp ordered simd`
2857 bool HasOrderedDirective = hasOrderedBlockAssocDirective(S: &AssociatedStmt);
2858 // If present then conservatively disable loop vectorization
2859 // analogously to how `emitSimdlenSafelenClause` does.
2860 if (HasOrderedDirective)
2861 LoopStack.setParallel(/*Enable=*/false);
2862}
2863
2864void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) {
2865 // Walk clauses and process safelen/lastprivate.
2866 LoopStack.setParallel(/*Enable=*/true);
2867 LoopStack.setVectorizeEnable();
2868 const Stmt *AssociatedStmt = D.getAssociatedStmt();
2869 applyConservativeSimdOrderedDirective(AssociatedStmt: *AssociatedStmt, LoopStack);
2870 emitSimdlenSafelenClause(CGF&: *this, D);
2871 if (const auto *C = D.getSingleClause<OMPOrderClause>())
2872 if (C->getKind() == OMPC_ORDER_concurrent)
2873 LoopStack.setParallel(/*Enable=*/true);
2874 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S: D);
2875 if ((EKind == OMPD_simd ||
2876 (getLangOpts().OpenMPSimd && isOpenMPSimdDirective(DKind: EKind))) &&
2877 llvm::any_of(Range: D.getClausesOfKind<OMPReductionClause>(),
2878 P: [](const OMPReductionClause *C) {
2879 return C->getModifier() == OMPC_REDUCTION_inscan;
2880 }))
2881 // Disable parallel access in case of prefix sum.
2882 LoopStack.setParallel(/*Enable=*/false);
2883}
2884
2885void CodeGenFunction::EmitOMPSimdFinal(
2886 const OMPLoopDirective &D,
2887 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
2888 if (!HaveInsertPoint())
2889 return;
2890 llvm::BasicBlock *DoneBB = nullptr;
2891 auto IC = D.counters().begin();
2892 auto IPC = D.private_counters().begin();
2893 for (const Expr *F : D.finals()) {
2894 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: (*IC))->getDecl());
2895 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: (*IPC))->getDecl());
2896 const auto *CED = dyn_cast<OMPCapturedExprDecl>(Val: OrigVD);
2897 if (LocalDeclMap.count(Val: OrigVD) || CapturedStmtInfo->lookup(VD: OrigVD) ||
2898 OrigVD->hasGlobalStorage() || CED) {
2899 if (!DoneBB) {
2900 if (llvm::Value *Cond = CondGen(*this)) {
2901 // If the first post-update expression is found, emit conditional
2902 // block if it was requested.
2903 llvm::BasicBlock *ThenBB = createBasicBlock(name: ".omp.final.then");
2904 DoneBB = createBasicBlock(name: ".omp.final.done");
2905 Builder.CreateCondBr(Cond, True: ThenBB, False: DoneBB);
2906 EmitBlock(BB: ThenBB);
2907 }
2908 }
2909 Address OrigAddr = Address::invalid();
2910 if (CED) {
2911 OrigAddr = EmitLValue(E: CED->getInit()->IgnoreImpCasts()).getAddress();
2912 } else {
2913 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
2914 /*RefersToEnclosingVariableOrCapture=*/false,
2915 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
2916 OrigAddr = EmitLValue(E: &DRE).getAddress();
2917 }
2918 OMPPrivateScope VarScope(*this);
2919 VarScope.addPrivate(LocalVD: OrigVD, Addr: OrigAddr);
2920 (void)VarScope.Privatize();
2921 EmitIgnoredExpr(E: F);
2922 }
2923 ++IC;
2924 ++IPC;
2925 }
2926 if (DoneBB)
2927 EmitBlock(BB: DoneBB, /*IsFinished=*/true);
2928}
2929
2930static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
2931 const OMPLoopDirective &S,
2932 CodeGenFunction::JumpDest LoopExit) {
2933 CGF.EmitOMPLoopBody(D: S, LoopExit);
2934 CGF.EmitStopPoint(S: &S);
2935}
2936
2937/// Emit a helper variable and return corresponding lvalue.
2938static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
2939 const DeclRefExpr *Helper) {
2940 auto VDecl = cast<VarDecl>(Val: Helper->getDecl());
2941 CGF.EmitVarDecl(D: *VDecl);
2942 return CGF.EmitLValue(E: Helper);
2943}
2944
2945static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
2946 const RegionCodeGenTy &SimdInitGen,
2947 const RegionCodeGenTy &BodyCodeGen) {
2948 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
2949 PrePostActionTy &) {
2950 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
2951 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2952 SimdInitGen(CGF);
2953
2954 BodyCodeGen(CGF);
2955 };
2956 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
2957 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
2958 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
2959
2960 BodyCodeGen(CGF);
2961 };
2962 const Expr *IfCond = nullptr;
2963 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
2964 if (isOpenMPSimdDirective(DKind: EKind)) {
2965 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2966 if (CGF.getLangOpts().OpenMP >= 50 &&
2967 (C->getNameModifier() == OMPD_unknown ||
2968 C->getNameModifier() == OMPD_simd)) {
2969 IfCond = C->getCondition();
2970 break;
2971 }
2972 }
2973 }
2974 if (IfCond) {
2975 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
2976 } else {
2977 RegionCodeGenTy ThenRCG(ThenGen);
2978 ThenRCG(CGF);
2979 }
2980}
2981
2982static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
2983 PrePostActionTy &Action) {
2984 Action.Enter(CGF);
2985 OMPLoopScope PreInitScope(CGF, S);
2986 // if (PreCond) {
2987 // for (IV in 0..LastIteration) BODY;
2988 // <Final counter/linear vars updates>;
2989 // }
2990
2991 // The presence of lower/upper bound variable depends on the actual directive
2992 // kind in the AST node. The variables must be emitted because some of the
2993 // expressions associated with the loop will use them.
2994 OpenMPDirectiveKind DKind = S.getDirectiveKind();
2995 if (isOpenMPDistributeDirective(DKind) ||
2996 isOpenMPWorksharingDirective(DKind) || isOpenMPTaskLoopDirective(DKind) ||
2997 isOpenMPGenericLoopDirective(DKind)) {
2998 (void)EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: S.getLowerBoundVariable()));
2999 (void)EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: S.getUpperBoundVariable()));
3000 }
3001
3002 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3003 // Emit: if (PreCond) - begin.
3004 // If the condition constant folds and can be elided, avoid emitting the
3005 // whole loop.
3006 bool CondConstant;
3007 llvm::BasicBlock *ContBlock = nullptr;
3008 if (CGF.ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
3009 if (!CondConstant)
3010 return;
3011 } else {
3012 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "simd.if.then");
3013 ContBlock = CGF.createBasicBlock(name: "simd.if.end");
3014 emitPreCond(CGF, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
3015 TrueCount: CGF.getProfileCount(S: &S));
3016 CGF.EmitBlock(BB: ThenBlock);
3017 CGF.incrementProfileCounter(S: &S);
3018 }
3019
3020 // Emit the loop iteration variable.
3021 const Expr *IVExpr = S.getIterationVariable();
3022 const auto *IVDecl = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IVExpr)->getDecl());
3023 CGF.EmitVarDecl(D: *IVDecl);
3024 CGF.EmitIgnoredExpr(E: S.getInit());
3025
3026 // Emit the iterations count variable.
3027 // If it is not a variable, Sema decided to calculate iterations count on
3028 // each iteration (e.g., it is foldable into a constant).
3029 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
3030 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
3031 // Emit calculation of the iterations count.
3032 CGF.EmitIgnoredExpr(E: S.getCalcLastIteration());
3033 }
3034
3035 emitAlignedClause(CGF, D: S);
3036 (void)CGF.EmitOMPLinearClauseInit(D: S);
3037 {
3038 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
3039 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
3040 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
3041 CGF.EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
3042 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
3043 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
3044 CGF, S, CGF.EmitLValue(E: S.getIterationVariable()));
3045 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
3046 (void)LoopScope.Privatize();
3047 if (isOpenMPTargetExecutionDirective(DKind: EKind))
3048 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
3049
3050 emitCommonSimdLoop(
3051 CGF, S,
3052 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3053 CGF.EmitOMPSimdInit(D: S);
3054 },
3055 BodyCodeGen: [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3056 CGF.EmitOMPInnerLoop(
3057 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: S.getCond(), IncExpr: S.getInc(),
3058 BodyGen: [&S](CodeGenFunction &CGF) {
3059 emitOMPLoopBodyWithStopPoint(CGF, S,
3060 LoopExit: CodeGenFunction::JumpDest());
3061 },
3062 PostIncGen: [](CodeGenFunction &) {});
3063 });
3064 CGF.EmitOMPSimdFinal(D: S, CondGen: [](CodeGenFunction &) { return nullptr; });
3065 // Emit final copy of the lastprivate variables at the end of loops.
3066 if (HasLastprivateClause)
3067 CGF.EmitOMPLastprivateClauseFinal(D: S, /*NoFinals=*/true);
3068 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_simd);
3069 emitPostUpdateForReductionClause(CGF, D: S,
3070 CondGen: [](CodeGenFunction &) { return nullptr; });
3071 LoopScope.restoreMap();
3072 CGF.EmitOMPLinearClauseFinal(D: S, CondGen: [](CodeGenFunction &) { return nullptr; });
3073 }
3074 // Emit: if (PreCond) - end.
3075 if (ContBlock) {
3076 CGF.EmitBranch(Block: ContBlock);
3077 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
3078 }
3079}
3080
3081// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3082// available for "loop bind(thread)", which maps to "simd".
3083static bool isSimdSupportedByOpenMPIRBuilder(const OMPLoopDirective &S) {
3084 // Check for unsupported clauses
3085 for (OMPClause *C : S.clauses()) {
3086 // Currently only order, simdlen and safelen clauses are supported
3087 if (!(isa<OMPSimdlenClause>(Val: C) || isa<OMPSafelenClause>(Val: C) ||
3088 isa<OMPOrderClause>(Val: C) || isa<OMPAlignedClause>(Val: C)))
3089 return false;
3090 }
3091
3092 // Check if we have a statement with the ordered-blockassoc directive.
3093 // Visit the statement hierarchy to find a compound statement
3094 // with a ordered-blockassoc directive in it.
3095 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Val: S.getRawStmt())) {
3096 if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
3097 for (const Stmt *SubStmt : SyntacticalLoop->children()) {
3098 if (!SubStmt)
3099 continue;
3100 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: SubStmt)) {
3101 for (const Stmt *CSSubStmt : CS->children()) {
3102 if (!CSSubStmt)
3103 continue;
3104 if (isa<OMPOrderedBlockAssocDirective>(Val: CSSubStmt)) {
3105 return false;
3106 }
3107 }
3108 }
3109 }
3110 }
3111 }
3112 return true;
3113}
3114
3115static llvm::MapVector<llvm::Value *, llvm::Value *>
3116GetAlignedMapping(const OMPLoopDirective &S, CodeGenFunction &CGF) {
3117 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;
3118 for (const auto *Clause : S.getClausesOfKind<OMPAlignedClause>()) {
3119 llvm::APInt ClauseAlignment(64, 0);
3120 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
3121 auto *AlignmentCI =
3122 cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AlignmentExpr));
3123 ClauseAlignment = AlignmentCI->getValue();
3124 }
3125 for (const Expr *E : Clause->varlist()) {
3126 llvm::APInt Alignment(ClauseAlignment);
3127 if (Alignment == 0) {
3128 // OpenMP [2.8.1, Description]
3129 // If no optional parameter is specified, implementation-defined default
3130 // alignments for SIMD instructions on the target platforms are assumed.
3131 Alignment =
3132 CGF.getContext()
3133 .toCharUnitsFromBits(BitSize: CGF.getContext().getOpenMPDefaultSimdAlign(
3134 T: E->getType()->getPointeeType()))
3135 .getQuantity();
3136 }
3137 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
3138 "alignment is not power of 2");
3139 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
3140 AlignedVars[PtrValue] = CGF.Builder.getInt64(C: Alignment.getSExtValue());
3141 }
3142 }
3143 return AlignedVars;
3144}
3145
3146// Pass OMPLoopDirective (instead of OMPSimdDirective) to make this function
3147// available for "loop bind(thread)", which maps to "simd".
3148static void emitOMPSimdDirective(const OMPLoopDirective &S,
3149 CodeGenFunction &CGF, CodeGenModule &CGM) {
3150 bool UseOMPIRBuilder =
3151 CGM.getLangOpts().OpenMPIRBuilder && isSimdSupportedByOpenMPIRBuilder(S);
3152 if (UseOMPIRBuilder) {
3153 auto &&CodeGenIRBuilder = [&S, &CGM, UseOMPIRBuilder](CodeGenFunction &CGF,
3154 PrePostActionTy &) {
3155 // Use the OpenMPIRBuilder if enabled.
3156 if (UseOMPIRBuilder) {
3157 llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =
3158 GetAlignedMapping(S, CGF);
3159 // Emit the associated statement and get its loop representation.
3160 const Stmt *Inner = S.getRawStmt();
3161 llvm::CanonicalLoopInfo *CLI =
3162 CGF.EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
3163
3164 llvm::OpenMPIRBuilder &OMPBuilder =
3165 CGM.getOpenMPRuntime().getOMPBuilder();
3166 // Add SIMD specific metadata
3167 llvm::ConstantInt *Simdlen = nullptr;
3168 if (const auto *C = S.getSingleClause<OMPSimdlenClause>()) {
3169 RValue Len = CGF.EmitAnyExpr(E: C->getSimdlen(), aggSlot: AggValueSlot::ignored(),
3170 /*ignoreResult=*/true);
3171 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3172 Simdlen = Val;
3173 }
3174 llvm::ConstantInt *Safelen = nullptr;
3175 if (const auto *C = S.getSingleClause<OMPSafelenClause>()) {
3176 RValue Len = CGF.EmitAnyExpr(E: C->getSafelen(), aggSlot: AggValueSlot::ignored(),
3177 /*ignoreResult=*/true);
3178 auto *Val = cast<llvm::ConstantInt>(Val: Len.getScalarVal());
3179 Safelen = Val;
3180 }
3181 llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;
3182 if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3183 if (C->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {
3184 Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;
3185 }
3186 }
3187 // Add simd metadata to the collapsed loop. Do not generate
3188 // another loop for if clause. Support for if clause is done earlier.
3189 OMPBuilder.applySimd(Loop: CLI, AlignedVars,
3190 /*IfCond*/ nullptr, Order, Simdlen, Safelen);
3191 return;
3192 }
3193 };
3194 {
3195 auto LPCRegion =
3196 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
3197 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3198 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd,
3199 CodeGen: CodeGenIRBuilder);
3200 }
3201 return;
3202 }
3203
3204 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
3205 CGF.OMPFirstScanLoop = true;
3206 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3207 emitOMPSimdRegion(CGF, S, Action);
3208 };
3209 {
3210 auto LPCRegion =
3211 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
3212 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
3213 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd, CodeGen);
3214 }
3215 // Check for outer lastprivate conditional update.
3216 checkForLastprivateConditionalUpdate(CGF, S);
3217}
3218
3219void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
3220 emitOMPSimdDirective(S, CGF&: *this, CGM);
3221}
3222
3223void CodeGenFunction::EmitOMPTileDirective(const OMPTileDirective &S) {
3224 // Emit the de-sugared statement.
3225 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
3226 EmitStmt(S: S.getTransformedStmt());
3227}
3228
3229void CodeGenFunction::EmitOMPStripeDirective(const OMPStripeDirective &S) {
3230 // Emit the de-sugared statement.
3231 OMPTransformDirectiveScopeRAII StripeScope(*this, &S);
3232 EmitStmt(S: S.getTransformedStmt());
3233}
3234
3235void CodeGenFunction::EmitOMPReverseDirective(const OMPReverseDirective &S) {
3236 // Emit the de-sugared statement.
3237 OMPTransformDirectiveScopeRAII ReverseScope(*this, &S);
3238 EmitStmt(S: S.getTransformedStmt());
3239}
3240
3241void CodeGenFunction::EmitOMPSplitDirective(const OMPSplitDirective &S) {
3242 // Emit the de-sugared statement (the split loops).
3243 OMPTransformDirectiveScopeRAII SplitScope(*this, &S);
3244 EmitStmt(S: S.getTransformedStmt());
3245}
3246
3247void CodeGenFunction::EmitOMPInterchangeDirective(
3248 const OMPInterchangeDirective &S) {
3249 // Emit the de-sugared statement.
3250 OMPTransformDirectiveScopeRAII InterchangeScope(*this, &S);
3251 EmitStmt(S: S.getTransformedStmt());
3252}
3253
3254void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
3255 // Emit the de-sugared statement
3256 OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
3257 EmitStmt(S: S.getTransformedStmt());
3258}
3259
3260void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
3261 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
3262
3263 if (UseOMPIRBuilder) {
3264 auto DL = SourceLocToDebugLoc(Location: S.getBeginLoc());
3265 const Stmt *Inner = S.getRawStmt();
3266
3267 // Consume nested loop. Clear the entire remaining loop stack because a
3268 // fully unrolled loop is non-transformable. For partial unrolling the
3269 // generated outer loop is pushed back to the stack.
3270 llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
3271 OMPLoopNestStack.clear();
3272
3273 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
3274
3275 bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
3276 llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
3277
3278 if (S.hasClausesOfKind<OMPFullClause>()) {
3279 assert(ExpectedOMPLoopDepth == 0);
3280 OMPBuilder.unrollLoopFull(DL, Loop: CLI);
3281 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3282 uint64_t Factor = 0;
3283 if (Expr *FactorExpr = PartialClause->getFactor()) {
3284 Factor = FactorExpr->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
3285 assert(Factor >= 1 && "Only positive factors are valid");
3286 }
3287 OMPBuilder.unrollLoopPartial(DL, Loop: CLI, Factor,
3288 UnrolledCLI: NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
3289 } else {
3290 OMPBuilder.unrollLoopHeuristic(DL, Loop: CLI);
3291 }
3292
3293 assert((!NeedsUnrolledCLI || UnrolledCLI) &&
3294 "NeedsUnrolledCLI implies UnrolledCLI to be set");
3295 if (UnrolledCLI)
3296 OMPLoopNestStack.push_back(Elt: UnrolledCLI);
3297
3298 return;
3299 }
3300
3301 // This function is only called if the unrolled loop is not consumed by any
3302 // other loop-associated construct. Such a loop-associated construct will have
3303 // used the transformed AST.
3304
3305 // Set the unroll metadata for the next emitted loop.
3306 LoopStack.setUnrollState(LoopAttributes::Enable);
3307
3308 if (S.hasClausesOfKind<OMPFullClause>()) {
3309 LoopStack.setUnrollState(LoopAttributes::Full);
3310 } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
3311 if (Expr *FactorExpr = PartialClause->getFactor()) {
3312 uint64_t Factor =
3313 FactorExpr->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
3314 assert(Factor >= 1 && "Only positive factors are valid");
3315 LoopStack.setUnrollCount(Factor);
3316 }
3317 }
3318
3319 EmitStmt(S: S.getAssociatedStmt());
3320}
3321
3322void CodeGenFunction::EmitOMPOuterLoop(
3323 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
3324 CodeGenFunction::OMPPrivateScope &LoopScope,
3325 const CodeGenFunction::OMPLoopArguments &LoopArgs,
3326 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
3327 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
3328 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3329
3330 const Expr *IVExpr = S.getIterationVariable();
3331 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3332 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3333
3334 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "omp.dispatch.end");
3335
3336 // Start the loop with a block that tests the condition.
3337 llvm::BasicBlock *CondBlock = createBasicBlock(name: "omp.dispatch.cond");
3338 EmitBlock(BB: CondBlock);
3339 const SourceRange R = S.getSourceRange();
3340 OMPLoopNestStack.clear();
3341 LoopStack.push(Header: CondBlock, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
3342 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
3343
3344 llvm::Value *BoolCondVal = nullptr;
3345 if (!DynamicOrOrdered) {
3346 // UB = min(UB, GlobalUB) or
3347 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
3348 // 'distribute parallel for')
3349 EmitIgnoredExpr(E: LoopArgs.EUB);
3350 // IV = LB
3351 EmitIgnoredExpr(E: LoopArgs.Init);
3352 // IV < UB
3353 BoolCondVal = EvaluateExprAsBool(E: LoopArgs.Cond);
3354 } else {
3355 BoolCondVal =
3356 RT.emitForNext(CGF&: *this, Loc: S.getBeginLoc(), IVSize, IVSigned, IL: LoopArgs.IL,
3357 LB: LoopArgs.LB, UB: LoopArgs.UB, ST: LoopArgs.ST);
3358 }
3359
3360 // If there are any cleanups between here and the loop-exit scope,
3361 // create a block to stage a loop exit along.
3362 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
3363 if (LoopScope.requiresCleanups())
3364 ExitBlock = createBasicBlock(name: "omp.dispatch.cleanup");
3365
3366 llvm::BasicBlock *LoopBody = createBasicBlock(name: "omp.dispatch.body");
3367 Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock);
3368 if (ExitBlock != LoopExit.getBlock()) {
3369 EmitBlock(BB: ExitBlock);
3370 EmitBranchThroughCleanup(Dest: LoopExit);
3371 }
3372 EmitBlock(BB: LoopBody);
3373
3374 // Emit "IV = LB" (in case of static schedule, we have already calculated new
3375 // LB for loop condition and emitted it above).
3376 if (DynamicOrOrdered)
3377 EmitIgnoredExpr(E: LoopArgs.Init);
3378
3379 // Create a block for the increment.
3380 JumpDest Continue = getJumpDestInCurrentScope(Name: "omp.dispatch.inc");
3381 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
3382
3383 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3384 emitCommonSimdLoop(
3385 CGF&: *this, S,
3386 SimdInitGen: [&S, IsMonotonic, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3387 // Generate !llvm.loop.parallel metadata for loads and stores for loops
3388 // with dynamic/guided scheduling and without ordered clause.
3389 if (!isOpenMPSimdDirective(DKind: EKind)) {
3390 CGF.LoopStack.setParallel(!IsMonotonic);
3391 if (const auto *C = S.getSingleClause<OMPOrderClause>())
3392 if (C->getKind() == OMPC_ORDER_concurrent)
3393 CGF.LoopStack.setParallel(/*Enable=*/true);
3394 } else {
3395 CGF.EmitOMPSimdInit(D: S);
3396 }
3397 },
3398 BodyCodeGen: [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
3399 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3400 SourceLocation Loc = S.getBeginLoc();
3401 // when 'distribute' is not combined with a 'for':
3402 // while (idx <= UB) { BODY; ++idx; }
3403 // when 'distribute' is combined with a 'for'
3404 // (e.g. 'distribute parallel for')
3405 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3406 CGF.EmitOMPInnerLoop(
3407 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: LoopArgs.Cond, IncExpr: LoopArgs.IncExpr,
3408 BodyGen: [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3409 CodeGenLoop(CGF, S, LoopExit);
3410 },
3411 PostIncGen: [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
3412 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
3413 });
3414 });
3415
3416 EmitBlock(BB: Continue.getBlock());
3417 BreakContinueStack.pop_back();
3418 if (!DynamicOrOrdered) {
3419 // Emit "LB = LB + Stride", "UB = UB + Stride".
3420 EmitIgnoredExpr(E: LoopArgs.NextLB);
3421 EmitIgnoredExpr(E: LoopArgs.NextUB);
3422 }
3423
3424 EmitBranch(Block: CondBlock);
3425 OMPLoopNestStack.clear();
3426 LoopStack.pop();
3427 // Emit the fall-through block.
3428 EmitBlock(BB: LoopExit.getBlock());
3429
3430 // Tell the runtime we are done.
3431 auto &&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](CodeGenFunction &CGF) {
3432 if (!DynamicOrOrdered)
3433 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
3434 DKind: LoopArgs.DKind);
3435 };
3436 OMPCancelStack.emitExit(CGF&: *this, Kind: EKind, CodeGen);
3437}
3438
3439void CodeGenFunction::EmitOMPForOuterLoop(
3440 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
3441 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
3442 const OMPLoopArguments &LoopArgs,
3443 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3444 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3445
3446 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
3447 const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind: ScheduleKind.Schedule);
3448
3449 assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
3450 LoopArgs.Chunk != nullptr)) &&
3451 "static non-chunked schedule does not need outer loop");
3452
3453 // Emit outer loop.
3454 //
3455 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3456 // When schedule(dynamic,chunk_size) is specified, the iterations are
3457 // distributed to threads in the team in chunks as the threads request them.
3458 // Each thread executes a chunk of iterations, then requests another chunk,
3459 // until no chunks remain to be distributed. Each chunk contains chunk_size
3460 // iterations, except for the last chunk to be distributed, which may have
3461 // fewer iterations. When no chunk_size is specified, it defaults to 1.
3462 //
3463 // When schedule(guided,chunk_size) is specified, the iterations are assigned
3464 // to threads in the team in chunks as the executing threads request them.
3465 // Each thread executes a chunk of iterations, then requests another chunk,
3466 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
3467 // each chunk is proportional to the number of unassigned iterations divided
3468 // by the number of threads in the team, decreasing to 1. For a chunk_size
3469 // with value k (greater than 1), the size of each chunk is determined in the
3470 // same way, with the restriction that the chunks do not contain fewer than k
3471 // iterations (except for the last chunk to be assigned, which may have fewer
3472 // than k iterations).
3473 //
3474 // When schedule(auto) is specified, the decision regarding scheduling is
3475 // delegated to the compiler and/or runtime system. The programmer gives the
3476 // implementation the freedom to choose any possible mapping of iterations to
3477 // threads in the team.
3478 //
3479 // When schedule(runtime) is specified, the decision regarding scheduling is
3480 // deferred until run time, and the schedule and chunk size are taken from the
3481 // run-sched-var ICV. If the ICV is set to auto, the schedule is
3482 // implementation defined
3483 //
3484 // __kmpc_dispatch_init();
3485 // while(__kmpc_dispatch_next(&LB, &UB)) {
3486 // idx = LB;
3487 // while (idx <= UB) { BODY; ++idx;
3488 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
3489 // } // inner loop
3490 // }
3491 // __kmpc_dispatch_deinit();
3492 //
3493 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3494 // When schedule(static, chunk_size) is specified, iterations are divided into
3495 // chunks of size chunk_size, and the chunks are assigned to the threads in
3496 // the team in a round-robin fashion in the order of the thread number.
3497 //
3498 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
3499 // while (idx <= UB) { BODY; ++idx; } // inner loop
3500 // LB = LB + ST;
3501 // UB = UB + ST;
3502 // }
3503 //
3504
3505 const Expr *IVExpr = S.getIterationVariable();
3506 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3507 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3508
3509 if (DynamicOrOrdered) {
3510 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
3511 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
3512 llvm::Value *LBVal = DispatchBounds.first;
3513 llvm::Value *UBVal = DispatchBounds.second;
3514 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
3515 LoopArgs.Chunk};
3516 RT.emitForDispatchInit(CGF&: *this, Loc: S.getBeginLoc(), ScheduleKind, IVSize,
3517 IVSigned, Ordered, DispatchValues: DipatchRTInputValues);
3518 } else {
3519 CGOpenMPRuntime::StaticRTInput StaticInit(
3520 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
3521 LoopArgs.ST, LoopArgs.Chunk);
3522 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3523 RT.emitForStaticInit(CGF&: *this, Loc: S.getBeginLoc(), DKind: EKind, ScheduleKind,
3524 Values: StaticInit);
3525 }
3526
3527 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
3528 const unsigned IVSize,
3529 const bool IVSigned) {
3530 if (Ordered) {
3531 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
3532 IVSigned);
3533 }
3534 };
3535
3536 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
3537 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
3538 OuterLoopArgs.IncExpr = S.getInc();
3539 OuterLoopArgs.Init = S.getInit();
3540 OuterLoopArgs.Cond = S.getCond();
3541 OuterLoopArgs.NextLB = S.getNextLowerBound();
3542 OuterLoopArgs.NextUB = S.getNextUpperBound();
3543 OuterLoopArgs.DKind = LoopArgs.DKind;
3544 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, LoopArgs: OuterLoopArgs,
3545 CodeGenLoop: emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
3546 if (DynamicOrOrdered) {
3547 RT.emitForDispatchDeinit(CGF&: *this, Loc: S.getBeginLoc());
3548 }
3549}
3550
3551static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
3552 const unsigned IVSize, const bool IVSigned) {}
3553
3554void CodeGenFunction::EmitOMPDistributeOuterLoop(
3555 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
3556 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
3557 const CodeGenLoopTy &CodeGenLoopContent) {
3558
3559 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3560
3561 // Emit outer loop.
3562 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
3563 // dynamic
3564 //
3565
3566 const Expr *IVExpr = S.getIterationVariable();
3567 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3568 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3569 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3570
3571 CGOpenMPRuntime::StaticRTInput StaticInit(
3572 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
3573 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
3574 RT.emitDistributeStaticInit(CGF&: *this, Loc: S.getBeginLoc(), SchedKind: ScheduleKind, Values: StaticInit);
3575
3576 // for combined 'distribute' and 'for' the increment expression of distribute
3577 // is stored in DistInc. For 'distribute' alone, it is in Inc.
3578 Expr *IncExpr;
3579 if (isOpenMPLoopBoundSharingDirective(Kind: EKind))
3580 IncExpr = S.getDistInc();
3581 else
3582 IncExpr = S.getInc();
3583
3584 // this routine is shared by 'omp distribute parallel for' and
3585 // 'omp distribute': select the right EUB expression depending on the
3586 // directive
3587 OMPLoopArguments OuterLoopArgs;
3588 OuterLoopArgs.LB = LoopArgs.LB;
3589 OuterLoopArgs.UB = LoopArgs.UB;
3590 OuterLoopArgs.ST = LoopArgs.ST;
3591 OuterLoopArgs.IL = LoopArgs.IL;
3592 OuterLoopArgs.Chunk = LoopArgs.Chunk;
3593 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3594 ? S.getCombinedEnsureUpperBound()
3595 : S.getEnsureUpperBound();
3596 OuterLoopArgs.IncExpr = IncExpr;
3597 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3598 ? S.getCombinedInit()
3599 : S.getInit();
3600 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3601 ? S.getCombinedCond()
3602 : S.getCond();
3603 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3604 ? S.getCombinedNextLowerBound()
3605 : S.getNextLowerBound();
3606 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(Kind: EKind)
3607 ? S.getCombinedNextUpperBound()
3608 : S.getNextUpperBound();
3609 OuterLoopArgs.DKind = OMPD_distribute;
3610
3611 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
3612 LoopScope, LoopArgs: OuterLoopArgs, CodeGenLoop: CodeGenLoopContent,
3613 CodeGenOrdered: emitEmptyOrdered);
3614}
3615
3616static std::pair<LValue, LValue>
3617emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
3618 const OMPExecutableDirective &S) {
3619 const OMPLoopDirective &LS = cast<OMPLoopDirective>(Val: S);
3620 LValue LB =
3621 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getLowerBoundVariable()));
3622 LValue UB =
3623 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getUpperBoundVariable()));
3624
3625 // When composing 'distribute' with 'for' (e.g. as in 'distribute
3626 // parallel for') we need to use the 'distribute'
3627 // chunk lower and upper bounds rather than the whole loop iteration
3628 // space. These are parameters to the outlined function for 'parallel'
3629 // and we copy the bounds of the previous schedule into the
3630 // the current ones.
3631 LValue PrevLB = CGF.EmitLValue(E: LS.getPrevLowerBoundVariable());
3632 LValue PrevUB = CGF.EmitLValue(E: LS.getPrevUpperBoundVariable());
3633 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
3634 lvalue: PrevLB, Loc: LS.getPrevLowerBoundVariable()->getExprLoc());
3635 PrevLBVal = CGF.EmitScalarConversion(
3636 Src: PrevLBVal, SrcTy: LS.getPrevLowerBoundVariable()->getType(),
3637 DstTy: LS.getIterationVariable()->getType(),
3638 Loc: LS.getPrevLowerBoundVariable()->getExprLoc());
3639 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
3640 lvalue: PrevUB, Loc: LS.getPrevUpperBoundVariable()->getExprLoc());
3641 PrevUBVal = CGF.EmitScalarConversion(
3642 Src: PrevUBVal, SrcTy: LS.getPrevUpperBoundVariable()->getType(),
3643 DstTy: LS.getIterationVariable()->getType(),
3644 Loc: LS.getPrevUpperBoundVariable()->getExprLoc());
3645
3646 CGF.EmitStoreOfScalar(value: PrevLBVal, lvalue: LB);
3647 CGF.EmitStoreOfScalar(value: PrevUBVal, lvalue: UB);
3648
3649 return {LB, UB};
3650}
3651
3652/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
3653/// we need to use the LB and UB expressions generated by the worksharing
3654/// code generation support, whereas in non combined situations we would
3655/// just emit 0 and the LastIteration expression
3656/// This function is necessary due to the difference of the LB and UB
3657/// types for the RT emission routines for 'for_static_init' and
3658/// 'for_dispatch_init'
3659static std::pair<llvm::Value *, llvm::Value *>
3660emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
3661 const OMPExecutableDirective &S,
3662 Address LB, Address UB) {
3663 const OMPLoopDirective &LS = cast<OMPLoopDirective>(Val: S);
3664 const Expr *IVExpr = LS.getIterationVariable();
3665 // when implementing a dynamic schedule for a 'for' combined with a
3666 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
3667 // is not normalized as each team only executes its own assigned
3668 // distribute chunk
3669 QualType IteratorTy = IVExpr->getType();
3670 llvm::Value *LBVal =
3671 CGF.EmitLoadOfScalar(Addr: LB, /*Volatile=*/false, Ty: IteratorTy, Loc: S.getBeginLoc());
3672 llvm::Value *UBVal =
3673 CGF.EmitLoadOfScalar(Addr: UB, /*Volatile=*/false, Ty: IteratorTy, Loc: S.getBeginLoc());
3674 return {LBVal, UBVal};
3675}
3676
3677static void emitDistributeParallelForDistributeInnerBoundParams(
3678 CodeGenFunction &CGF, const OMPExecutableDirective &S,
3679 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
3680 const auto &Dir = cast<OMPLoopDirective>(Val: S);
3681 LValue LB =
3682 CGF.EmitLValue(E: cast<DeclRefExpr>(Val: Dir.getCombinedLowerBoundVariable()));
3683 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
3684 V: CGF.Builder.CreateLoad(Addr: LB.getAddress()), DestTy: CGF.SizeTy, /*isSigned=*/false);
3685 CapturedVars.push_back(Elt: LBCast);
3686 LValue UB =
3687 CGF.EmitLValue(E: cast<DeclRefExpr>(Val: Dir.getCombinedUpperBoundVariable()));
3688
3689 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
3690 V: CGF.Builder.CreateLoad(Addr: UB.getAddress()), DestTy: CGF.SizeTy, /*isSigned=*/false);
3691 CapturedVars.push_back(Elt: UBCast);
3692}
3693
3694static void
3695emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
3696 const OMPLoopDirective &S,
3697 CodeGenFunction::JumpDest LoopExit) {
3698 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3699 auto &&CGInlinedWorksharingLoop = [&S, EKind](CodeGenFunction &CGF,
3700 PrePostActionTy &Action) {
3701 Action.Enter(CGF);
3702 bool HasCancel = false;
3703 if (!isOpenMPSimdDirective(DKind: EKind)) {
3704 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &S))
3705 HasCancel = D->hasCancel();
3706 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(Val: &S))
3707 HasCancel = D->hasCancel();
3708 else if (const auto *D =
3709 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &S))
3710 HasCancel = D->hasCancel();
3711 }
3712 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
3713 CGF.EmitOMPWorksharingLoop(S, EUB: S.getPrevEnsureUpperBound(),
3714 CodeGenLoopBounds: emitDistributeParallelForInnerBounds,
3715 CGDispatchBounds: emitDistributeParallelForDispatchBounds);
3716 };
3717
3718 emitCommonOMPParallelDirective(
3719 CGF, S, InnermostKind: isOpenMPSimdDirective(DKind: EKind) ? OMPD_for_simd : OMPD_for,
3720 CodeGen: CGInlinedWorksharingLoop,
3721 CodeGenBoundParameters: emitDistributeParallelForDistributeInnerBoundParams);
3722}
3723
3724void CodeGenFunction::EmitOMPDistributeParallelForDirective(
3725 const OMPDistributeParallelForDirective &S) {
3726 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3727 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
3728 IncExpr: S.getDistInc());
3729 };
3730 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3731 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_distribute, CodeGen);
3732}
3733
3734void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
3735 const OMPDistributeParallelForSimdDirective &S) {
3736 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3737 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
3738 IncExpr: S.getDistInc());
3739 };
3740 OMPLexicalScope Scope(*this, S, OMPD_parallel);
3741 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_distribute, CodeGen);
3742}
3743
3744void CodeGenFunction::EmitOMPDistributeSimdDirective(
3745 const OMPDistributeSimdDirective &S) {
3746 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3747 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
3748 };
3749 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3750 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_simd, CodeGen);
3751}
3752
3753void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
3754 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
3755 // Emit SPMD target parallel for region as a standalone region.
3756 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3757 emitOMPSimdRegion(CGF, S, Action);
3758 };
3759 llvm::Function *Fn;
3760 llvm::Constant *Addr;
3761 // Emit target region as a standalone region.
3762 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3763 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
3764 assert(Fn && Addr && "Target device function emission failed.");
3765}
3766
3767void CodeGenFunction::EmitOMPTargetSimdDirective(
3768 const OMPTargetSimdDirective &S) {
3769 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3770 emitOMPSimdRegion(CGF, S, Action);
3771 };
3772 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
3773}
3774
3775namespace {
3776struct ScheduleKindModifiersTy {
3777 OpenMPScheduleClauseKind Kind;
3778 OpenMPScheduleClauseModifier M1;
3779 OpenMPScheduleClauseModifier M2;
3780 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
3781 OpenMPScheduleClauseModifier M1,
3782 OpenMPScheduleClauseModifier M2)
3783 : Kind(Kind), M1(M1), M2(M2) {}
3784};
3785} // namespace
3786
3787bool CodeGenFunction::EmitOMPWorksharingLoop(
3788 const OMPLoopDirective &S, Expr *EUB,
3789 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
3790 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
3791 // Emit the loop iteration variable.
3792 const auto *IVExpr = cast<DeclRefExpr>(Val: S.getIterationVariable());
3793 const auto *IVDecl = cast<VarDecl>(Val: IVExpr->getDecl());
3794 EmitVarDecl(D: *IVDecl);
3795
3796 // Emit the iterations count variable.
3797 // If it is not a variable, Sema decided to calculate iterations count on each
3798 // iteration (e.g., it is foldable into a constant).
3799 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
3800 EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
3801 // Emit calculation of the iterations count.
3802 EmitIgnoredExpr(E: S.getCalcLastIteration());
3803 }
3804
3805 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3806
3807 bool HasLastprivateClause;
3808 // Check pre-condition.
3809 {
3810 OMPLoopScope PreInitScope(*this, S);
3811 // Skip the entire loop if we don't meet the precondition.
3812 // If the condition constant folds and can be elided, avoid emitting the
3813 // whole loop.
3814 bool CondConstant;
3815 llvm::BasicBlock *ContBlock = nullptr;
3816 if (ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
3817 if (!CondConstant)
3818 return false;
3819 } else {
3820 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "omp.precond.then");
3821 ContBlock = createBasicBlock(name: "omp.precond.end");
3822 emitPreCond(CGF&: *this, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
3823 TrueCount: getProfileCount(S: &S));
3824 EmitBlock(BB: ThenBlock);
3825 incrementProfileCounter(S: &S);
3826 }
3827
3828 RunCleanupsScope DoacrossCleanupScope(*this);
3829 bool Ordered = false;
3830 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
3831 if (OrderedClause->getNumForLoops())
3832 RT.emitDoacrossInit(CGF&: *this, D: S, NumIterations: OrderedClause->getLoopNumIterations());
3833 else
3834 Ordered = true;
3835 }
3836
3837 emitAlignedClause(CGF&: *this, D: S);
3838 bool HasLinears = EmitOMPLinearClauseInit(D: S);
3839 // Emit helper vars inits.
3840
3841 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
3842 LValue LB = Bounds.first;
3843 LValue UB = Bounds.second;
3844 LValue ST =
3845 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()));
3846 LValue IL =
3847 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()));
3848
3849 // Emit 'then' code.
3850 {
3851 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
3852 OMPPrivateScope LoopScope(*this);
3853 if (EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope) || HasLinears) {
3854 // Emit implicit barrier to synchronize threads and avoid data races on
3855 // initialization of firstprivate variables and post-update of
3856 // lastprivate variables.
3857 CGM.getOpenMPRuntime().emitBarrierCall(
3858 CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
3859 /*ForceSimpleCall=*/true);
3860 }
3861 EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
3862 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
3863 *this, S, EmitLValue(E: S.getIterationVariable()));
3864 HasLastprivateClause = EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
3865 EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
3866 EmitOMPPrivateLoopCounters(S, LoopScope);
3867 EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
3868 (void)LoopScope.Privatize();
3869 if (isOpenMPTargetExecutionDirective(DKind: EKind))
3870 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF&: *this, D: S);
3871
3872 // Detect the loop schedule kind and chunk.
3873 const Expr *ChunkExpr = nullptr;
3874 OpenMPScheduleTy ScheduleKind;
3875 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
3876 ScheduleKind.Schedule = C->getScheduleKind();
3877 ScheduleKind.M1 = C->getFirstScheduleModifier();
3878 ScheduleKind.M2 = C->getSecondScheduleModifier();
3879 ChunkExpr = C->getChunkSize();
3880 } else {
3881 // Default behaviour for schedule clause.
3882 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
3883 CGF&: *this, S, ScheduleKind&: ScheduleKind.Schedule, ChunkExpr);
3884 }
3885 bool HasChunkSizeOne = false;
3886 llvm::Value *Chunk = nullptr;
3887 if (ChunkExpr) {
3888 Chunk = EmitScalarExpr(E: ChunkExpr);
3889 Chunk = EmitScalarConversion(Src: Chunk, SrcTy: ChunkExpr->getType(),
3890 DstTy: S.getIterationVariable()->getType(),
3891 Loc: S.getBeginLoc());
3892 Expr::EvalResult Result;
3893 if (ChunkExpr->EvaluateAsInt(Result, Ctx: getContext())) {
3894 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
3895 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
3896 }
3897 }
3898 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
3899 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3900 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
3901 // If the static schedule kind is specified or if the ordered clause is
3902 // specified, and if no monotonic modifier is specified, the effect will
3903 // be as if the monotonic modifier was specified.
3904 bool StaticChunkedOne =
3905 RT.isStaticChunked(ScheduleKind: ScheduleKind.Schedule,
3906 /* Chunked */ Chunk != nullptr) &&
3907 HasChunkSizeOne && isOpenMPLoopBoundSharingDirective(Kind: EKind);
3908 // GPU combined `distribute parallel for`: emit a single
3909 // for_static_init with the fused distr_static_chunk + static_chunkone
3910 // schedule (enum 93). The surrounding EmitOMPDistributeLoop must skip
3911 // its distribute_static_init under the same conditions. Both sites are
3912 // guarded by canEmitGPUFusedDistSchedule() alone so they cannot
3913 // disagree; the assert guards the invariant that makes this safe today,
3914 // aka that the implicit GPU default schedule is always static chunk-one.
3915 ScheduleKind.UseFusedDistChunkSchedule =
3916 canEmitGPUFusedDistSchedule(CGM, S, DKind: EKind);
3917 assert((!ScheduleKind.UseFusedDistChunkSchedule || StaticChunkedOne) &&
3918 "fused distribute schedule requires a static chunk-one schedule");
3919 bool IsMonotonic =
3920 Ordered ||
3921 (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
3922 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
3923 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
3924 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
3925 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
3926 if ((RT.isStaticNonchunked(ScheduleKind: ScheduleKind.Schedule,
3927 /* Chunked */ Chunk != nullptr) ||
3928 StaticChunkedOne) &&
3929 !Ordered) {
3930 JumpDest LoopExit =
3931 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
3932 emitCommonSimdLoop(
3933 CGF&: *this, S,
3934 SimdInitGen: [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) {
3935 if (isOpenMPSimdDirective(DKind: EKind)) {
3936 CGF.EmitOMPSimdInit(D: S);
3937 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
3938 if (C->getKind() == OMPC_ORDER_concurrent)
3939 CGF.LoopStack.setParallel(/*Enable=*/true);
3940 }
3941 },
3942 BodyCodeGen: [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
3943 &S, ScheduleKind, LoopExit, EKind,
3944 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
3945 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
3946 // When no chunk_size is specified, the iteration space is divided
3947 // into chunks that are approximately equal in size, and at most
3948 // one chunk is distributed to each thread. Note that the size of
3949 // the chunks is unspecified in this case.
3950 CGOpenMPRuntime::StaticRTInput StaticInit(
3951 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
3952 UB.getAddress(), ST.getAddress(),
3953 StaticChunkedOne ? Chunk : nullptr);
3954 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
3955 CGF, Loc: S.getBeginLoc(), DKind: EKind, ScheduleKind, Values: StaticInit);
3956 // UB = min(UB, GlobalUB);
3957 if (!StaticChunkedOne)
3958 CGF.EmitIgnoredExpr(E: S.getEnsureUpperBound());
3959 // IV = LB;
3960 CGF.EmitIgnoredExpr(E: S.getInit());
3961 // For unchunked static schedule generate:
3962 //
3963 // while (idx <= UB) {
3964 // BODY;
3965 // ++idx;
3966 // }
3967 //
3968 // For static schedule with chunk one:
3969 //
3970 // while (IV <= PrevUB) {
3971 // BODY;
3972 // IV += ST;
3973 // }
3974 CGF.EmitOMPInnerLoop(
3975 S, RequiresCleanup: LoopScope.requiresCleanups(),
3976 LoopCond: StaticChunkedOne ? S.getCombinedParForInDistCond()
3977 : S.getCond(),
3978 IncExpr: StaticChunkedOne ? S.getDistInc() : S.getInc(),
3979 BodyGen: [&S, LoopExit](CodeGenFunction &CGF) {
3980 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
3981 },
3982 PostIncGen: [](CodeGenFunction &) {});
3983 });
3984 EmitBlock(BB: LoopExit.getBlock());
3985 // Tell the runtime we are done.
3986 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
3987 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
3988 DKind: OMPD_for);
3989 };
3990 OMPCancelStack.emitExit(CGF&: *this, Kind: EKind, CodeGen);
3991 } else {
3992 // Emit the outer loop, which requests its work chunk [LB..UB] from
3993 // runtime and runs the inner loop to process it.
3994 OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
3995 ST.getAddress(), IL.getAddress(), Chunk,
3996 EUB);
3997 LoopArguments.DKind = OMPD_for;
3998 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
3999 LoopArgs: LoopArguments, CGDispatchBounds);
4000 }
4001 if (isOpenMPSimdDirective(DKind: EKind)) {
4002 EmitOMPSimdFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4003 return CGF.Builder.CreateIsNotNull(
4004 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4005 });
4006 }
4007 EmitOMPReductionClauseFinal(
4008 D: S, /*ReductionKind=*/isOpenMPSimdDirective(DKind: EKind)
4009 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
4010 : /*Parallel only*/ OMPD_parallel);
4011 // Emit post-update of the reduction variables if IsLastIter != 0.
4012 emitPostUpdateForReductionClause(
4013 CGF&: *this, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4014 return CGF.Builder.CreateIsNotNull(
4015 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4016 });
4017 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4018 if (HasLastprivateClause)
4019 EmitOMPLastprivateClauseFinal(
4020 D: S, NoFinals: isOpenMPSimdDirective(DKind: EKind),
4021 IsLastIterCond: Builder.CreateIsNotNull(Arg: EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
4022 LoopScope.restoreMap();
4023 EmitOMPLinearClauseFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4024 return CGF.Builder.CreateIsNotNull(
4025 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4026 });
4027 }
4028 DoacrossCleanupScope.ForceCleanup();
4029 // We're now done with the loop, so jump to the continuation block.
4030 if (ContBlock) {
4031 EmitBranch(Block: ContBlock);
4032 EmitBlock(BB: ContBlock, /*IsFinished=*/true);
4033 }
4034 }
4035 return HasLastprivateClause;
4036}
4037
4038/// The following two functions generate expressions for the loop lower
4039/// and upper bounds in case of static and dynamic (dispatch) schedule
4040/// of the associated 'for' or 'distribute' loop.
4041static std::pair<LValue, LValue>
4042emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4043 const auto &LS = cast<OMPLoopDirective>(Val: S);
4044 LValue LB =
4045 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getLowerBoundVariable()));
4046 LValue UB =
4047 EmitOMPHelperVar(CGF, Helper: cast<DeclRefExpr>(Val: LS.getUpperBoundVariable()));
4048 return {LB, UB};
4049}
4050
4051/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
4052/// consider the lower and upper bound expressions generated by the
4053/// worksharing loop support, but we use 0 and the iteration space size as
4054/// constants
4055static std::pair<llvm::Value *, llvm::Value *>
4056emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
4057 Address LB, Address UB) {
4058 const auto &LS = cast<OMPLoopDirective>(Val: S);
4059 const Expr *IVExpr = LS.getIterationVariable();
4060 const unsigned IVSize = CGF.getContext().getTypeSize(T: IVExpr->getType());
4061 llvm::Value *LBVal = CGF.Builder.getIntN(N: IVSize, C: 0);
4062 llvm::Value *UBVal = CGF.EmitScalarExpr(E: LS.getLastIteration());
4063 return {LBVal, UBVal};
4064}
4065
4066/// Emits internal temp array declarations for the directive with inscan
4067/// reductions.
4068/// The code is the following:
4069/// \code
4070/// size num_iters = <num_iters>;
4071/// <type> buffer[num_iters];
4072/// \endcode
4073static void emitScanBasedDirectiveDecls(
4074 CodeGenFunction &CGF, const OMPLoopDirective &S,
4075 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4076 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4077 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4078 SmallVector<const Expr *, 4> Shareds;
4079 SmallVector<const Expr *, 4> Privates;
4080 SmallVector<const Expr *, 4> ReductionOps;
4081 SmallVector<const Expr *, 4> CopyArrayTemps;
4082 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4083 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4084 "Only inscan reductions are expected.");
4085 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
4086 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4087 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
4088 CopyArrayTemps.append(in_start: C->copy_array_temps().begin(),
4089 in_end: C->copy_array_temps().end());
4090 }
4091 {
4092 // Emit buffers for each reduction variables.
4093 // ReductionCodeGen is required to emit correctly the code for array
4094 // reductions.
4095 ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
4096 unsigned Count = 0;
4097 auto *ITA = CopyArrayTemps.begin();
4098 for (const Expr *IRef : Privates) {
4099 const auto *PrivateVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IRef)->getDecl());
4100 // Emit variably modified arrays, used for arrays/array sections
4101 // reductions.
4102 if (PrivateVD->getType()->isVariablyModifiedType()) {
4103 RedCG.emitSharedOrigLValue(CGF, N: Count);
4104 RedCG.emitAggregateType(CGF, N: Count);
4105 }
4106 CodeGenFunction::OpaqueValueMapping DimMapping(
4107 CGF,
4108 cast<OpaqueValueExpr>(
4109 Val: cast<VariableArrayType>(Val: (*ITA)->getType()->getAsArrayTypeUnsafe())
4110 ->getSizeExpr()),
4111 RValue::get(V: OMPScanNumIterations));
4112 // Emit temp buffer.
4113 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ITA)->getDecl()));
4114 ++ITA;
4115 ++Count;
4116 }
4117 }
4118}
4119
4120/// Copies final inscan reductions values to the original variables.
4121/// The code is the following:
4122/// \code
4123/// <orig_var> = buffer[num_iters-1];
4124/// \endcode
4125static void emitScanBasedDirectiveFinals(
4126 CodeGenFunction &CGF, const OMPLoopDirective &S,
4127 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
4128 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4129 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4130 SmallVector<const Expr *, 4> Shareds;
4131 SmallVector<const Expr *, 4> LHSs;
4132 SmallVector<const Expr *, 4> RHSs;
4133 SmallVector<const Expr *, 4> Privates;
4134 SmallVector<const Expr *, 4> CopyOps;
4135 SmallVector<const Expr *, 4> CopyArrayElems;
4136 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4137 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4138 "Only inscan reductions are expected.");
4139 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
4140 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
4141 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
4142 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4143 CopyOps.append(in_start: C->copy_ops().begin(), in_end: C->copy_ops().end());
4144 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
4145 in_end: C->copy_array_elems().end());
4146 }
4147 // Create temp var and copy LHS value to this temp value.
4148 // LHS = TMP[LastIter];
4149 llvm::Value *OMPLast = CGF.Builder.CreateNSWSub(
4150 LHS: OMPScanNumIterations,
4151 RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1, /*isSigned=*/IsSigned: false));
4152 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
4153 const Expr *PrivateExpr = Privates[I];
4154 const Expr *OrigExpr = Shareds[I];
4155 const Expr *CopyArrayElem = CopyArrayElems[I];
4156 CodeGenFunction::OpaqueValueMapping IdxMapping(
4157 CGF,
4158 cast<OpaqueValueExpr>(
4159 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4160 RValue::get(V: OMPLast));
4161 LValue DestLVal = CGF.EmitLValue(E: OrigExpr);
4162 LValue SrcLVal = CGF.EmitLValue(E: CopyArrayElem);
4163 CGF.EmitOMPCopy(
4164 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
4165 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
4166 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
4167 }
4168}
4169
4170/// Emits the code for the directive with inscan reductions.
4171/// The code is the following:
4172/// \code
4173/// #pragma omp ...
4174/// for (i: 0..<num_iters>) {
4175/// <input phase>;
4176/// buffer[i] = red;
4177/// }
4178/// #pragma omp master // in parallel region
4179/// for (int k = 0; k != ceil(log2(num_iters)); ++k)
4180/// for (size cnt = last_iter; cnt >= pow(2, k); --k)
4181/// buffer[i] op= buffer[i-pow(2,k)];
4182/// #pragma omp barrier // in parallel region
4183/// #pragma omp ...
4184/// for (0..<num_iters>) {
4185/// red = InclusiveScan ? buffer[i] : buffer[i-1];
4186/// <scan phase>;
4187/// }
4188/// \endcode
4189static void emitScanBasedDirective(
4190 CodeGenFunction &CGF, const OMPLoopDirective &S,
4191 llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
4192 llvm::function_ref<void(CodeGenFunction &)> FirstGen,
4193 llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
4194 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
4195 V: NumIteratorsGen(CGF), DestTy: CGF.SizeTy, /*isSigned=*/false);
4196 SmallVector<const Expr *, 4> Privates;
4197 SmallVector<const Expr *, 4> ReductionOps;
4198 SmallVector<const Expr *, 4> LHSs;
4199 SmallVector<const Expr *, 4> RHSs;
4200 SmallVector<const Expr *, 4> CopyArrayElems;
4201 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
4202 assert(C->getModifier() == OMPC_REDUCTION_inscan &&
4203 "Only inscan reductions are expected.");
4204 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
4205 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
4206 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
4207 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
4208 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
4209 in_end: C->copy_array_elems().end());
4210 }
4211 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
4212 {
4213 // Emit loop with input phase:
4214 // #pragma omp ...
4215 // for (i: 0..<num_iters>) {
4216 // <input phase>;
4217 // buffer[i] = red;
4218 // }
4219 CGF.OMPFirstScanLoop = true;
4220 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4221 FirstGen(CGF);
4222 }
4223 // #pragma omp barrier // in parallel region
4224 auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
4225 &ReductionOps,
4226 &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
4227 Action.Enter(CGF);
4228 // Emit prefix reduction:
4229 // #pragma omp master // in parallel region
4230 // for (int k = 0; k <= ceil(log2(n)); ++k)
4231 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
4232 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(name: "omp.outer.log.scan.body");
4233 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "omp.outer.log.scan.exit");
4234 llvm::Function *F =
4235 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::log2, Tys: CGF.DoubleTy);
4236 llvm::Value *Arg =
4237 CGF.Builder.CreateUIToFP(V: OMPScanNumIterations, DestTy: CGF.DoubleTy);
4238 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(callee: F, args: Arg);
4239 F = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ceil, Tys: CGF.DoubleTy);
4240 LogVal = CGF.EmitNounwindRuntimeCall(callee: F, args: LogVal);
4241 LogVal = CGF.Builder.CreateFPToUI(V: LogVal, DestTy: CGF.IntTy);
4242 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
4243 LHS: OMPScanNumIterations, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1));
4244 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: S.getBeginLoc());
4245 CGF.EmitBlock(BB: LoopBB);
4246 auto *Counter = CGF.Builder.CreatePHI(Ty: CGF.IntTy, NumReservedValues: 2);
4247 // size pow2k = 1;
4248 auto *Pow2K = CGF.Builder.CreatePHI(Ty: CGF.SizeTy, NumReservedValues: 2);
4249 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0), BB: InputBB);
4250 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), BB: InputBB);
4251 // for (size i = n - 1; i >= 2 ^ k; --i)
4252 // tmp[i] op= tmp[i-pow2k];
4253 llvm::BasicBlock *InnerLoopBB =
4254 CGF.createBasicBlock(name: "omp.inner.log.scan.body");
4255 llvm::BasicBlock *InnerExitBB =
4256 CGF.createBasicBlock(name: "omp.inner.log.scan.exit");
4257 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
4258 CGF.Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
4259 CGF.EmitBlock(BB: InnerLoopBB);
4260 auto *IVal = CGF.Builder.CreatePHI(Ty: CGF.SizeTy, NumReservedValues: 2);
4261 IVal->addIncoming(V: NMin1, BB: LoopBB);
4262 {
4263 CodeGenFunction::OMPPrivateScope PrivScope(CGF);
4264 auto *ILHS = LHSs.begin();
4265 auto *IRHS = RHSs.begin();
4266 for (const Expr *CopyArrayElem : CopyArrayElems) {
4267 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
4268 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
4269 Address LHSAddr = Address::invalid();
4270 {
4271 CodeGenFunction::OpaqueValueMapping IdxMapping(
4272 CGF,
4273 cast<OpaqueValueExpr>(
4274 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4275 RValue::get(V: IVal));
4276 LHSAddr = CGF.EmitLValue(E: CopyArrayElem).getAddress();
4277 }
4278 PrivScope.addPrivate(LocalVD: LHSVD, Addr: LHSAddr);
4279 Address RHSAddr = Address::invalid();
4280 {
4281 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(LHS: IVal, RHS: Pow2K);
4282 CodeGenFunction::OpaqueValueMapping IdxMapping(
4283 CGF,
4284 cast<OpaqueValueExpr>(
4285 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
4286 RValue::get(V: OffsetIVal));
4287 RHSAddr = CGF.EmitLValue(E: CopyArrayElem).getAddress();
4288 }
4289 PrivScope.addPrivate(LocalVD: RHSVD, Addr: RHSAddr);
4290 ++ILHS;
4291 ++IRHS;
4292 }
4293 PrivScope.Privatize();
4294 CGF.CGM.getOpenMPRuntime().emitReduction(
4295 CGF, Loc: S.getEndLoc(), Privates, LHSExprs: LHSs, RHSExprs: RHSs, ReductionOps,
4296 Options: {/*WithNowait=*/true, /*SimpleReduction=*/true,
4297 /*IsPrivateVarReduction*/ {}, .ReductionKind: OMPD_unknown});
4298 }
4299 llvm::Value *NextIVal =
4300 CGF.Builder.CreateNUWSub(LHS: IVal, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1));
4301 IVal->addIncoming(V: NextIVal, BB: CGF.Builder.GetInsertBlock());
4302 CmpI = CGF.Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
4303 CGF.Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
4304 CGF.EmitBlock(BB: InnerExitBB);
4305 llvm::Value *Next =
4306 CGF.Builder.CreateNUWAdd(LHS: Counter, RHS: llvm::ConstantInt::get(Ty: CGF.IntTy, V: 1));
4307 Counter->addIncoming(V: Next, BB: CGF.Builder.GetInsertBlock());
4308 // pow2k <<= 1;
4309 llvm::Value *NextPow2K =
4310 CGF.Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
4311 Pow2K->addIncoming(V: NextPow2K, BB: CGF.Builder.GetInsertBlock());
4312 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
4313 CGF.Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
4314 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: S.getEndLoc());
4315 CGF.EmitBlock(BB: ExitBB);
4316 };
4317 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4318 if (isOpenMPParallelDirective(DKind: EKind)) {
4319 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
4320 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4321 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4322 /*ForceSimpleCall=*/true);
4323 } else {
4324 RegionCodeGenTy RCG(CodeGen);
4325 RCG(CGF);
4326 }
4327
4328 CGF.OMPFirstScanLoop = false;
4329 SecondGen(CGF);
4330}
4331
4332static bool emitWorksharingDirective(CodeGenFunction &CGF,
4333 const OMPLoopDirective &S,
4334 bool HasCancel) {
4335 bool HasLastprivates;
4336 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4337 if (llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
4338 P: [](const OMPReductionClause *C) {
4339 return C->getModifier() == OMPC_REDUCTION_inscan;
4340 })) {
4341 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
4342 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
4343 OMPLoopScope LoopScope(CGF, S);
4344 return CGF.EmitScalarExpr(E: S.getNumIterations());
4345 };
4346 const auto &&FirstGen = [&S, HasCancel, EKind](CodeGenFunction &CGF) {
4347 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4348 (void)CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4349 CodeGenLoopBounds: emitForLoopBounds,
4350 CGDispatchBounds: emitDispatchForLoopBounds);
4351 // Emit an implicit barrier at the end.
4352 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc: S.getBeginLoc(),
4353 Kind: OMPD_for);
4354 };
4355 const auto &&SecondGen = [&S, HasCancel, EKind,
4356 &HasLastprivates](CodeGenFunction &CGF) {
4357 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4358 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4359 CodeGenLoopBounds: emitForLoopBounds,
4360 CGDispatchBounds: emitDispatchForLoopBounds);
4361 };
4362 if (!isOpenMPParallelDirective(DKind: EKind))
4363 emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
4364 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
4365 if (!isOpenMPParallelDirective(DKind: EKind))
4366 emitScanBasedDirectiveFinals(CGF, S, NumIteratorsGen);
4367 } else {
4368 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);
4369 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(),
4370 CodeGenLoopBounds: emitForLoopBounds,
4371 CGDispatchBounds: emitDispatchForLoopBounds);
4372 }
4373 return HasLastprivates;
4374}
4375
4376// Pass OMPLoopDirective (instead of OMPForDirective) to make this check
4377// available for "loop bind(parallel)", which maps to "for".
4378static bool isForSupportedByOpenMPIRBuilder(const OMPLoopDirective &S,
4379 bool HasCancel) {
4380 if (HasCancel)
4381 return false;
4382 for (OMPClause *C : S.clauses()) {
4383 if (isa<OMPNowaitClause, OMPBindClause>(Val: C))
4384 continue;
4385
4386 if (auto *SC = dyn_cast<OMPScheduleClause>(Val: C)) {
4387 if (SC->getFirstScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4388 return false;
4389 if (SC->getSecondScheduleModifier() != OMPC_SCHEDULE_MODIFIER_unknown)
4390 return false;
4391 switch (SC->getScheduleKind()) {
4392 case OMPC_SCHEDULE_auto:
4393 case OMPC_SCHEDULE_dynamic:
4394 case OMPC_SCHEDULE_runtime:
4395 case OMPC_SCHEDULE_guided:
4396 case OMPC_SCHEDULE_static:
4397 continue;
4398 case OMPC_SCHEDULE_unknown:
4399 return false;
4400 }
4401 }
4402
4403 return false;
4404 }
4405
4406 return true;
4407}
4408
4409static llvm::omp::ScheduleKind
4410convertClauseKindToSchedKind(OpenMPScheduleClauseKind ScheduleClauseKind) {
4411 switch (ScheduleClauseKind) {
4412 case OMPC_SCHEDULE_unknown:
4413 return llvm::omp::OMP_SCHEDULE_Default;
4414 case OMPC_SCHEDULE_auto:
4415 return llvm::omp::OMP_SCHEDULE_Auto;
4416 case OMPC_SCHEDULE_dynamic:
4417 return llvm::omp::OMP_SCHEDULE_Dynamic;
4418 case OMPC_SCHEDULE_guided:
4419 return llvm::omp::OMP_SCHEDULE_Guided;
4420 case OMPC_SCHEDULE_runtime:
4421 return llvm::omp::OMP_SCHEDULE_Runtime;
4422 case OMPC_SCHEDULE_static:
4423 return llvm::omp::OMP_SCHEDULE_Static;
4424 }
4425 llvm_unreachable("Unhandled schedule kind");
4426}
4427
4428// Pass OMPLoopDirective (instead of OMPForDirective) to make this function
4429// available for "loop bind(parallel)", which maps to "for".
4430static void emitOMPForDirective(const OMPLoopDirective &S, CodeGenFunction &CGF,
4431 CodeGenModule &CGM, bool HasCancel) {
4432 bool HasLastprivates = false;
4433 bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder &&
4434 isForSupportedByOpenMPIRBuilder(S, HasCancel);
4435 auto &&CodeGen = [&S, &CGM, HasCancel, &HasLastprivates,
4436 UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
4437 // Use the OpenMPIRBuilder if enabled.
4438 if (UseOMPIRBuilder) {
4439 bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
4440
4441 llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;
4442 llvm::Value *ChunkSize = nullptr;
4443 if (auto *SchedClause = S.getSingleClause<OMPScheduleClause>()) {
4444 SchedKind =
4445 convertClauseKindToSchedKind(ScheduleClauseKind: SchedClause->getScheduleKind());
4446 if (const Expr *ChunkSizeExpr = SchedClause->getChunkSize())
4447 ChunkSize = CGF.EmitScalarExpr(E: ChunkSizeExpr);
4448 }
4449
4450 // Emit the associated statement and get its loop representation.
4451 const Stmt *Inner = S.getRawStmt();
4452 llvm::CanonicalLoopInfo *CLI =
4453 CGF.EmitOMPCollapsedCanonicalLoopNest(S: Inner, Depth: 1);
4454
4455 llvm::OpenMPIRBuilder &OMPBuilder =
4456 CGM.getOpenMPRuntime().getOMPBuilder();
4457 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4458 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
4459 cantFail(ValOrErr: OMPBuilder.applyWorkshareLoop(
4460 DL: CGF.Builder.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,
4461 SchedKind, ChunkSize, /*HasSimdModifier=*/false,
4462 /*HasMonotonicModifier=*/false, /*HasNonmonotonicModifier=*/false,
4463 /*HasOrderedClause=*/false));
4464 return;
4465 }
4466
4467 HasLastprivates = emitWorksharingDirective(CGF, S, HasCancel);
4468 };
4469 {
4470 auto LPCRegion =
4471 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
4472 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
4473 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_for, CodeGen,
4474 HasCancel);
4475 }
4476
4477 if (!UseOMPIRBuilder) {
4478 // Emit an implicit barrier at the end.
4479 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4480 CGM.getOpenMPRuntime().emitBarrierCall(CGF, Loc: S.getBeginLoc(), Kind: OMPD_for);
4481 }
4482 // Check for outer lastprivate conditional update.
4483 checkForLastprivateConditionalUpdate(CGF, S);
4484}
4485
4486void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
4487 return emitOMPForDirective(S, CGF&: *this, CGM, HasCancel: S.hasCancel());
4488}
4489
4490void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
4491 bool HasLastprivates = false;
4492 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
4493 PrePostActionTy &) {
4494 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
4495 };
4496 {
4497 auto LPCRegion =
4498 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4499 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4500 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_simd, CodeGen);
4501 }
4502
4503 // Emit an implicit barrier at the end.
4504 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
4505 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_for);
4506 // Check for outer lastprivate conditional update.
4507 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4508}
4509
4510static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
4511 const Twine &Name,
4512 llvm::Value *Init = nullptr) {
4513 LValue LVal = CGF.MakeAddrLValue(Addr: CGF.CreateMemTemp(T: Ty, Name), T: Ty);
4514 if (Init)
4515 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Init), Dst: LVal, /*isInit*/ true);
4516 return LVal;
4517}
4518
4519void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
4520 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4521 const auto *CS = dyn_cast<CompoundStmt>(Val: CapturedStmt);
4522 bool HasLastprivates = false;
4523 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
4524 auto &&CodeGen = [&S, CapturedStmt, CS, EKind,
4525 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
4526 const ASTContext &C = CGF.getContext();
4527 QualType KmpInt32Ty =
4528 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4529 // Emit helper vars inits.
4530 LValue LB = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.lb.",
4531 Init: CGF.Builder.getInt32(C: 0));
4532 llvm::ConstantInt *GlobalUBVal = CS != nullptr
4533 ? CGF.Builder.getInt32(C: CS->size() - 1)
4534 : CGF.Builder.getInt32(C: 0);
4535 LValue UB =
4536 createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.ub.", Init: GlobalUBVal);
4537 LValue ST = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.st.",
4538 Init: CGF.Builder.getInt32(C: 1));
4539 LValue IL = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.il.",
4540 Init: CGF.Builder.getInt32(C: 0));
4541 // Loop counter.
4542 LValue IV = createSectionLVal(CGF, Ty: KmpInt32Ty, Name: ".omp.sections.iv.");
4543 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4544 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
4545 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
4546 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
4547 // Generate condition for loop.
4548 BinaryOperator *Cond = BinaryOperator::Create(
4549 C, lhs: &IVRefExpr, rhs: &UBRefExpr, opc: BO_LE, ResTy: C.BoolTy, VK: VK_PRValue, OK: OK_Ordinary,
4550 opLoc: S.getBeginLoc(), FPFeatures: FPOptionsOverride());
4551 // Increment for loop counter.
4552 UnaryOperator *Inc = UnaryOperator::Create(
4553 C, input: &IVRefExpr, opc: UO_PreInc, type: KmpInt32Ty, VK: VK_PRValue, OK: OK_Ordinary,
4554 l: S.getBeginLoc(), CanOverflow: true, FPFeatures: FPOptionsOverride());
4555 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
4556 // Iterate through all sections and emit a switch construct:
4557 // switch (IV) {
4558 // case 0:
4559 // <SectionStmt[0]>;
4560 // break;
4561 // ...
4562 // case <NumSection> - 1:
4563 // <SectionStmt[<NumSection> - 1]>;
4564 // break;
4565 // }
4566 // .omp.sections.exit:
4567 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".omp.sections.exit");
4568 llvm::SwitchInst *SwitchStmt =
4569 CGF.Builder.CreateSwitch(V: CGF.EmitLoadOfScalar(lvalue: IV, Loc: S.getBeginLoc()),
4570 Dest: ExitBB, NumCases: CS == nullptr ? 1 : CS->size());
4571 if (CS) {
4572 unsigned CaseNumber = 0;
4573 for (const Stmt *SubStmt : CS->children()) {
4574 auto CaseBB = CGF.createBasicBlock(name: ".omp.sections.case");
4575 CGF.EmitBlock(BB: CaseBB);
4576 SwitchStmt->addCase(OnVal: CGF.Builder.getInt32(C: CaseNumber), Dest: CaseBB);
4577 CGF.EmitStmt(S: SubStmt);
4578 CGF.EmitBranch(Block: ExitBB);
4579 ++CaseNumber;
4580 }
4581 } else {
4582 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(name: ".omp.sections.case");
4583 CGF.EmitBlock(BB: CaseBB);
4584 SwitchStmt->addCase(OnVal: CGF.Builder.getInt32(C: 0), Dest: CaseBB);
4585 CGF.EmitStmt(S: CapturedStmt);
4586 CGF.EmitBranch(Block: ExitBB);
4587 }
4588 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
4589 };
4590
4591 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
4592 if (CGF.EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope)) {
4593 // Emit implicit barrier to synchronize threads and avoid data races on
4594 // initialization of firstprivate variables and post-update of lastprivate
4595 // variables.
4596 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
4597 CGF, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
4598 /*ForceSimpleCall=*/true);
4599 }
4600 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
4601 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
4602 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
4603 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
4604 (void)LoopScope.Privatize();
4605 if (isOpenMPTargetExecutionDirective(DKind: EKind))
4606 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
4607
4608 // Emit static non-chunked loop.
4609 OpenMPScheduleTy ScheduleKind;
4610 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
4611 CGOpenMPRuntime::StaticRTInput StaticInit(
4612 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
4613 LB.getAddress(), UB.getAddress(), ST.getAddress());
4614 CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, Loc: S.getBeginLoc(), DKind: EKind,
4615 ScheduleKind, Values: StaticInit);
4616 // UB = min(UB, GlobalUB);
4617 llvm::Value *UBVal = CGF.EmitLoadOfScalar(lvalue: UB, Loc: S.getBeginLoc());
4618 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
4619 C: CGF.Builder.CreateICmpSLT(LHS: UBVal, RHS: GlobalUBVal), True: UBVal, False: GlobalUBVal);
4620 CGF.EmitStoreOfScalar(value: MinUBGlobalUB, lvalue: UB);
4621 // IV = LB;
4622 CGF.EmitStoreOfScalar(value: CGF.EmitLoadOfScalar(lvalue: LB, Loc: S.getBeginLoc()), lvalue: IV);
4623 // while (idx <= UB) { BODY; ++idx; }
4624 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, LoopCond: Cond, IncExpr: Inc, BodyGen,
4625 PostIncGen: [](CodeGenFunction &) {});
4626 // Tell the runtime we are done.
4627 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
4628 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, Loc: S.getEndLoc(),
4629 DKind: OMPD_sections);
4630 };
4631 CGF.OMPCancelStack.emitExit(CGF, Kind: EKind, CodeGen);
4632 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
4633 // Emit post-update of the reduction variables if IsLastIter != 0.
4634 emitPostUpdateForReductionClause(CGF, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
4635 return CGF.Builder.CreateIsNotNull(
4636 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
4637 });
4638
4639 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4640 if (HasLastprivates)
4641 CGF.EmitOMPLastprivateClauseFinal(
4642 D: S, /*NoFinals=*/false,
4643 IsLastIterCond: CGF.Builder.CreateIsNotNull(
4644 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
4645 };
4646
4647 bool HasCancel = false;
4648 if (auto *OSD = dyn_cast<OMPSectionsDirective>(Val: &S))
4649 HasCancel = OSD->hasCancel();
4650 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &S))
4651 HasCancel = OPSD->hasCancel();
4652 OMPCancelStackRAII CancelRegion(*this, EKind, HasCancel);
4653 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_sections, CodeGen,
4654 HasCancel);
4655 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
4656 // clause. Otherwise the barrier will be generated by the codegen for the
4657 // directive.
4658 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
4659 // Emit implicit barrier to synchronize threads and avoid data races on
4660 // initialization of firstprivate variables.
4661 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(),
4662 Kind: OMPD_unknown);
4663 }
4664}
4665
4666void CodeGenFunction::EmitOMPScopeDirective(const OMPScopeDirective &S) {
4667 {
4668 // Emit code for 'scope' region
4669 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4670 Action.Enter(CGF);
4671 OMPPrivateScope PrivateScope(CGF);
4672 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
4673 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
4674 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
4675 (void)PrivateScope.Privatize();
4676 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
4677 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
4678 };
4679 auto LPCRegion =
4680 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4681 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4682 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_scope, CodeGen);
4683 }
4684 // Emit an implicit barrier at the end.
4685 if (!S.getSingleClause<OMPNowaitClause>()) {
4686 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_scope);
4687 }
4688 // Check for outer lastprivate conditional update.
4689 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4690}
4691
4692void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
4693 if (CGM.getLangOpts().OpenMPIRBuilder) {
4694 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4695 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4696 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4697
4698 auto FiniCB = [](InsertPointTy IP) {
4699 // Don't FinalizeOMPRegion because this is done inside of OMPIRBuilder for
4700 // sections.
4701 return llvm::Error::success();
4702 };
4703
4704 const CapturedStmt *ICS = S.getInnermostCapturedStmt();
4705 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
4706 const auto *CS = dyn_cast<CompoundStmt>(Val: CapturedStmt);
4707 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4708 if (CS) {
4709 for (const Stmt *SubStmt : CS->children()) {
4710 auto SectionCB = [this, SubStmt](
4711 InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4712 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4713 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(CGF&: *this, RegionBodyStmt: SubStmt, AllocaIP: AllocIP,
4714 CodeGenIP, RegionName: "section");
4715 return llvm::Error::success();
4716 };
4717 SectionCBVector.push_back(Elt: SectionCB);
4718 }
4719 } else {
4720 auto SectionCB =
4721 [this, CapturedStmt](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4722 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4723 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4724 CGF&: *this, RegionBodyStmt: CapturedStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "section");
4725 return llvm::Error::success();
4726 };
4727 SectionCBVector.push_back(Elt: SectionCB);
4728 }
4729
4730 // Privatization callback that performs appropriate action for
4731 // shared/private/firstprivate/lastprivate/copyin/... variables.
4732 //
4733 // TODO: This defaults to shared right now.
4734 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4735 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4736 // The next line is appropriate only for variables (Val) with the
4737 // data-sharing attribute "shared".
4738 ReplVal = &Val;
4739
4740 return CodeGenIP;
4741 };
4742
4743 CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
4744 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
4745 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
4746 AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
4747 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4748 cantFail(ValOrErr: OMPBuilder.createSections(
4749 Loc: Builder, AllocaIP, SectionCBs: SectionCBVector, PrivCB, FiniCB, IsCancellable: S.hasCancel(),
4750 IsNowait: S.getSingleClause<OMPNowaitClause>()));
4751 Builder.restoreIP(IP: AfterIP);
4752 return;
4753 }
4754 {
4755 auto LPCRegion =
4756 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4757 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4758 EmitSections(S);
4759 }
4760 // Emit an implicit barrier at the end.
4761 if (!S.getSingleClause<OMPNowaitClause>()) {
4762 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(),
4763 Kind: OMPD_sections);
4764 }
4765 // Check for outer lastprivate conditional update.
4766 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4767}
4768
4769void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
4770 if (CGM.getLangOpts().OpenMPIRBuilder) {
4771 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4772 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4773
4774 const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
4775 auto FiniCB = [this](InsertPointTy IP) {
4776 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
4777 return llvm::Error::success();
4778 };
4779
4780 auto BodyGenCB = [SectionRegionBodyStmt,
4781 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4782 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4783 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4784 CGF&: *this, RegionBodyStmt: SectionRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "section");
4785 return llvm::Error::success();
4786 };
4787
4788 LexicalScope Scope(*this, S.getSourceRange());
4789 EmitStopPoint(S: &S);
4790 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4791 cantFail(ValOrErr: OMPBuilder.createSection(Loc: Builder, BodyGenCB, FiniCB));
4792 Builder.restoreIP(IP: AfterIP);
4793
4794 return;
4795 }
4796 LexicalScope Scope(*this, S.getSourceRange());
4797 EmitStopPoint(S: &S);
4798 EmitStmt(S: S.getAssociatedStmt());
4799}
4800
4801void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
4802 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
4803 llvm::SmallVector<const Expr *, 8> DestExprs;
4804 llvm::SmallVector<const Expr *, 8> SrcExprs;
4805 llvm::SmallVector<const Expr *, 8> AssignmentOps;
4806 // Check if there are any 'copyprivate' clauses associated with this
4807 // 'single' construct.
4808 // Build a list of copyprivate variables along with helper expressions
4809 // (<source>, <destination>, <destination>=<source> expressions)
4810 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
4811 CopyprivateVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
4812 DestExprs.append(in_start: C->destination_exprs().begin(),
4813 in_end: C->destination_exprs().end());
4814 SrcExprs.append(in_start: C->source_exprs().begin(), in_end: C->source_exprs().end());
4815 AssignmentOps.append(in_start: C->assignment_ops().begin(),
4816 in_end: C->assignment_ops().end());
4817 }
4818 // Emit code for 'single' region along with 'copyprivate' clauses
4819 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4820 Action.Enter(CGF);
4821 OMPPrivateScope SingleScope(CGF);
4822 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope&: SingleScope);
4823 CGF.EmitOMPPrivateClause(D: S, PrivateScope&: SingleScope);
4824 (void)SingleScope.Privatize();
4825 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
4826 };
4827 {
4828 auto LPCRegion =
4829 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
4830 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4831 CGM.getOpenMPRuntime().emitSingleRegion(CGF&: *this, SingleOpGen: CodeGen, Loc: S.getBeginLoc(),
4832 CopyprivateVars, DestExprs,
4833 SrcExprs, AssignmentOps);
4834 }
4835 // Emit an implicit barrier at the end (to avoid data race on firstprivate
4836 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
4837 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
4838 CGM.getOpenMPRuntime().emitBarrierCall(
4839 CGF&: *this, Loc: S.getBeginLoc(),
4840 Kind: S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
4841 }
4842 // Check for outer lastprivate conditional update.
4843 checkForLastprivateConditionalUpdate(CGF&: *this, S);
4844}
4845
4846static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4847 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4848 Action.Enter(CGF);
4849 CGF.EmitStmt(S: S.getRawStmt());
4850 };
4851 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
4852}
4853
4854void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
4855 if (CGM.getLangOpts().OpenMPIRBuilder) {
4856 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4857 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4858
4859 const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
4860
4861 auto FiniCB = [this](InsertPointTy IP) {
4862 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
4863 return llvm::Error::success();
4864 };
4865
4866 auto BodyGenCB = [MasterRegionBodyStmt,
4867 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4868 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4869 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4870 CGF&: *this, RegionBodyStmt: MasterRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "master");
4871 return llvm::Error::success();
4872 };
4873
4874 LexicalScope Scope(*this, S.getSourceRange());
4875 EmitStopPoint(S: &S);
4876 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4877 cantFail(ValOrErr: OMPBuilder.createMaster(Loc: Builder, BodyGenCB, FiniCB));
4878 Builder.restoreIP(IP: AfterIP);
4879
4880 return;
4881 }
4882 LexicalScope Scope(*this, S.getSourceRange());
4883 EmitStopPoint(S: &S);
4884 emitMaster(CGF&: *this, S);
4885}
4886
4887static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
4888 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4889 Action.Enter(CGF);
4890 CGF.EmitStmt(S: S.getRawStmt());
4891 };
4892 Expr *Filter = nullptr;
4893 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4894 Filter = FilterClause->getThreadID();
4895 CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: CodeGen, Loc: S.getBeginLoc(),
4896 Filter);
4897}
4898
4899void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {
4900 if (CGM.getLangOpts().OpenMPIRBuilder) {
4901 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4902 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4903
4904 const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
4905 const Expr *Filter = nullptr;
4906 if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
4907 Filter = FilterClause->getThreadID();
4908 llvm::Value *FilterVal = Filter
4909 ? EmitScalarExpr(E: Filter, IgnoreResultAssign: CGM.Int32Ty)
4910 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
4911
4912 auto FiniCB = [this](InsertPointTy IP) {
4913 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
4914 return llvm::Error::success();
4915 };
4916
4917 auto BodyGenCB = [MaskedRegionBodyStmt,
4918 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4919 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4920 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4921 CGF&: *this, RegionBodyStmt: MaskedRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "masked");
4922 return llvm::Error::success();
4923 };
4924
4925 LexicalScope Scope(*this, S.getSourceRange());
4926 EmitStopPoint(S: &S);
4927 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
4928 ValOrErr: OMPBuilder.createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal));
4929 Builder.restoreIP(IP: AfterIP);
4930
4931 return;
4932 }
4933 LexicalScope Scope(*this, S.getSourceRange());
4934 EmitStopPoint(S: &S);
4935 emitMasked(CGF&: *this, S);
4936}
4937
4938void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
4939 if (CGM.getLangOpts().OpenMPIRBuilder) {
4940 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
4941 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4942
4943 const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
4944 const Expr *Hint = nullptr;
4945 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4946 Hint = HintClause->getHint();
4947
4948 // TODO: This is slightly different from what's currently being done in
4949 // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
4950 // about typing is final.
4951 llvm::Value *HintInst = nullptr;
4952 if (Hint)
4953 HintInst =
4954 Builder.CreateIntCast(V: EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, isSigned: false);
4955
4956 auto FiniCB = [this](InsertPointTy IP) {
4957 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
4958 return llvm::Error::success();
4959 };
4960
4961 auto BodyGenCB = [CriticalRegionBodyStmt,
4962 this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
4963 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
4964 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
4965 CGF&: *this, RegionBodyStmt: CriticalRegionBodyStmt, AllocaIP: AllocIP, CodeGenIP, RegionName: "critical");
4966 return llvm::Error::success();
4967 };
4968
4969 LexicalScope Scope(*this, S.getSourceRange());
4970 EmitStopPoint(S: &S);
4971 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
4972 cantFail(ValOrErr: OMPBuilder.createCritical(Loc: Builder, BodyGenCB, FiniCB,
4973 CriticalName: S.getDirectiveName().getAsString(),
4974 HintInst));
4975 Builder.restoreIP(IP: AfterIP);
4976
4977 return;
4978 }
4979
4980 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4981 Action.Enter(CGF);
4982 CGF.EmitStmt(S: S.getAssociatedStmt());
4983 };
4984 const Expr *Hint = nullptr;
4985 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
4986 Hint = HintClause->getHint();
4987 LexicalScope Scope(*this, S.getSourceRange());
4988 EmitStopPoint(S: &S);
4989 CGM.getOpenMPRuntime().emitCriticalRegion(CGF&: *this,
4990 CriticalName: S.getDirectiveName().getAsString(),
4991 CriticalOpGen: CodeGen, Loc: S.getBeginLoc(), Hint);
4992}
4993
4994void CodeGenFunction::EmitOMPParallelForDirective(
4995 const OMPParallelForDirective &S) {
4996 // Emit directive as a combined directive that consists of two implicit
4997 // directives: 'parallel' with 'for' directive.
4998 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4999 Action.Enter(CGF);
5000 emitOMPCopyinClause(CGF, S);
5001 (void)emitWorksharingDirective(CGF, S, HasCancel: S.hasCancel());
5002 };
5003 {
5004 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5005 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
5006 CGCapturedStmtInfo CGSI(CR_OpenMP);
5007 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5008 OMPLoopScope LoopScope(CGF, S);
5009 return CGF.EmitScalarExpr(E: S.getNumIterations());
5010 };
5011 bool IsInscan = llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
5012 P: [](const OMPReductionClause *C) {
5013 return C->getModifier() == OMPC_REDUCTION_inscan;
5014 });
5015 if (IsInscan)
5016 emitScanBasedDirectiveDecls(CGF&: *this, S, NumIteratorsGen);
5017 auto LPCRegion =
5018 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5019 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for, CodeGen,
5020 CodeGenBoundParameters: emitEmptyBoundParameters);
5021 if (IsInscan)
5022 emitScanBasedDirectiveFinals(CGF&: *this, S, NumIteratorsGen);
5023 }
5024 // Check for outer lastprivate conditional update.
5025 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5026}
5027
5028void CodeGenFunction::EmitOMPParallelForSimdDirective(
5029 const OMPParallelForSimdDirective &S) {
5030 // Emit directive as a combined directive that consists of two implicit
5031 // directives: 'parallel' with 'for' directive.
5032 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5033 Action.Enter(CGF);
5034 emitOMPCopyinClause(CGF, S);
5035 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
5036 };
5037 {
5038 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
5039 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
5040 CGCapturedStmtInfo CGSI(CR_OpenMP);
5041 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
5042 OMPLoopScope LoopScope(CGF, S);
5043 return CGF.EmitScalarExpr(E: S.getNumIterations());
5044 };
5045 bool IsInscan = llvm::any_of(Range: S.getClausesOfKind<OMPReductionClause>(),
5046 P: [](const OMPReductionClause *C) {
5047 return C->getModifier() == OMPC_REDUCTION_inscan;
5048 });
5049 if (IsInscan)
5050 emitScanBasedDirectiveDecls(CGF&: *this, S, NumIteratorsGen);
5051 auto LPCRegion =
5052 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5053 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for_simd, CodeGen,
5054 CodeGenBoundParameters: emitEmptyBoundParameters);
5055 if (IsInscan)
5056 emitScanBasedDirectiveFinals(CGF&: *this, S, NumIteratorsGen);
5057 }
5058 // Check for outer lastprivate conditional update.
5059 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5060}
5061
5062void CodeGenFunction::EmitOMPParallelMasterDirective(
5063 const OMPParallelMasterDirective &S) {
5064 // Emit directive as a combined directive that consists of two implicit
5065 // directives: 'parallel' with 'master' directive.
5066 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5067 Action.Enter(CGF);
5068 OMPPrivateScope PrivateScope(CGF);
5069 emitOMPCopyinClause(CGF, S);
5070 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
5071 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
5072 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
5073 (void)PrivateScope.Privatize();
5074 emitMaster(CGF, S);
5075 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
5076 };
5077 {
5078 auto LPCRegion =
5079 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5080 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master, CodeGen,
5081 CodeGenBoundParameters: emitEmptyBoundParameters);
5082 emitPostUpdateForReductionClause(CGF&: *this, D: S,
5083 CondGen: [](CodeGenFunction &) { return nullptr; });
5084 }
5085 // Check for outer lastprivate conditional update.
5086 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5087}
5088
5089void CodeGenFunction::EmitOMPParallelMaskedDirective(
5090 const OMPParallelMaskedDirective &S) {
5091 // Emit directive as a combined directive that consists of two implicit
5092 // directives: 'parallel' with 'masked' directive.
5093 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5094 Action.Enter(CGF);
5095 OMPPrivateScope PrivateScope(CGF);
5096 emitOMPCopyinClause(CGF, S);
5097 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
5098 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
5099 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
5100 (void)PrivateScope.Privatize();
5101 emitMasked(CGF, S);
5102 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
5103 };
5104 {
5105 auto LPCRegion =
5106 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5107 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked, CodeGen,
5108 CodeGenBoundParameters: emitEmptyBoundParameters);
5109 emitPostUpdateForReductionClause(CGF&: *this, D: S,
5110 CondGen: [](CodeGenFunction &) { return nullptr; });
5111 }
5112 // Check for outer lastprivate conditional update.
5113 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5114}
5115
5116void CodeGenFunction::EmitOMPParallelSectionsDirective(
5117 const OMPParallelSectionsDirective &S) {
5118 // Emit directive as a combined directive that consists of two implicit
5119 // directives: 'parallel' with 'sections' directive.
5120 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5121 Action.Enter(CGF);
5122 emitOMPCopyinClause(CGF, S);
5123 CGF.EmitSections(S);
5124 };
5125 {
5126 auto LPCRegion =
5127 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5128 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_sections, CodeGen,
5129 CodeGenBoundParameters: emitEmptyBoundParameters);
5130 }
5131 // Check for outer lastprivate conditional update.
5132 checkForLastprivateConditionalUpdate(CGF&: *this, S);
5133}
5134
5135namespace {
5136/// Get the list of variables declared in the context of the untied tasks.
5137class CheckVarsEscapingUntiedTaskDeclContext final
5138 : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
5139 llvm::SmallVector<const VarDecl *, 4> PrivateDecls;
5140
5141public:
5142 explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
5143 ~CheckVarsEscapingUntiedTaskDeclContext() = default;
5144 void VisitDeclStmt(const DeclStmt *S) {
5145 if (!S)
5146 return;
5147 // Need to privatize only local vars, static locals can be processed as is.
5148 for (const Decl *D : S->decls()) {
5149 if (const auto *VD = dyn_cast_or_null<VarDecl>(Val: D))
5150 if (VD->hasLocalStorage())
5151 PrivateDecls.push_back(Elt: VD);
5152 }
5153 }
5154 void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
5155 void VisitCapturedStmt(const CapturedStmt *) {}
5156 void VisitLambdaExpr(const LambdaExpr *) {}
5157 void VisitBlockExpr(const BlockExpr *) {}
5158 void VisitStmt(const Stmt *S) {
5159 if (!S)
5160 return;
5161 for (const Stmt *Child : S->children())
5162 if (Child)
5163 Visit(S: Child);
5164 }
5165
5166 /// Swaps list of vars with the provided one.
5167 ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
5168};
5169} // anonymous namespace
5170
5171static void buildDependences(const OMPExecutableDirective &S,
5172 OMPTaskDataTy &Data) {
5173
5174 // First look for 'omp_all_memory' and add this first.
5175 bool OmpAllMemory = false;
5176 if (llvm::any_of(
5177 Range: S.getClausesOfKind<OMPDependClause>(), P: [](const OMPDependClause *C) {
5178 return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||
5179 C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;
5180 })) {
5181 OmpAllMemory = true;
5182 // Since both OMPC_DEPEND_outallmemory and OMPC_DEPEND_inoutallmemory are
5183 // equivalent to the runtime, always use OMPC_DEPEND_outallmemory to
5184 // simplify.
5185 OMPTaskDataTy::DependData &DD =
5186 Data.Dependences.emplace_back(Args: OMPC_DEPEND_outallmemory,
5187 /*IteratorExpr=*/Args: nullptr);
5188 // Add a nullptr Expr to simplify the codegen in emitDependData.
5189 DD.DepExprs.push_back(Elt: nullptr);
5190 }
5191 // Add remaining dependences skipping any 'out' or 'inout' if they are
5192 // overridden by 'omp_all_memory'.
5193 for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
5194 OpenMPDependClauseKind Kind = C->getDependencyKind();
5195 if (Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)
5196 continue;
5197 if (OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))
5198 continue;
5199 OMPTaskDataTy::DependData &DD =
5200 Data.Dependences.emplace_back(Args: C->getDependencyKind(), Args: C->getModifier());
5201 DD.DepExprs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5202 }
5203}
5204
5205void CodeGenFunction::EmitOMPTaskBasedDirective(
5206 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
5207 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
5208 OMPTaskDataTy &Data) {
5209 // Emit outlined function for task construct.
5210 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CapturedRegion);
5211 auto I = CS->getCapturedDecl()->param_begin();
5212 auto PartId = std::next(x: I);
5213 auto TaskT = std::next(x: I, n: 4);
5214 // Check if the task is final
5215 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
5216 // If the condition constant folds and can be elided, try to avoid emitting
5217 // the condition and the dead arm of the if/else.
5218 const Expr *Cond = Clause->getCondition();
5219 bool CondConstant;
5220 if (ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant))
5221 Data.Final.setInt(CondConstant);
5222 else
5223 Data.Final.setPointer(EvaluateExprAsBool(E: Cond));
5224 } else {
5225 // By default the task is not final.
5226 Data.Final.setInt(/*IntVal=*/false);
5227 }
5228 // Check if the task has 'priority' clause.
5229 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
5230 const Expr *Prio = Clause->getPriority();
5231 Data.Priority.setInt(/*IntVal=*/true);
5232 Data.Priority.setPointer(EmitScalarConversion(
5233 Src: EmitScalarExpr(E: Prio), SrcTy: Prio->getType(),
5234 DstTy: getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
5235 Loc: Prio->getExprLoc()));
5236 }
5237 // The first function argument for tasks is a thread id, the second one is a
5238 // part id (0 for tied tasks, >=0 for untied task).
5239 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
5240 // Get list of private variables.
5241 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
5242 auto IRef = C->varlist_begin();
5243 for (const Expr *IInit : C->private_copies()) {
5244 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
5245 if (EmittedAsPrivate.insert(V: OrigVD->getCanonicalDecl()).second) {
5246 Data.PrivateVars.push_back(Elt: *IRef);
5247 Data.PrivateCopies.push_back(Elt: IInit);
5248 }
5249 ++IRef;
5250 }
5251 }
5252 EmittedAsPrivate.clear();
5253 // Get list of firstprivate variables.
5254 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5255 auto IRef = C->varlist_begin();
5256 auto IElemInitRef = C->inits().begin();
5257 for (const Expr *IInit : C->private_copies()) {
5258 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
5259 if (EmittedAsPrivate.insert(V: OrigVD->getCanonicalDecl()).second) {
5260 Data.FirstprivateVars.push_back(Elt: *IRef);
5261 Data.FirstprivateCopies.push_back(Elt: IInit);
5262 Data.FirstprivateInits.push_back(Elt: *IElemInitRef);
5263 }
5264 ++IRef;
5265 ++IElemInitRef;
5266 }
5267 }
5268 // Get list of lastprivate variables (for taskloops).
5269 llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
5270 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
5271 auto IRef = C->varlist_begin();
5272 auto ID = C->destination_exprs().begin();
5273 for (const Expr *IInit : C->private_copies()) {
5274 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRef)->getDecl());
5275 if (EmittedAsPrivate.insert(V: OrigVD->getCanonicalDecl()).second) {
5276 Data.LastprivateVars.push_back(Elt: *IRef);
5277 Data.LastprivateCopies.push_back(Elt: IInit);
5278 }
5279 LastprivateDstsOrigs.insert(
5280 KV: std::make_pair(x: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ID)->getDecl()),
5281 y: cast<DeclRefExpr>(Val: *IRef)));
5282 ++IRef;
5283 ++ID;
5284 }
5285 }
5286 SmallVector<const Expr *, 4> LHSs;
5287 SmallVector<const Expr *, 4> RHSs;
5288 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
5289 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5290 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5291 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
5292 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
5293 in_end: C->reduction_ops().end());
5294 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
5295 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
5296 }
5297 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
5298 CGF&: *this, Loc: S.getBeginLoc(), LHSExprs: LHSs, RHSExprs: RHSs, Data);
5299 // Build list of dependences.
5300 buildDependences(S, Data);
5301 // Get list of local vars for untied tasks.
5302 if (!Data.Tied) {
5303 CheckVarsEscapingUntiedTaskDeclContext Checker;
5304 Checker.Visit(S: S.getInnermostCapturedStmt()->getCapturedStmt());
5305 Data.PrivateLocals.append(in_start: Checker.getPrivateDecls().begin(),
5306 in_end: Checker.getPrivateDecls().end());
5307 }
5308 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
5309 CapturedRegion](CodeGenFunction &CGF,
5310 PrePostActionTy &Action) {
5311 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
5312 std::pair<Address, Address>>
5313 UntiedLocalVars;
5314 // Set proper addresses for generated private copies.
5315 OMPPrivateScope Scope(CGF);
5316 // Generate debug info for variables present in shared clause.
5317 if (auto *DI = CGF.getDebugInfo()) {
5318 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
5319 CGF.CapturedStmtInfo->getCaptureFields();
5320 llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
5321 if (CaptureFields.size() && ContextValue) {
5322 unsigned CharWidth = CGF.getContext().getCharWidth();
5323 // The shared variables are packed together as members of structure.
5324 // So the address of each shared variable can be computed by adding
5325 // offset of it (within record) to the base address of record. For each
5326 // shared variable, debug intrinsic llvm.dbg.declare is generated with
5327 // appropriate expressions (DIExpression).
5328 // Ex:
5329 // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
5330 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5331 // metadata !svar1,
5332 // metadata !DIExpression(DW_OP_deref))
5333 // call void @llvm.dbg.declare(metadata %struct.anon* %12,
5334 // metadata !svar2,
5335 // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
5336 for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
5337 const VarDecl *SharedVar = It->first;
5338 RecordDecl *CaptureRecord = It->second->getParent();
5339 const ASTRecordLayout &Layout =
5340 CGF.getContext().getASTRecordLayout(D: CaptureRecord);
5341 unsigned Offset =
5342 Layout.getFieldOffset(FieldNo: It->second->getFieldIndex()) / CharWidth;
5343 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5344 (void)DI->EmitDeclareOfAutoVariable(Decl: SharedVar, AI: ContextValue,
5345 Builder&: CGF.Builder, UsePointerValue: false);
5346 // Get the call dbg.declare instruction we just created and update
5347 // its DIExpression to add offset to base address.
5348 auto UpdateExpr = [](llvm::LLVMContext &Ctx, auto *Declare,
5349 unsigned Offset) {
5350 SmallVector<uint64_t, 8> Ops;
5351 // Add offset to the base address if non zero.
5352 if (Offset) {
5353 Ops.push_back(Elt: llvm::dwarf::DW_OP_plus_uconst);
5354 Ops.push_back(Elt: Offset);
5355 }
5356 Ops.push_back(Elt: llvm::dwarf::DW_OP_deref);
5357 Declare->setExpression(llvm::DIExpression::get(Context&: Ctx, Elements: Ops));
5358 };
5359 llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
5360 if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(Val: &Last))
5361 UpdateExpr(DDI->getContext(), DDI, Offset);
5362 // If we're emitting using the new debug info format into a block
5363 // without a terminator, the record will be "trailing".
5364 assert(!Last.isTerminator() && "unexpected terminator");
5365 if (auto *Marker =
5366 CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {
5367 for (llvm::DbgVariableRecord &DVR : llvm::reverse(
5368 C: llvm::filterDbgVars(R: Marker->getDbgRecordRange()))) {
5369 UpdateExpr(Last.getContext(), &DVR, Offset);
5370 break;
5371 }
5372 }
5373 }
5374 }
5375 }
5376 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> FirstprivatePtrs;
5377 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
5378 !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
5379 enum { PrivatesParam = 2, CopyFnParam = 3 };
5380 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5381 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: CopyFnParam)));
5382 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(
5383 VD: CS->getCapturedDecl()->getParam(i: PrivatesParam)));
5384 // Map privates.
5385 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
5386 llvm::SmallVector<llvm::Value *, 16> CallArgs;
5387 llvm::SmallVector<llvm::Type *, 4> ParamTypes;
5388 CallArgs.push_back(Elt: PrivatesPtr);
5389 ParamTypes.push_back(Elt: PrivatesPtr->getType());
5390 for (const Expr *E : Data.PrivateVars) {
5391 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
5392 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5393 T: CGF.getContext().getPointerType(T: E->getType()), Name: ".priv.ptr.addr");
5394 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5395 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5396 ParamTypes.push_back(Elt: PrivatePtr.getType());
5397 }
5398 for (const Expr *E : Data.FirstprivateVars) {
5399 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
5400 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5401 T: CGF.getContext().getPointerType(T: E->getType()),
5402 Name: ".firstpriv.ptr.addr");
5403 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5404 FirstprivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5405 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5406 ParamTypes.push_back(Elt: PrivatePtr.getType());
5407 }
5408 for (const Expr *E : Data.LastprivateVars) {
5409 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
5410 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5411 T: CGF.getContext().getPointerType(T: E->getType()),
5412 Name: ".lastpriv.ptr.addr");
5413 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5414 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5415 ParamTypes.push_back(Elt: PrivatePtr.getType());
5416 }
5417 for (const VarDecl *VD : Data.PrivateLocals) {
5418 QualType Ty = VD->getType().getNonReferenceType();
5419 if (VD->getType()->isLValueReferenceType())
5420 Ty = CGF.getContext().getPointerType(T: Ty);
5421 if (isAllocatableDecl(VD))
5422 Ty = CGF.getContext().getPointerType(T: Ty);
5423 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5424 T: CGF.getContext().getPointerType(T: Ty), Name: ".local.ptr.addr");
5425 auto Result = UntiedLocalVars.insert(
5426 KV: std::make_pair(x&: VD, y: std::make_pair(x&: PrivatePtr, y: Address::invalid())));
5427 // If key exists update in place.
5428 if (Result.second == false)
5429 *Result.first = std::make_pair(
5430 x&: VD, y: std::make_pair(x&: PrivatePtr, y: Address::invalid()));
5431 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5432 ParamTypes.push_back(Elt: PrivatePtr.getType());
5433 }
5434 auto *CopyFnTy = llvm::FunctionType::get(Result: CGF.Builder.getVoidTy(),
5435 Params: ParamTypes, /*isVarArg=*/false);
5436 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5437 CGF, Loc: S.getBeginLoc(), OutlinedFn: {CopyFnTy, CopyFn}, Args: CallArgs);
5438 for (const auto &Pair : LastprivateDstsOrigs) {
5439 const auto *OrigVD = cast<VarDecl>(Val: Pair.second->getDecl());
5440 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
5441 /*RefersToEnclosingVariableOrCapture=*/
5442 CGF.CapturedStmtInfo->lookup(VD: OrigVD) != nullptr,
5443 Pair.second->getType(), VK_LValue,
5444 Pair.second->getExprLoc());
5445 Scope.addPrivate(LocalVD: Pair.first, Addr: CGF.EmitLValue(E: &DRE).getAddress());
5446 }
5447 for (const auto &Pair : PrivatePtrs) {
5448 Address Replacement = Address(
5449 CGF.Builder.CreateLoad(Addr: Pair.second),
5450 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5451 CGF.getContext().getDeclAlign(D: Pair.first));
5452 Scope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5453 if (auto *DI = CGF.getDebugInfo())
5454 if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
5455 (void)DI->EmitDeclareOfAutoVariable(
5456 Decl: Pair.first, AI: Pair.second.getBasePointer(), Builder&: CGF.Builder,
5457 /*UsePointerValue*/ true);
5458 }
5459 // Adjust mapping for internal locals by mapping actual memory instead of
5460 // a pointer to this memory.
5461 for (auto &Pair : UntiedLocalVars) {
5462 QualType VDType = Pair.first->getType().getNonReferenceType();
5463 if (Pair.first->getType()->isLValueReferenceType())
5464 VDType = CGF.getContext().getPointerType(T: VDType);
5465 if (isAllocatableDecl(VD: Pair.first)) {
5466 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Pair.second.first);
5467 Address Replacement(
5468 Ptr,
5469 CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: VDType)),
5470 CGF.getPointerAlign());
5471 Pair.second.first = Replacement;
5472 Ptr = CGF.Builder.CreateLoad(Addr: Replacement);
5473 Replacement = Address(Ptr, CGF.ConvertTypeForMem(T: VDType),
5474 CGF.getContext().getDeclAlign(D: Pair.first));
5475 Pair.second.second = Replacement;
5476 } else {
5477 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Pair.second.first);
5478 Address Replacement(Ptr, CGF.ConvertTypeForMem(T: VDType),
5479 CGF.getContext().getDeclAlign(D: Pair.first));
5480 Pair.second.first = Replacement;
5481 }
5482 }
5483 }
5484 if (Data.Reductions) {
5485 OMPPrivateScope FirstprivateScope(CGF);
5486 for (const auto &Pair : FirstprivatePtrs) {
5487 Address Replacement(
5488 CGF.Builder.CreateLoad(Addr: Pair.second),
5489 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5490 CGF.getContext().getDeclAlign(D: Pair.first));
5491 FirstprivateScope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5492 }
5493 (void)FirstprivateScope.Privatize();
5494 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5495 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5496 Data.ReductionCopies, Data.ReductionOps);
5497 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5498 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: 9)));
5499 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5500 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5501 RedCG.emitAggregateType(CGF, N: Cnt);
5502 // FIXME: This must removed once the runtime library is fixed.
5503 // Emit required threadprivate variables for
5504 // initializer/combiner/finalizer.
5505 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5506 RCG&: RedCG, N: Cnt);
5507 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5508 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5509 Replacement = Address(
5510 CGF.EmitScalarConversion(Src: Replacement.emitRawPointer(CGF),
5511 SrcTy: CGF.getContext().VoidPtrTy,
5512 DstTy: CGF.getContext().getPointerType(
5513 T: Data.ReductionCopies[Cnt]->getType()),
5514 Loc: Data.ReductionCopies[Cnt]->getExprLoc()),
5515 CGF.ConvertTypeForMem(T: Data.ReductionCopies[Cnt]->getType()),
5516 Replacement.getAlignment());
5517 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5518 Scope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5519 }
5520 }
5521 // Privatize all private variables except for in_reduction items.
5522 (void)Scope.Privatize();
5523 SmallVector<const Expr *, 4> InRedVars;
5524 SmallVector<const Expr *, 4> InRedPrivs;
5525 SmallVector<const Expr *, 4> InRedOps;
5526 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5527 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5528 auto IPriv = C->privates().begin();
5529 auto IRed = C->reduction_ops().begin();
5530 auto ITD = C->taskgroup_descriptors().begin();
5531 for (const Expr *Ref : C->varlist()) {
5532 InRedVars.emplace_back(Args&: Ref);
5533 InRedPrivs.emplace_back(Args: *IPriv);
5534 InRedOps.emplace_back(Args: *IRed);
5535 TaskgroupDescriptors.emplace_back(Args: *ITD);
5536 std::advance(i&: IPriv, n: 1);
5537 std::advance(i&: IRed, n: 1);
5538 std::advance(i&: ITD, n: 1);
5539 }
5540 }
5541 // Privatize in_reduction items here, because taskgroup descriptors must be
5542 // privatized earlier.
5543 OMPPrivateScope InRedScope(CGF);
5544 if (!InRedVars.empty()) {
5545 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5546 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5547 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5548 RedCG.emitAggregateType(CGF, N: Cnt);
5549 // The taskgroup descriptor variable is always implicit firstprivate and
5550 // privatized already during processing of the firstprivates.
5551 // FIXME: This must removed once the runtime library is fixed.
5552 // Emit required threadprivate variables for
5553 // initializer/combiner/finalizer.
5554 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5555 RCG&: RedCG, N: Cnt);
5556 llvm::Value *ReductionsPtr;
5557 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5558 ReductionsPtr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: TRExpr),
5559 Loc: TRExpr->getExprLoc());
5560 } else {
5561 ReductionsPtr = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
5562 }
5563 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5564 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5565 Replacement = Address(
5566 CGF.EmitScalarConversion(
5567 Src: Replacement.emitRawPointer(CGF), SrcTy: CGF.getContext().VoidPtrTy,
5568 DstTy: CGF.getContext().getPointerType(T: InRedPrivs[Cnt]->getType()),
5569 Loc: InRedPrivs[Cnt]->getExprLoc()),
5570 CGF.ConvertTypeForMem(T: InRedPrivs[Cnt]->getType()),
5571 Replacement.getAlignment());
5572 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5573 InRedScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5574 }
5575 }
5576 (void)InRedScope.Privatize();
5577
5578 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,
5579 UntiedLocalVars);
5580 Action.Enter(CGF);
5581 BodyGen(CGF);
5582 };
5583 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
5584 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5585 D: S, ThreadIDVar: *I, PartIDVar: *PartId, TaskTVar: *TaskT, InnermostKind: EKind, CodeGen, Tied: Data.Tied, NumberOfParts&: Data.NumberOfParts);
5586 OMPLexicalScope Scope(*this, S, std::nullopt,
5587 !isOpenMPParallelDirective(DKind: EKind) &&
5588 !isOpenMPSimdDirective(DKind: EKind));
5589 TaskGen(*this, OutlinedFn, Data);
5590}
5591
5592static ImplicitParamDecl *
5593createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
5594 QualType Ty, CapturedDecl *CD,
5595 SourceLocation Loc) {
5596 auto *OrigVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: Ty,
5597 ParamKind: ImplicitParamKind::Other);
5598 auto *OrigRef = DeclRefExpr::Create(
5599 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: OrigVD,
5600 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: Ty, VK: VK_LValue);
5601 auto *PrivateVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: Ty,
5602 ParamKind: ImplicitParamKind::Other);
5603 auto *PrivateRef = DeclRefExpr::Create(
5604 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: PrivateVD,
5605 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: Ty, VK: VK_LValue);
5606 QualType ElemType = C.getBaseElementType(QT: Ty);
5607 auto *InitVD = ImplicitParamDecl::Create(C, DC: CD, IdLoc: Loc, /*Id=*/nullptr, T: ElemType,
5608 ParamKind: ImplicitParamKind::Other);
5609 auto *InitRef = DeclRefExpr::Create(
5610 Context: C, QualifierLoc: NestedNameSpecifierLoc(), TemplateKWLoc: SourceLocation(), D: InitVD,
5611 /*RefersToEnclosingVariableOrCapture=*/false, NameLoc: Loc, T: ElemType, VK: VK_LValue);
5612 PrivateVD->setInitStyle(VarDecl::CInit);
5613 PrivateVD->setInit(ImplicitCastExpr::Create(Context: C, T: ElemType, Kind: CK_LValueToRValue,
5614 Operand: InitRef, /*BasePath=*/nullptr,
5615 Cat: VK_PRValue, FPO: FPOptionsOverride()));
5616 Data.FirstprivateVars.emplace_back(Args&: OrigRef);
5617 Data.FirstprivateCopies.emplace_back(Args&: PrivateRef);
5618 Data.FirstprivateInits.emplace_back(Args&: InitRef);
5619 return OrigVD;
5620}
5621
5622void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
5623 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
5624 OMPTargetDataInfo &InputInfo) {
5625 // Emit outlined function for task construct.
5626 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_task);
5627 Address CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
5628 CanQualType SharedsTy =
5629 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
5630 auto I = CS->getCapturedDecl()->param_begin();
5631 auto PartId = std::next(x: I);
5632 auto TaskT = std::next(x: I, n: 4);
5633 OMPTaskDataTy Data;
5634 // The task is not final.
5635 Data.Final.setInt(/*IntVal=*/false);
5636 // Get list of firstprivate variables.
5637 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
5638 auto IRef = C->varlist_begin();
5639 auto IElemInitRef = C->inits().begin();
5640 for (auto *IInit : C->private_copies()) {
5641 Data.FirstprivateVars.push_back(Elt: *IRef);
5642 Data.FirstprivateCopies.push_back(Elt: IInit);
5643 Data.FirstprivateInits.push_back(Elt: *IElemInitRef);
5644 ++IRef;
5645 ++IElemInitRef;
5646 }
5647 }
5648 SmallVector<const Expr *, 4> LHSs;
5649 SmallVector<const Expr *, 4> RHSs;
5650 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5651 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5652 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5653 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
5654 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
5655 in_end: C->reduction_ops().end());
5656 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
5657 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
5658 }
5659 OMPPrivateScope TargetScope(*this);
5660 VarDecl *BPVD = nullptr;
5661 VarDecl *PVD = nullptr;
5662 VarDecl *SVD = nullptr;
5663 VarDecl *MVD = nullptr;
5664 if (InputInfo.NumberOfTargetItems > 0) {
5665 auto *CD = CapturedDecl::Create(
5666 C&: getContext(), DC: getContext().getTranslationUnitDecl(), /*NumParams=*/0);
5667 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
5668 QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
5669 EltTy: getContext().VoidPtrTy, ArySize: ArrSize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5670 /*IndexTypeQuals=*/0);
5671 BPVD = createImplicitFirstprivateForType(
5672 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5673 PVD = createImplicitFirstprivateForType(
5674 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5675 QualType SizesType = getContext().getConstantArrayType(
5676 EltTy: getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
5677 ArySize: ArrSize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5678 /*IndexTypeQuals=*/0);
5679 SVD = createImplicitFirstprivateForType(C&: getContext(), Data, Ty: SizesType, CD,
5680 Loc: S.getBeginLoc());
5681 TargetScope.addPrivate(LocalVD: BPVD, Addr: InputInfo.BasePointersArray);
5682 TargetScope.addPrivate(LocalVD: PVD, Addr: InputInfo.PointersArray);
5683 TargetScope.addPrivate(LocalVD: SVD, Addr: InputInfo.SizesArray);
5684 // If there is no user-defined mapper, the mapper array will be nullptr. In
5685 // this case, we don't need to privatize it.
5686 if (!isa_and_nonnull<llvm::ConstantPointerNull>(
5687 Val: InputInfo.MappersArray.emitRawPointer(CGF&: *this))) {
5688 MVD = createImplicitFirstprivateForType(
5689 C&: getContext(), Data, Ty: BaseAndPointerAndMapperType, CD, Loc: S.getBeginLoc());
5690 TargetScope.addPrivate(LocalVD: MVD, Addr: InputInfo.MappersArray);
5691 }
5692 }
5693 (void)TargetScope.Privatize();
5694 buildDependences(S, Data);
5695 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
5696 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,
5697 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
5698 // Set proper addresses for generated private copies.
5699 OMPPrivateScope Scope(CGF);
5700 if (!Data.FirstprivateVars.empty()) {
5701 enum { PrivatesParam = 2, CopyFnParam = 3 };
5702 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
5703 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: CopyFnParam)));
5704 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(
5705 VD: CS->getCapturedDecl()->getParam(i: PrivatesParam)));
5706 // Map privates.
5707 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
5708 llvm::SmallVector<llvm::Value *, 16> CallArgs;
5709 llvm::SmallVector<llvm::Type *, 4> ParamTypes;
5710 CallArgs.push_back(Elt: PrivatesPtr);
5711 ParamTypes.push_back(Elt: PrivatesPtr->getType());
5712 for (const Expr *E : Data.FirstprivateVars) {
5713 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
5714 RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
5715 T: CGF.getContext().getPointerType(T: E->getType()),
5716 Name: ".firstpriv.ptr.addr");
5717 PrivatePtrs.emplace_back(Args&: VD, Args&: PrivatePtr);
5718 CallArgs.push_back(Elt: PrivatePtr.getPointer());
5719 ParamTypes.push_back(Elt: PrivatePtr.getType());
5720 }
5721 auto *CopyFnTy = llvm::FunctionType::get(Result: CGF.Builder.getVoidTy(),
5722 Params: ParamTypes, /*isVarArg=*/false);
5723 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
5724 CGF, Loc: S.getBeginLoc(), OutlinedFn: {CopyFnTy, CopyFn}, Args: CallArgs);
5725 for (const auto &Pair : PrivatePtrs) {
5726 Address Replacement(
5727 CGF.Builder.CreateLoad(Addr: Pair.second),
5728 CGF.ConvertTypeForMem(T: Pair.first->getType().getNonReferenceType()),
5729 CGF.getContext().getDeclAlign(D: Pair.first));
5730 Scope.addPrivate(LocalVD: Pair.first, Addr: Replacement);
5731 }
5732 }
5733 CGF.processInReduction(S, Data, CGF, CS, Scope);
5734 if (InputInfo.NumberOfTargetItems > 0) {
5735 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
5736 Addr: CGF.GetAddrOfLocalVar(VD: BPVD), /*Index=*/0);
5737 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
5738 Addr: CGF.GetAddrOfLocalVar(VD: PVD), /*Index=*/0);
5739 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
5740 Addr: CGF.GetAddrOfLocalVar(VD: SVD), /*Index=*/0);
5741 // If MVD is nullptr, the mapper array is not privatized
5742 if (MVD)
5743 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
5744 Addr: CGF.GetAddrOfLocalVar(VD: MVD), /*Index=*/0);
5745 }
5746
5747 Action.Enter(CGF);
5748 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
5749 auto *TL = S.getSingleClause<OMPThreadLimitClause>();
5750 if (CGF.CGM.getLangOpts().OpenMP >= 51 &&
5751 needsTaskBasedThreadLimit(DKind: EKind) && TL) {
5752 // Emit __kmpc_set_thread_limit() to set the thread_limit for the task
5753 // enclosing this target region. This will indirectly set the thread_limit
5754 // for every applicable construct within target region.
5755 CGF.CGM.getOpenMPRuntime().emitThreadLimitClause(
5756 CGF, ThreadLimit: TL->getThreadLimit().front(), Loc: S.getBeginLoc());
5757 }
5758 BodyGen(CGF);
5759 };
5760 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
5761 D: S, ThreadIDVar: *I, PartIDVar: *PartId, TaskTVar: *TaskT, InnermostKind: EKind, CodeGen, /*Tied=*/true,
5762 NumberOfParts&: Data.NumberOfParts);
5763 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
5764 IntegerLiteral IfCond(getContext(), TrueOrFalse,
5765 getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
5766 SourceLocation());
5767 CGM.getOpenMPRuntime().emitTaskCall(CGF&: *this, Loc: S.getBeginLoc(), D: S, TaskFunction: OutlinedFn,
5768 SharedsTy, Shareds: CapturedStruct, IfCond: &IfCond, Data);
5769}
5770
5771void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
5772 OMPTaskDataTy &Data,
5773 CodeGenFunction &CGF,
5774 const CapturedStmt *CS,
5775 OMPPrivateScope &Scope) {
5776 OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
5777 if (Data.Reductions) {
5778 OpenMPDirectiveKind CapturedRegion = EKind;
5779 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
5780 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
5781 Data.ReductionCopies, Data.ReductionOps);
5782 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
5783 Addr: CGF.GetAddrOfLocalVar(VD: CS->getCapturedDecl()->getParam(i: 4)));
5784 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
5785 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5786 RedCG.emitAggregateType(CGF, N: Cnt);
5787 // FIXME: This must removed once the runtime library is fixed.
5788 // Emit required threadprivate variables for
5789 // initializer/combiner/finalizer.
5790 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5791 RCG&: RedCG, N: Cnt);
5792 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5793 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5794 Replacement = Address(
5795 CGF.EmitScalarConversion(Src: Replacement.emitRawPointer(CGF),
5796 SrcTy: CGF.getContext().VoidPtrTy,
5797 DstTy: CGF.getContext().getPointerType(
5798 T: Data.ReductionCopies[Cnt]->getType()),
5799 Loc: Data.ReductionCopies[Cnt]->getExprLoc()),
5800 CGF.ConvertTypeForMem(T: Data.ReductionCopies[Cnt]->getType()),
5801 Replacement.getAlignment());
5802 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5803 Scope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5804 }
5805 }
5806 (void)Scope.Privatize();
5807 SmallVector<const Expr *, 4> InRedVars;
5808 SmallVector<const Expr *, 4> InRedPrivs;
5809 SmallVector<const Expr *, 4> InRedOps;
5810 SmallVector<const Expr *, 4> TaskgroupDescriptors;
5811 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
5812 auto IPriv = C->privates().begin();
5813 auto IRed = C->reduction_ops().begin();
5814 auto ITD = C->taskgroup_descriptors().begin();
5815 for (const Expr *Ref : C->varlist()) {
5816 InRedVars.emplace_back(Args&: Ref);
5817 InRedPrivs.emplace_back(Args: *IPriv);
5818 InRedOps.emplace_back(Args: *IRed);
5819 TaskgroupDescriptors.emplace_back(Args: *ITD);
5820 std::advance(i&: IPriv, n: 1);
5821 std::advance(i&: IRed, n: 1);
5822 std::advance(i&: ITD, n: 1);
5823 }
5824 }
5825 OMPPrivateScope InRedScope(CGF);
5826 if (!InRedVars.empty()) {
5827 ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
5828 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
5829 RedCG.emitSharedOrigLValue(CGF, N: Cnt);
5830 RedCG.emitAggregateType(CGF, N: Cnt);
5831 // FIXME: This must removed once the runtime library is fixed.
5832 // Emit required threadprivate variables for
5833 // initializer/combiner/finalizer.
5834 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, Loc: S.getBeginLoc(),
5835 RCG&: RedCG, N: Cnt);
5836 llvm::Value *ReductionsPtr;
5837 if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
5838 ReductionsPtr =
5839 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: TRExpr), Loc: TRExpr->getExprLoc());
5840 } else {
5841 ReductionsPtr = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
5842 }
5843 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
5844 CGF, Loc: S.getBeginLoc(), ReductionsPtr, SharedLVal: RedCG.getSharedLValue(N: Cnt));
5845 Replacement = Address(
5846 CGF.EmitScalarConversion(
5847 Src: Replacement.emitRawPointer(CGF), SrcTy: CGF.getContext().VoidPtrTy,
5848 DstTy: CGF.getContext().getPointerType(T: InRedPrivs[Cnt]->getType()),
5849 Loc: InRedPrivs[Cnt]->getExprLoc()),
5850 CGF.ConvertTypeForMem(T: InRedPrivs[Cnt]->getType()),
5851 Replacement.getAlignment());
5852 Replacement = RedCG.adjustPrivateAddress(CGF, N: Cnt, PrivateAddr: Replacement);
5853 InRedScope.addPrivate(LocalVD: RedCG.getBaseDecl(N: Cnt), Addr: Replacement);
5854 }
5855 }
5856 (void)InRedScope.Privatize();
5857}
5858
5859void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
5860 // Emit outlined function for task construct.
5861 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_task);
5862 Address CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
5863 CanQualType SharedsTy =
5864 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
5865 const Expr *IfCond = nullptr;
5866 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
5867 if (C->getNameModifier() == OMPD_unknown ||
5868 C->getNameModifier() == OMPD_task) {
5869 IfCond = C->getCondition();
5870 break;
5871 }
5872 }
5873
5874 OMPTaskDataTy Data;
5875 // Check if we should emit tied or untied task.
5876 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
5877 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
5878 CGF.EmitStmt(S: CS->getCapturedStmt());
5879 };
5880 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5881 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5882 const OMPTaskDataTy &Data) {
5883 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, Loc: S.getBeginLoc(), D: S, TaskFunction: OutlinedFn,
5884 SharedsTy, Shareds: CapturedStruct, IfCond,
5885 Data);
5886 };
5887 auto LPCRegion =
5888 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
5889 EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_task, BodyGen, TaskGen, Data);
5890}
5891
5892void CodeGenFunction::EmitOMPTaskyieldDirective(
5893 const OMPTaskyieldDirective &S) {
5894 CGM.getOpenMPRuntime().emitTaskyieldCall(CGF&: *this, Loc: S.getBeginLoc());
5895}
5896
5897void CodeGenFunction::EmitOMPErrorDirective(const OMPErrorDirective &S) {
5898 const OMPMessageClause *MC = S.getSingleClause<OMPMessageClause>();
5899 Expr *ME = MC ? MC->getMessageString() : nullptr;
5900 const OMPSeverityClause *SC = S.getSingleClause<OMPSeverityClause>();
5901 bool IsFatal = false;
5902 if (!SC || SC->getSeverityKind() == OMPC_SEVERITY_fatal)
5903 IsFatal = true;
5904 CGM.getOpenMPRuntime().emitErrorCall(CGF&: *this, Loc: S.getBeginLoc(), ME, IsFatal);
5905}
5906
5907void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
5908 CGM.getOpenMPRuntime().emitBarrierCall(CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_barrier);
5909}
5910
5911void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
5912 OMPTaskDataTy Data;
5913 // Build list of dependences
5914 buildDependences(S, Data);
5915 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
5916 CGM.getOpenMPRuntime().emitTaskwaitCall(CGF&: *this, Loc: S.getBeginLoc(), Data);
5917}
5918
5919static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T) {
5920 return T.clauses().empty();
5921}
5922
5923void CodeGenFunction::EmitOMPTaskgroupDirective(
5924 const OMPTaskgroupDirective &S) {
5925 OMPLexicalScope Scope(*this, S, OMPD_unknown);
5926 if (CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(T: S)) {
5927 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
5928 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5929 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
5930 AllocaInsertPt->getIterator());
5931
5932 auto BodyGenCB = [&, this](InsertPointTy AllocIP, InsertPointTy CodeGenIP,
5933 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
5934 Builder.restoreIP(IP: CodeGenIP);
5935 EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
5936 return llvm::Error::success();
5937 };
5938 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
5939 if (!CapturedStmtInfo)
5940 CapturedStmtInfo = &CapStmtInfo;
5941 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
5942 cantFail(ValOrErr: OMPBuilder.createTaskgroup(Loc: Builder, AllocaIP,
5943 /*DeallocBlocks=*/{}, BodyGenCB));
5944 Builder.restoreIP(IP: AfterIP);
5945 return;
5946 }
5947 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5948 Action.Enter(CGF);
5949 if (const Expr *E = S.getReductionRef()) {
5950 SmallVector<const Expr *, 4> LHSs;
5951 SmallVector<const Expr *, 4> RHSs;
5952 OMPTaskDataTy Data;
5953 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
5954 Data.ReductionVars.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5955 Data.ReductionOrigs.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
5956 Data.ReductionCopies.append(in_start: C->privates().begin(), in_end: C->privates().end());
5957 Data.ReductionOps.append(in_start: C->reduction_ops().begin(),
5958 in_end: C->reduction_ops().end());
5959 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
5960 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
5961 }
5962 llvm::Value *ReductionDesc =
5963 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, Loc: S.getBeginLoc(),
5964 LHSExprs: LHSs, RHSExprs: RHSs, Data);
5965 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
5966 CGF.EmitVarDecl(D: *VD);
5967 CGF.EmitStoreOfScalar(Value: ReductionDesc, Addr: CGF.GetAddrOfLocalVar(VD),
5968 /*Volatile=*/false, Ty: E->getType());
5969 }
5970 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
5971 };
5972 CGM.getOpenMPRuntime().emitTaskgroupRegion(CGF&: *this, TaskgroupOpGen: CodeGen, Loc: S.getBeginLoc());
5973}
5974
5975void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
5976 llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
5977 ? llvm::AtomicOrdering::NotAtomic
5978 : llvm::AtomicOrdering::AcquireRelease;
5979 CGM.getOpenMPRuntime().emitFlush(
5980 CGF&: *this,
5981 Vars: [&S]() -> ArrayRef<const Expr *> {
5982 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
5983 return llvm::ArrayRef(FlushClause->varlist_begin(),
5984 FlushClause->varlist_end());
5985 return {};
5986 }(),
5987 Loc: S.getBeginLoc(), AO);
5988}
5989
5990void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {
5991 const auto *DO = S.getSingleClause<OMPDepobjClause>();
5992 LValue DOLVal = EmitLValue(E: DO->getDepobj());
5993 if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
5994 // Build list and emit dependences
5995 OMPTaskDataTy Data;
5996 buildDependences(S, Data);
5997 for (auto &Dep : Data.Dependences) {
5998 Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
5999 CGF&: *this, Dependencies: Dep, Loc: DC->getBeginLoc());
6000 EmitStoreOfScalar(value: DepAddr.emitRawPointer(CGF&: *this), lvalue: DOLVal);
6001 }
6002 return;
6003 }
6004 if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
6005 CGM.getOpenMPRuntime().emitDestroyClause(CGF&: *this, DepobjLVal: DOLVal, Loc: DC->getBeginLoc());
6006 return;
6007 }
6008 if (const auto *UC = S.getSingleClause<OMPUpdateDependObjectsClause>()) {
6009 CGM.getOpenMPRuntime().emitUpdateDependObjectsClause(
6010 CGF&: *this, DepobjLVal: DOLVal, NewDepKind: UC->getDependencyKind(), Loc: UC->getBeginLoc());
6011 return;
6012 }
6013}
6014
6015void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) {
6016 if (!OMPParentLoopDirectiveForScan)
6017 return;
6018 const OMPExecutableDirective &ParentDir = *OMPParentLoopDirectiveForScan;
6019 bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();
6020 SmallVector<const Expr *, 4> Shareds;
6021 SmallVector<const Expr *, 4> Privates;
6022 SmallVector<const Expr *, 4> LHSs;
6023 SmallVector<const Expr *, 4> RHSs;
6024 SmallVector<const Expr *, 4> ReductionOps;
6025 SmallVector<const Expr *, 4> CopyOps;
6026 SmallVector<const Expr *, 4> CopyArrayTemps;
6027 SmallVector<const Expr *, 4> CopyArrayElems;
6028 for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
6029 if (C->getModifier() != OMPC_REDUCTION_inscan)
6030 continue;
6031 Shareds.append(in_start: C->varlist_begin(), in_end: C->varlist_end());
6032 Privates.append(in_start: C->privates().begin(), in_end: C->privates().end());
6033 LHSs.append(in_start: C->lhs_exprs().begin(), in_end: C->lhs_exprs().end());
6034 RHSs.append(in_start: C->rhs_exprs().begin(), in_end: C->rhs_exprs().end());
6035 ReductionOps.append(in_start: C->reduction_ops().begin(), in_end: C->reduction_ops().end());
6036 CopyOps.append(in_start: C->copy_ops().begin(), in_end: C->copy_ops().end());
6037 CopyArrayTemps.append(in_start: C->copy_array_temps().begin(),
6038 in_end: C->copy_array_temps().end());
6039 CopyArrayElems.append(in_start: C->copy_array_elems().begin(),
6040 in_end: C->copy_array_elems().end());
6041 }
6042 if (ParentDir.getDirectiveKind() == OMPD_simd ||
6043 (getLangOpts().OpenMPSimd &&
6044 isOpenMPSimdDirective(DKind: ParentDir.getDirectiveKind()))) {
6045 // For simd directive and simd-based directives in simd only mode, use the
6046 // following codegen:
6047 // int x = 0;
6048 // #pragma omp simd reduction(inscan, +: x)
6049 // for (..) {
6050 // <first part>
6051 // #pragma omp scan inclusive(x)
6052 // <second part>
6053 // }
6054 // is transformed to:
6055 // int x = 0;
6056 // for (..) {
6057 // int x_priv = 0;
6058 // <first part>
6059 // x = x_priv + x;
6060 // x_priv = x;
6061 // <second part>
6062 // }
6063 // and
6064 // int x = 0;
6065 // #pragma omp simd reduction(inscan, +: x)
6066 // for (..) {
6067 // <first part>
6068 // #pragma omp scan exclusive(x)
6069 // <second part>
6070 // }
6071 // to
6072 // int x = 0;
6073 // for (..) {
6074 // int x_priv = 0;
6075 // <second part>
6076 // int temp = x;
6077 // x = x_priv + x;
6078 // x_priv = temp;
6079 // <first part>
6080 // }
6081 llvm::BasicBlock *OMPScanReduce = createBasicBlock(name: "omp.inscan.reduce");
6082 EmitBranch(Block: IsInclusive
6083 ? OMPScanReduce
6084 : BreakContinueStack.back().ContinueBlock.getBlock());
6085 EmitBlock(BB: OMPScanDispatch);
6086 {
6087 // New scope for correct construction/destruction of temp variables for
6088 // exclusive scan.
6089 LexicalScope Scope(*this, S.getSourceRange());
6090 EmitBranch(Block: IsInclusive ? OMPBeforeScanBlock : OMPAfterScanBlock);
6091 EmitBlock(BB: OMPScanReduce);
6092 if (!IsInclusive) {
6093 // Create temp var and copy LHS value to this temp value.
6094 // TMP = LHS;
6095 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6096 const Expr *PrivateExpr = Privates[I];
6097 const Expr *TempExpr = CopyArrayTemps[I];
6098 EmitAutoVarDecl(
6099 D: *cast<VarDecl>(Val: cast<DeclRefExpr>(Val: TempExpr)->getDecl()));
6100 LValue DestLVal = EmitLValue(E: TempExpr);
6101 LValue SrcLVal = EmitLValue(E: LHSs[I]);
6102 EmitOMPCopy(OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(),
6103 SrcAddr: SrcLVal.getAddress(),
6104 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6105 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()),
6106 Copy: CopyOps[I]);
6107 }
6108 }
6109 CGM.getOpenMPRuntime().emitReduction(
6110 CGF&: *this, Loc: ParentDir.getEndLoc(), Privates, LHSExprs: LHSs, RHSExprs: RHSs, ReductionOps,
6111 Options: {/*WithNowait=*/true, /*SimpleReduction=*/true,
6112 /*IsPrivateVarReduction*/ {}, .ReductionKind: OMPD_simd});
6113 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6114 const Expr *PrivateExpr = Privates[I];
6115 LValue DestLVal;
6116 LValue SrcLVal;
6117 if (IsInclusive) {
6118 DestLVal = EmitLValue(E: RHSs[I]);
6119 SrcLVal = EmitLValue(E: LHSs[I]);
6120 } else {
6121 const Expr *TempExpr = CopyArrayTemps[I];
6122 DestLVal = EmitLValue(E: RHSs[I]);
6123 SrcLVal = EmitLValue(E: TempExpr);
6124 }
6125 EmitOMPCopy(
6126 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6127 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6128 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6129 }
6130 }
6131 EmitBranch(Block: IsInclusive ? OMPAfterScanBlock : OMPBeforeScanBlock);
6132 OMPScanExitBlock = IsInclusive
6133 ? BreakContinueStack.back().ContinueBlock.getBlock()
6134 : OMPScanReduce;
6135 EmitBlock(BB: OMPAfterScanBlock);
6136 return;
6137 }
6138 if (!IsInclusive) {
6139 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6140 EmitBlock(BB: OMPScanExitBlock);
6141 }
6142 if (OMPFirstScanLoop) {
6143 // Emit buffer[i] = red; at the end of the input phase.
6144 const auto *IVExpr = cast<OMPLoopDirective>(Val: ParentDir)
6145 .getIterationVariable()
6146 ->IgnoreParenImpCasts();
6147 LValue IdxLVal = EmitLValue(E: IVExpr);
6148 llvm::Value *IdxVal = EmitLoadOfScalar(lvalue: IdxLVal, Loc: IVExpr->getExprLoc());
6149 IdxVal = Builder.CreateIntCast(V: IdxVal, DestTy: SizeTy, /*isSigned=*/false);
6150 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6151 const Expr *PrivateExpr = Privates[I];
6152 const Expr *OrigExpr = Shareds[I];
6153 const Expr *CopyArrayElem = CopyArrayElems[I];
6154 OpaqueValueMapping IdxMapping(
6155 *this,
6156 cast<OpaqueValueExpr>(
6157 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
6158 RValue::get(V: IdxVal));
6159 LValue DestLVal = EmitLValue(E: CopyArrayElem);
6160 LValue SrcLVal = EmitLValue(E: OrigExpr);
6161 EmitOMPCopy(
6162 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6163 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6164 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6165 }
6166 }
6167 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6168 if (IsInclusive) {
6169 EmitBlock(BB: OMPScanExitBlock);
6170 EmitBranch(Block: BreakContinueStack.back().ContinueBlock.getBlock());
6171 }
6172 EmitBlock(BB: OMPScanDispatch);
6173 if (!OMPFirstScanLoop) {
6174 // Emit red = buffer[i]; at the entrance to the scan phase.
6175 const auto *IVExpr = cast<OMPLoopDirective>(Val: ParentDir)
6176 .getIterationVariable()
6177 ->IgnoreParenImpCasts();
6178 LValue IdxLVal = EmitLValue(E: IVExpr);
6179 llvm::Value *IdxVal = EmitLoadOfScalar(lvalue: IdxLVal, Loc: IVExpr->getExprLoc());
6180 IdxVal = Builder.CreateIntCast(V: IdxVal, DestTy: SizeTy, /*isSigned=*/false);
6181 llvm::BasicBlock *ExclusiveExitBB = nullptr;
6182 if (!IsInclusive) {
6183 llvm::BasicBlock *ContBB = createBasicBlock(name: "omp.exclusive.dec");
6184 ExclusiveExitBB = createBasicBlock(name: "omp.exclusive.copy.exit");
6185 llvm::Value *Cmp = Builder.CreateIsNull(Arg: IdxVal);
6186 Builder.CreateCondBr(Cond: Cmp, True: ExclusiveExitBB, False: ContBB);
6187 EmitBlock(BB: ContBB);
6188 // Use idx - 1 iteration for exclusive scan.
6189 IdxVal = Builder.CreateNUWSub(LHS: IdxVal, RHS: llvm::ConstantInt::get(Ty: SizeTy, V: 1));
6190 }
6191 for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
6192 const Expr *PrivateExpr = Privates[I];
6193 const Expr *OrigExpr = Shareds[I];
6194 const Expr *CopyArrayElem = CopyArrayElems[I];
6195 OpaqueValueMapping IdxMapping(
6196 *this,
6197 cast<OpaqueValueExpr>(
6198 Val: cast<ArraySubscriptExpr>(Val: CopyArrayElem)->getIdx()),
6199 RValue::get(V: IdxVal));
6200 LValue SrcLVal = EmitLValue(E: CopyArrayElem);
6201 LValue DestLVal = EmitLValue(E: OrigExpr);
6202 EmitOMPCopy(
6203 OriginalType: PrivateExpr->getType(), DestAddr: DestLVal.getAddress(), SrcAddr: SrcLVal.getAddress(),
6204 DestVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSs[I])->getDecl()),
6205 SrcVD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSs[I])->getDecl()), Copy: CopyOps[I]);
6206 }
6207 if (!IsInclusive) {
6208 EmitBlock(BB: ExclusiveExitBB);
6209 }
6210 }
6211 EmitBranch(Block: (OMPFirstScanLoop == IsInclusive) ? OMPBeforeScanBlock
6212 : OMPAfterScanBlock);
6213 EmitBlock(BB: OMPAfterScanBlock);
6214}
6215
6216void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
6217 const CodeGenLoopTy &CodeGenLoop,
6218 Expr *IncExpr) {
6219 // Emit the loop iteration variable.
6220 const auto *IVExpr = cast<DeclRefExpr>(Val: S.getIterationVariable());
6221 const auto *IVDecl = cast<VarDecl>(Val: IVExpr->getDecl());
6222 EmitVarDecl(D: *IVDecl);
6223
6224 // Emit the iterations count variable.
6225 // If it is not a variable, Sema decided to calculate iterations count on each
6226 // iteration (e.g., it is foldable into a constant).
6227 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
6228 EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
6229 // Emit calculation of the iterations count.
6230 EmitIgnoredExpr(E: S.getCalcLastIteration());
6231 }
6232
6233 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
6234
6235 bool HasLastprivateClause = false;
6236 // Check pre-condition.
6237 {
6238 OMPLoopScope PreInitScope(*this, S);
6239 // Skip the entire loop if we don't meet the precondition.
6240 // If the condition constant folds and can be elided, avoid emitting the
6241 // whole loop.
6242 bool CondConstant;
6243 llvm::BasicBlock *ContBlock = nullptr;
6244 if (ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
6245 if (!CondConstant)
6246 return;
6247 } else {
6248 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "omp.precond.then");
6249 ContBlock = createBasicBlock(name: "omp.precond.end");
6250 emitPreCond(CGF&: *this, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
6251 TrueCount: getProfileCount(S: &S));
6252 EmitBlock(BB: ThenBlock);
6253 incrementProfileCounter(S: &S);
6254 }
6255
6256 emitAlignedClause(CGF&: *this, D: S);
6257 // Emit 'then' code.
6258 {
6259 // Emit helper vars inits.
6260
6261 LValue LB = EmitOMPHelperVar(
6262 CGF&: *this, Helper: cast<DeclRefExpr>(
6263 Val: (isOpenMPLoopBoundSharingDirective(Kind: S.getDirectiveKind())
6264 ? S.getCombinedLowerBoundVariable()
6265 : S.getLowerBoundVariable())));
6266 LValue UB = EmitOMPHelperVar(
6267 CGF&: *this, Helper: cast<DeclRefExpr>(
6268 Val: (isOpenMPLoopBoundSharingDirective(Kind: S.getDirectiveKind())
6269 ? S.getCombinedUpperBoundVariable()
6270 : S.getUpperBoundVariable())));
6271 LValue ST =
6272 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()));
6273 LValue IL =
6274 EmitOMPHelperVar(CGF&: *this, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()));
6275
6276 OMPPrivateScope LoopScope(*this);
6277 if (EmitOMPFirstprivateClause(D: S, PrivateScope&: LoopScope)) {
6278 // Emit implicit barrier to synchronize threads and avoid data races
6279 // on initialization of firstprivate variables and post-update of
6280 // lastprivate variables.
6281 CGM.getOpenMPRuntime().emitBarrierCall(
6282 CGF&: *this, Loc: S.getBeginLoc(), Kind: OMPD_unknown, /*EmitChecks=*/false,
6283 /*ForceSimpleCall=*/true);
6284 }
6285 EmitOMPPrivateClause(D: S, PrivateScope&: LoopScope);
6286 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()) &&
6287 !isOpenMPParallelDirective(DKind: S.getDirectiveKind()) &&
6288 !isOpenMPTeamsDirective(DKind: S.getDirectiveKind()))
6289 EmitOMPReductionClauseInit(D: S, PrivateScope&: LoopScope);
6290 HasLastprivateClause = EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
6291 EmitOMPPrivateLoopCounters(S, LoopScope);
6292 (void)LoopScope.Privatize();
6293 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
6294 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF&: *this, D: S);
6295
6296 // Detect the distribute schedule kind and chunk.
6297 llvm::Value *Chunk = nullptr;
6298 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
6299 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
6300 ScheduleKind = C->getDistScheduleKind();
6301 if (const Expr *Ch = C->getChunkSize()) {
6302 Chunk = EmitScalarExpr(E: Ch);
6303 Chunk = EmitScalarConversion(Src: Chunk, SrcTy: Ch->getType(),
6304 DstTy: S.getIterationVariable()->getType(),
6305 Loc: S.getBeginLoc());
6306 }
6307 } else {
6308 // Default behaviour for dist_schedule clause.
6309 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
6310 CGF&: *this, S, ScheduleKind, Chunk);
6311 }
6312 const unsigned IVSize = getContext().getTypeSize(T: IVExpr->getType());
6313 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
6314
6315 // GPU fused schedule: omit the outer distribute loop and let the inner
6316 // worksharing loop schedule the flattened team/thread iteration space.
6317 if (canEmitGPUFusedDistSchedule(CGM, S, DKind: S.getDirectiveKind())) {
6318 JumpDest LoopExit =
6319 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
6320 CodeGenLoop(*this, S, LoopExit);
6321 EmitBlock(BB: LoopExit.getBlock());
6322 } else {
6323 // OpenMP [2.10.8, distribute Construct, Description]
6324 // If dist_schedule is specified, kind must be static. If specified,
6325 // iterations are divided into chunks of size chunk_size, chunks are
6326 // assigned to the teams of the league in a round-robin fashion in the
6327 // order of the team number. When no chunk_size is specified, the
6328 // iteration space is divided into chunks that are approximately equal
6329 // in size, and at most one chunk is distributed to each team of the
6330 // league. The size of the chunks is unspecified in this case.
6331 bool StaticChunked =
6332 RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
6333 isOpenMPLoopBoundSharingDirective(Kind: S.getDirectiveKind());
6334 if (RT.isStaticNonchunked(ScheduleKind,
6335 /* Chunked */ Chunk != nullptr) ||
6336 StaticChunked) {
6337 CGOpenMPRuntime::StaticRTInput StaticInit(
6338 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
6339 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6340 StaticChunked ? Chunk : nullptr);
6341 RT.emitDistributeStaticInit(CGF&: *this, Loc: S.getBeginLoc(), SchedKind: ScheduleKind,
6342 Values: StaticInit);
6343 JumpDest LoopExit =
6344 getJumpDestInCurrentScope(Target: createBasicBlock(name: "omp.loop.exit"));
6345 // UB = min(UB, GlobalUB);
6346 EmitIgnoredExpr(
6347 E: isOpenMPLoopBoundSharingDirective(Kind: S.getDirectiveKind())
6348 ? S.getCombinedEnsureUpperBound()
6349 : S.getEnsureUpperBound());
6350 // IV = LB;
6351 EmitIgnoredExpr(
6352 E: isOpenMPLoopBoundSharingDirective(Kind: S.getDirectiveKind())
6353 ? S.getCombinedInit()
6354 : S.getInit());
6355
6356 const Expr *Cond =
6357 isOpenMPLoopBoundSharingDirective(Kind: S.getDirectiveKind())
6358 ? S.getCombinedCond()
6359 : S.getCond();
6360
6361 if (StaticChunked)
6362 Cond = S.getCombinedDistCond();
6363
6364 // For static unchunked schedules generate:
6365 //
6366 // 1. For distribute alone, codegen
6367 // while (idx <= UB) {
6368 // BODY;
6369 // ++idx;
6370 // }
6371 //
6372 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
6373 // while (idx <= UB) {
6374 // <CodeGen rest of pragma>(LB, UB);
6375 // idx += ST;
6376 // }
6377 //
6378 // For static chunk one schedule generate:
6379 //
6380 // while (IV <= GlobalUB) {
6381 // <CodeGen rest of pragma>(LB, UB);
6382 // LB += ST;
6383 // UB += ST;
6384 // UB = min(UB, GlobalUB);
6385 // IV = LB;
6386 // }
6387 //
6388 emitCommonSimdLoop(
6389 CGF&: *this, S,
6390 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6391 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()))
6392 CGF.EmitOMPSimdInit(D: S);
6393 },
6394 BodyCodeGen: [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
6395 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
6396 CGF.EmitOMPInnerLoop(
6397 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: Cond, IncExpr,
6398 BodyGen: [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
6399 CodeGenLoop(CGF, S, LoopExit);
6400 },
6401 PostIncGen: [&S, StaticChunked](CodeGenFunction &CGF) {
6402 if (StaticChunked) {
6403 CGF.EmitIgnoredExpr(E: S.getCombinedNextLowerBound());
6404 CGF.EmitIgnoredExpr(E: S.getCombinedNextUpperBound());
6405 CGF.EmitIgnoredExpr(E: S.getCombinedEnsureUpperBound());
6406 CGF.EmitIgnoredExpr(E: S.getCombinedInit());
6407 }
6408 });
6409 });
6410 EmitBlock(BB: LoopExit.getBlock());
6411 // Tell the runtime we are done.
6412 RT.emitForStaticFinish(CGF&: *this, Loc: S.getEndLoc(), DKind: OMPD_distribute);
6413 } else {
6414 // Emit the outer loop, which requests its work chunk [LB..UB] from
6415 // runtime and runs the inner loop to process it.
6416 const OMPLoopArguments LoopArguments = {
6417 LB.getAddress(), UB.getAddress(), ST.getAddress(),
6418 IL.getAddress(), Chunk};
6419 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArgs: LoopArguments,
6420 CodeGenLoopContent: CodeGenLoop);
6421 }
6422 }
6423 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind())) {
6424 EmitOMPSimdFinal(D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
6425 return CGF.Builder.CreateIsNotNull(
6426 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
6427 });
6428 }
6429 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()) &&
6430 !isOpenMPParallelDirective(DKind: S.getDirectiveKind()) &&
6431 !isOpenMPTeamsDirective(DKind: S.getDirectiveKind())) {
6432 EmitOMPReductionClauseFinal(D: S, ReductionKind: OMPD_simd);
6433 // Emit post-update of the reduction variables if IsLastIter != 0.
6434 emitPostUpdateForReductionClause(
6435 CGF&: *this, D: S, CondGen: [IL, &S](CodeGenFunction &CGF) {
6436 return CGF.Builder.CreateIsNotNull(
6437 Arg: CGF.EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc()));
6438 });
6439 }
6440 // Emit final copy of the lastprivate variables if IsLastIter != 0.
6441 if (HasLastprivateClause) {
6442 EmitOMPLastprivateClauseFinal(
6443 D: S, /*NoFinals=*/false,
6444 IsLastIterCond: Builder.CreateIsNotNull(Arg: EmitLoadOfScalar(lvalue: IL, Loc: S.getBeginLoc())));
6445 }
6446 }
6447
6448 // We're now done with the loop, so jump to the continuation block.
6449 if (ContBlock) {
6450 EmitBranch(Block: ContBlock);
6451 EmitBlock(BB: ContBlock, IsFinished: true);
6452 }
6453 }
6454}
6455
6456// Pass OMPLoopDirective (instead of OMPDistributeDirective) to make this
6457// function available for "loop bind(teams)", which maps to "distribute".
6458static void emitOMPDistributeDirective(const OMPLoopDirective &S,
6459 CodeGenFunction &CGF,
6460 CodeGenModule &CGM) {
6461 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
6462 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
6463 };
6464 OMPLexicalScope Scope(CGF, S, OMPD_unknown);
6465 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute, CodeGen);
6466}
6467
6468void CodeGenFunction::EmitOMPDistributeDirective(
6469 const OMPDistributeDirective &S) {
6470 emitOMPDistributeDirective(S, CGF&: *this, CGM);
6471}
6472
6473static llvm::Function *
6474emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S,
6475 const OMPExecutableDirective &D) {
6476 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
6477 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
6478 CGF.CapturedStmtInfo = &CapStmtInfo;
6479 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(S: *S, D);
6480 Fn->setDoesNotRecurse();
6481 return Fn;
6482}
6483
6484template <typename T>
6485static void emitRestoreIP(CodeGenFunction &CGF, const T *C,
6486 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6487 llvm::OpenMPIRBuilder &OMPBuilder) {
6488
6489 unsigned NumLoops = C->getNumLoops();
6490 QualType Int64Ty = CGF.CGM.getContext().getIntTypeForBitwidth(
6491 /*DestWidth=*/64, /*Signed=*/1);
6492 llvm::SmallVector<llvm::Value *> StoreValues;
6493 for (unsigned I = 0; I < NumLoops; I++) {
6494 const Expr *CounterVal = C->getLoopData(I);
6495 assert(CounterVal);
6496 llvm::Value *StoreValue = CGF.EmitScalarConversion(
6497 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
6498 Loc: CounterVal->getExprLoc());
6499 StoreValues.emplace_back(Args&: StoreValue);
6500 }
6501 OMPDoacrossKind<T> ODK;
6502 bool IsDependSource = ODK.isSource(C);
6503 CGF.Builder.restoreIP(
6504 IP: OMPBuilder.createOrderedDepend(Loc: CGF.Builder, AllocaIP, NumLoops,
6505 StoreValues, Name: ".cnt.addr", IsDependSource));
6506}
6507
6508void CodeGenFunction::EmitOMPOrderedStandaloneDirective(
6509 const OMPOrderedStandaloneDirective &S) {
6510 assert((S.hasClausesOfKind<OMPDependClause>() ||
6511 S.hasClausesOfKind<OMPDoacrossClause>()) &&
6512 "Standalone ordered directive should have either depend or doacross "
6513 "clause");
6514 // The ordered-standalone directive.
6515 assert(!S.hasAssociatedStmt() && "No associated statement must be in "
6516 "ordered depend|doacross construct.");
6517
6518 if (CGM.getLangOpts().OpenMPIRBuilder) {
6519 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6520 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6521
6522 InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
6523 AllocaInsertPt->getIterator());
6524 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6525 emitRestoreIP(CGF&: *this, C: DC, AllocaIP, OMPBuilder);
6526 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6527 emitRestoreIP(CGF&: *this, C: DC, AllocaIP, OMPBuilder);
6528 return;
6529 }
6530
6531 if (S.hasClausesOfKind<OMPDependClause>()) {
6532 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
6533 CGM.getOpenMPRuntime().emitDoacrossOrdered(CGF&: *this, C: DC);
6534 } else if (S.hasClausesOfKind<OMPDoacrossClause>()) {
6535 for (const auto *DC : S.getClausesOfKind<OMPDoacrossClause>())
6536 CGM.getOpenMPRuntime().emitDoacrossOrdered(CGF&: *this, C: DC);
6537 }
6538}
6539
6540void CodeGenFunction::EmitOMPOrderedBlockAssocDirective(
6541 const OMPOrderedBlockAssocDirective &S) {
6542 if (CGM.getLangOpts().OpenMPIRBuilder) {
6543 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
6544 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
6545
6546 // The ordered directive with threads or simd clause, or without clause.
6547 // Without clause, it behaves as if the threads clause is specified.
6548 const auto *C = S.getSingleClause<OMPSIMDClause>();
6549
6550 auto FiniCB = [this](InsertPointTy IP) {
6551 OMPBuilderCBHelpers::FinalizeOMPRegion(CGF&: *this, IP);
6552 return llvm::Error::success();
6553 };
6554
6555 auto BodyGenCB = [&S, C, this](InsertPointTy AllocIP,
6556 InsertPointTy CodeGenIP,
6557 ArrayRef<llvm::BasicBlock *> DeallocBlocks) {
6558 Builder.restoreIP(IP: CodeGenIP);
6559
6560 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6561 if (C) {
6562 llvm::BasicBlock *FiniBB = splitBBWithSuffix(
6563 Builder, /*CreateBranch=*/false, Suffix: ".ordered.after");
6564 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
6565 GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
6566 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, S: CS, D: S);
6567 assert(S.getBeginLoc().isValid() &&
6568 "Outlined function call location must be valid.");
6569 ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: S.getBeginLoc());
6570 OMPBuilderCBHelpers::EmitCaptureStmt(CGF&: *this, CodeGenIP, FiniBB&: *FiniBB,
6571 Fn: OutlinedFn, Args: CapturedVars);
6572 } else {
6573 OMPBuilderCBHelpers::EmitOMPInlinedRegionBody(
6574 CGF&: *this, RegionBodyStmt: CS->getCapturedStmt(), AllocaIP: AllocIP, CodeGenIP, RegionName: "ordered");
6575 }
6576 return llvm::Error::success();
6577 };
6578
6579 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6580 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
6581 ValOrErr: OMPBuilder.createOrderedThreadsSimd(Loc: Builder, BodyGenCB, FiniCB, IsThreads: !C));
6582 Builder.restoreIP(IP: AfterIP);
6583 return;
6584 }
6585
6586 const auto *C = S.getSingleClause<OMPSIMDClause>();
6587 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
6588 PrePostActionTy &Action) {
6589 const CapturedStmt *CS = S.getInnermostCapturedStmt();
6590 if (C) {
6591 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
6592 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
6593 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, S: CS, D: S);
6594 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc: S.getBeginLoc(),
6595 OutlinedFn, Args: CapturedVars);
6596 } else {
6597 Action.Enter(CGF);
6598 CGF.EmitStmt(S: CS->getCapturedStmt());
6599 }
6600 };
6601 OMPLexicalScope Scope(*this, S, OMPD_unknown);
6602 CGM.getOpenMPRuntime().emitOrderedRegion(CGF&: *this, OrderedOpGen: CodeGen, Loc: S.getBeginLoc(), IsThreads: !C);
6603}
6604
6605static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
6606 QualType SrcType, QualType DestType,
6607 SourceLocation Loc) {
6608 assert(CGF.hasScalarEvaluationKind(DestType) &&
6609 "DestType must have scalar evaluation kind.");
6610 assert(!Val.isAggregate() && "Must be a scalar or complex.");
6611 return Val.isScalar() ? CGF.EmitScalarConversion(Src: Val.getScalarVal(), SrcTy: SrcType,
6612 DstTy: DestType, Loc)
6613 : CGF.EmitComplexToScalarConversion(
6614 Src: Val.getComplexVal(), SrcTy: SrcType, DstTy: DestType, Loc);
6615}
6616
6617static CodeGenFunction::ComplexPairTy
6618convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
6619 QualType DestType, SourceLocation Loc) {
6620 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
6621 "DestType must have complex evaluation kind.");
6622 CodeGenFunction::ComplexPairTy ComplexVal;
6623 if (Val.isScalar()) {
6624 // Convert the input element to the element type of the complex.
6625 QualType DestElementType =
6626 DestType->castAs<ComplexType>()->getElementType();
6627 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
6628 Src: Val.getScalarVal(), SrcTy: SrcType, DstTy: DestElementType, Loc);
6629 ComplexVal = CodeGenFunction::ComplexPairTy(
6630 ScalarVal, llvm::Constant::getNullValue(Ty: ScalarVal->getType()));
6631 } else {
6632 assert(Val.isComplex() && "Must be a scalar or complex.");
6633 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
6634 QualType DestElementType =
6635 DestType->castAs<ComplexType>()->getElementType();
6636 ComplexVal.first = CGF.EmitScalarConversion(
6637 Src: Val.getComplexVal().first, SrcTy: SrcElementType, DstTy: DestElementType, Loc);
6638 ComplexVal.second = CGF.EmitScalarConversion(
6639 Src: Val.getComplexVal().second, SrcTy: SrcElementType, DstTy: DestElementType, Loc);
6640 }
6641 return ComplexVal;
6642}
6643
6644static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6645 LValue LVal, RValue RVal) {
6646 if (LVal.isGlobalReg())
6647 CGF.EmitStoreThroughGlobalRegLValue(Src: RVal, Dst: LVal);
6648 else
6649 CGF.EmitAtomicStore(rvalue: RVal, lvalue: LVal, AO, IsVolatile: LVal.isVolatile(), /*isInit=*/false);
6650}
6651
6652static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
6653 llvm::AtomicOrdering AO, LValue LVal,
6654 SourceLocation Loc) {
6655 if (LVal.isGlobalReg())
6656 return CGF.EmitLoadOfLValue(V: LVal, Loc);
6657 return CGF.EmitAtomicLoad(
6658 lvalue: LVal, loc: Loc, AO: llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO),
6659 IsVolatile: LVal.isVolatile());
6660}
6661
6662void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
6663 QualType RValTy, SourceLocation Loc) {
6664 switch (getEvaluationKind(T: LVal.getType())) {
6665 case TEK_Scalar:
6666 EmitStoreThroughLValue(Src: RValue::get(V: convertToScalarValue(
6667 CGF&: *this, Val: RVal, SrcType: RValTy, DestType: LVal.getType(), Loc)),
6668 Dst: LVal);
6669 break;
6670 case TEK_Complex:
6671 EmitStoreOfComplex(
6672 V: convertToComplexValue(CGF&: *this, Val: RVal, SrcType: RValTy, DestType: LVal.getType(), Loc), dest: LVal,
6673 /*isInit=*/false);
6674 break;
6675 case TEK_Aggregate:
6676 llvm_unreachable("Must be a scalar or complex.");
6677 }
6678}
6679
6680static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
6681 const Expr *X, const Expr *V,
6682 SourceLocation Loc) {
6683 // v = x;
6684 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
6685 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
6686 LValue XLValue = CGF.EmitLValue(E: X);
6687 LValue VLValue = CGF.EmitLValue(E: V);
6688 RValue Res = emitSimpleAtomicLoad(CGF, AO, LVal: XLValue, Loc);
6689 // OpenMP, 2.17.7, atomic Construct
6690 // If the read or capture clause is specified and the acquire, acq_rel, or
6691 // seq_cst clause is specified then the strong flush on exit from the atomic
6692 // operation is also an acquire flush.
6693 switch (AO) {
6694 case llvm::AtomicOrdering::Acquire:
6695 case llvm::AtomicOrdering::AcquireRelease:
6696 case llvm::AtomicOrdering::SequentiallyConsistent:
6697 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
6698 AO: llvm::AtomicOrdering::Acquire);
6699 break;
6700 case llvm::AtomicOrdering::Monotonic:
6701 case llvm::AtomicOrdering::Release:
6702 break;
6703 case llvm::AtomicOrdering::NotAtomic:
6704 case llvm::AtomicOrdering::Unordered:
6705 llvm_unreachable("Unexpected ordering.");
6706 }
6707 CGF.emitOMPSimpleStore(LVal: VLValue, RVal: Res, RValTy: X->getType().getNonReferenceType(), Loc);
6708 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: V);
6709}
6710
6711static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,
6712 llvm::AtomicOrdering AO, const Expr *X,
6713 const Expr *E, SourceLocation Loc) {
6714 // x = expr;
6715 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
6716 emitSimpleAtomicStore(CGF, AO, LVal: CGF.EmitLValue(E: X), RVal: CGF.EmitAnyExpr(E));
6717 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
6718 // OpenMP, 2.17.7, atomic Construct
6719 // If the write, update, or capture clause is specified and the release,
6720 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6721 // the atomic operation is also a release flush.
6722 switch (AO) {
6723 case llvm::AtomicOrdering::Release:
6724 case llvm::AtomicOrdering::AcquireRelease:
6725 case llvm::AtomicOrdering::SequentiallyConsistent:
6726 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
6727 AO: llvm::AtomicOrdering::Release);
6728 break;
6729 case llvm::AtomicOrdering::Acquire:
6730 case llvm::AtomicOrdering::Monotonic:
6731 break;
6732 case llvm::AtomicOrdering::NotAtomic:
6733 case llvm::AtomicOrdering::Unordered:
6734 llvm_unreachable("Unexpected ordering.");
6735 }
6736}
6737
6738static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
6739 RValue Update,
6740 BinaryOperatorKind BO,
6741 llvm::AtomicOrdering AO,
6742 bool IsXLHSInRHSPart) {
6743 ASTContext &Context = CGF.getContext();
6744 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
6745 // expression is simple and atomic is allowed for the given type for the
6746 // target platform.
6747 if (BO == BO_Comma || !Update.isScalar() || !X.isSimple() ||
6748 (!isa<llvm::ConstantInt>(Val: Update.getScalarVal()) &&
6749 (Update.getScalarVal()->getType() != X.getAddress().getElementType())) ||
6750 !Context.getTargetInfo().hasBuiltinAtomic(
6751 AtomicSizeInBits: Context.getTypeSize(T: X.getType()), AlignmentInBits: Context.toBits(CharSize: X.getAlignment())))
6752 return std::make_pair(x: false, y: RValue::get(V: nullptr));
6753
6754 auto &&CheckAtomicSupport = [&CGF](llvm::Type *T, BinaryOperatorKind BO) {
6755 if (T->isIntegerTy())
6756 return true;
6757
6758 if (T->isFloatingPointTy() && (BO == BO_Add || BO == BO_Sub))
6759 return llvm::isPowerOf2_64(Value: CGF.CGM.getDataLayout().getTypeStoreSize(Ty: T));
6760
6761 return false;
6762 };
6763
6764 if (!CheckAtomicSupport(Update.getScalarVal()->getType(), BO) ||
6765 !CheckAtomicSupport(X.getAddress().getElementType(), BO))
6766 return std::make_pair(x: false, y: RValue::get(V: nullptr));
6767
6768 bool IsInteger = X.getAddress().getElementType()->isIntegerTy();
6769 llvm::AtomicRMWInst::BinOp RMWOp;
6770 switch (BO) {
6771 case BO_Add:
6772 RMWOp = IsInteger ? llvm::AtomicRMWInst::Add : llvm::AtomicRMWInst::FAdd;
6773 break;
6774 case BO_Sub:
6775 if (!IsXLHSInRHSPart)
6776 return std::make_pair(x: false, y: RValue::get(V: nullptr));
6777 RMWOp = IsInteger ? llvm::AtomicRMWInst::Sub : llvm::AtomicRMWInst::FSub;
6778 break;
6779 case BO_And:
6780 RMWOp = llvm::AtomicRMWInst::And;
6781 break;
6782 case BO_Or:
6783 RMWOp = llvm::AtomicRMWInst::Or;
6784 break;
6785 case BO_Xor:
6786 RMWOp = llvm::AtomicRMWInst::Xor;
6787 break;
6788 case BO_LT:
6789 if (IsInteger)
6790 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6791 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
6792 : llvm::AtomicRMWInst::Max)
6793 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
6794 : llvm::AtomicRMWInst::UMax);
6795 else
6796 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMin
6797 : llvm::AtomicRMWInst::FMax;
6798 break;
6799 case BO_GT:
6800 if (IsInteger)
6801 RMWOp = X.getType()->hasSignedIntegerRepresentation()
6802 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
6803 : llvm::AtomicRMWInst::Min)
6804 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
6805 : llvm::AtomicRMWInst::UMin);
6806 else
6807 RMWOp = IsXLHSInRHSPart ? llvm::AtomicRMWInst::FMax
6808 : llvm::AtomicRMWInst::FMin;
6809 break;
6810 case BO_Assign:
6811 RMWOp = llvm::AtomicRMWInst::Xchg;
6812 break;
6813 case BO_Mul:
6814 case BO_Div:
6815 case BO_Rem:
6816 case BO_Shl:
6817 case BO_Shr:
6818 case BO_LAnd:
6819 case BO_LOr:
6820 return std::make_pair(x: false, y: RValue::get(V: nullptr));
6821 case BO_PtrMemD:
6822 case BO_PtrMemI:
6823 case BO_LE:
6824 case BO_GE:
6825 case BO_EQ:
6826 case BO_NE:
6827 case BO_Cmp:
6828 case BO_AddAssign:
6829 case BO_SubAssign:
6830 case BO_AndAssign:
6831 case BO_OrAssign:
6832 case BO_XorAssign:
6833 case BO_MulAssign:
6834 case BO_DivAssign:
6835 case BO_RemAssign:
6836 case BO_ShlAssign:
6837 case BO_ShrAssign:
6838 case BO_Comma:
6839 llvm_unreachable("Unsupported atomic update operation");
6840 }
6841 llvm::Value *UpdateVal = Update.getScalarVal();
6842 if (auto *IC = dyn_cast<llvm::ConstantInt>(Val: UpdateVal)) {
6843 if (IsInteger)
6844 UpdateVal = CGF.Builder.CreateIntCast(
6845 V: IC, DestTy: X.getAddress().getElementType(),
6846 isSigned: X.getType()->hasSignedIntegerRepresentation());
6847 else
6848 UpdateVal = CGF.Builder.CreateCast(Op: llvm::Instruction::CastOps::UIToFP, V: IC,
6849 DestTy: X.getAddress().getElementType());
6850 }
6851 llvm::AtomicRMWInst *Res =
6852 CGF.emitAtomicRMWInst(Op: RMWOp, Addr: X.getAddress(), Val: UpdateVal, Order: AO);
6853 return std::make_pair(x: true, y: RValue::get(V: Res));
6854}
6855
6856std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
6857 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
6858 llvm::AtomicOrdering AO, SourceLocation Loc,
6859 const llvm::function_ref<RValue(RValue)> CommonGen) {
6860 // Update expressions are allowed to have the following forms:
6861 // x binop= expr; -> xrval + expr;
6862 // x++, ++x -> xrval + 1;
6863 // x--, --x -> xrval - 1;
6864 // x = x binop expr; -> xrval binop expr
6865 // x = expr Op x; - > expr binop xrval;
6866 auto Res = emitOMPAtomicRMW(CGF&: *this, X, Update: E, BO, AO, IsXLHSInRHSPart);
6867 if (!Res.first) {
6868 if (X.isGlobalReg()) {
6869 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
6870 // 'xrval'.
6871 EmitStoreThroughLValue(Src: CommonGen(EmitLoadOfLValue(V: X, Loc)), Dst: X);
6872 } else {
6873 // Perform compare-and-swap procedure.
6874 EmitAtomicUpdate(LVal: X, AO, UpdateOp: CommonGen, IsVolatile: X.getType().isVolatileQualified());
6875 }
6876 }
6877 return Res;
6878}
6879
6880static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,
6881 llvm::AtomicOrdering AO, const Expr *X,
6882 const Expr *E, const Expr *UE,
6883 bool IsXLHSInRHSPart, SourceLocation Loc) {
6884 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6885 "Update expr in 'atomic update' must be a binary operator.");
6886 const auto *BOUE = cast<BinaryOperator>(Val: UE->IgnoreImpCasts());
6887 // Update expressions are allowed to have the following forms:
6888 // x binop= expr; -> xrval + expr;
6889 // x++, ++x -> xrval + 1;
6890 // x--, --x -> xrval - 1;
6891 // x = x binop expr; -> xrval binop expr
6892 // x = expr Op x; - > expr binop xrval;
6893 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
6894 LValue XLValue = CGF.EmitLValue(E: X);
6895 RValue ExprRValue = CGF.EmitAnyExpr(E);
6896 const auto *LHS = cast<OpaqueValueExpr>(Val: BOUE->getLHS()->IgnoreImpCasts());
6897 const auto *RHS = cast<OpaqueValueExpr>(Val: BOUE->getRHS()->IgnoreImpCasts());
6898 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
6899 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
6900 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
6901 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6902 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6903 return CGF.EmitAnyExpr(E: UE);
6904 };
6905 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
6906 X: XLValue, E: ExprRValue, BO: BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, CommonGen: Gen);
6907 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
6908 // OpenMP, 2.17.7, atomic Construct
6909 // If the write, update, or capture clause is specified and the release,
6910 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
6911 // the atomic operation is also a release flush.
6912 switch (AO) {
6913 case llvm::AtomicOrdering::Release:
6914 case llvm::AtomicOrdering::AcquireRelease:
6915 case llvm::AtomicOrdering::SequentiallyConsistent:
6916 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
6917 AO: llvm::AtomicOrdering::Release);
6918 break;
6919 case llvm::AtomicOrdering::Acquire:
6920 case llvm::AtomicOrdering::Monotonic:
6921 break;
6922 case llvm::AtomicOrdering::NotAtomic:
6923 case llvm::AtomicOrdering::Unordered:
6924 llvm_unreachable("Unexpected ordering.");
6925 }
6926}
6927
6928static RValue convertToType(CodeGenFunction &CGF, RValue Value,
6929 QualType SourceType, QualType ResType,
6930 SourceLocation Loc) {
6931 switch (CGF.getEvaluationKind(T: ResType)) {
6932 case TEK_Scalar:
6933 return RValue::get(
6934 V: convertToScalarValue(CGF, Val: Value, SrcType: SourceType, DestType: ResType, Loc));
6935 case TEK_Complex: {
6936 auto Res = convertToComplexValue(CGF, Val: Value, SrcType: SourceType, DestType: ResType, Loc);
6937 return RValue::getComplex(V1: Res.first, V2: Res.second);
6938 }
6939 case TEK_Aggregate:
6940 break;
6941 }
6942 llvm_unreachable("Must be a scalar or complex.");
6943}
6944
6945static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
6946 llvm::AtomicOrdering AO,
6947 bool IsPostfixUpdate, const Expr *V,
6948 const Expr *X, const Expr *E,
6949 const Expr *UE, bool IsXLHSInRHSPart,
6950 SourceLocation Loc) {
6951 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
6952 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
6953 RValue NewVVal;
6954 LValue VLValue = CGF.EmitLValue(E: V);
6955 LValue XLValue = CGF.EmitLValue(E: X);
6956 RValue ExprRValue = CGF.EmitAnyExpr(E);
6957 QualType NewVValType;
6958 if (UE) {
6959 // 'x' is updated with some additional value.
6960 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
6961 "Update expr in 'atomic capture' must be a binary operator.");
6962 const auto *BOUE = cast<BinaryOperator>(Val: UE->IgnoreImpCasts());
6963 // Update expressions are allowed to have the following forms:
6964 // x binop= expr; -> xrval + expr;
6965 // x++, ++x -> xrval + 1;
6966 // x--, --x -> xrval - 1;
6967 // x = x binop expr; -> xrval binop expr
6968 // x = expr Op x; - > expr binop xrval;
6969 const auto *LHS = cast<OpaqueValueExpr>(Val: BOUE->getLHS()->IgnoreImpCasts());
6970 const auto *RHS = cast<OpaqueValueExpr>(Val: BOUE->getRHS()->IgnoreImpCasts());
6971 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
6972 NewVValType = XRValExpr->getType();
6973 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
6974 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
6975 IsPostfixUpdate](RValue XRValue) {
6976 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6977 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
6978 RValue Res = CGF.EmitAnyExpr(E: UE);
6979 NewVVal = IsPostfixUpdate ? XRValue : Res;
6980 return Res;
6981 };
6982 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
6983 X: XLValue, E: ExprRValue, BO: BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, CommonGen: Gen);
6984 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
6985 if (Res.first) {
6986 // 'atomicrmw' instruction was generated.
6987 if (IsPostfixUpdate) {
6988 // Use old value from 'atomicrmw'.
6989 NewVVal = Res.second;
6990 } else {
6991 // 'atomicrmw' does not provide new value, so evaluate it using old
6992 // value of 'x'.
6993 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
6994 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
6995 NewVVal = CGF.EmitAnyExpr(E: UE);
6996 }
6997 }
6998 } else {
6999 // 'x' is simply rewritten with some 'expr'.
7000 NewVValType = X->getType().getNonReferenceType();
7001 ExprRValue = convertToType(CGF, Value: ExprRValue, SourceType: E->getType(),
7002 ResType: X->getType().getNonReferenceType(), Loc);
7003 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
7004 NewVVal = XRValue;
7005 return ExprRValue;
7006 };
7007 // Try to perform atomicrmw xchg, otherwise simple exchange.
7008 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
7009 X: XLValue, E: ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
7010 Loc, CommonGen: Gen);
7011 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: X);
7012 if (Res.first) {
7013 // 'atomicrmw' instruction was generated.
7014 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
7015 }
7016 }
7017 // Emit post-update store to 'v' of old/new 'x' value.
7018 CGF.emitOMPSimpleStore(LVal: VLValue, RVal: NewVVal, RValTy: NewVValType, Loc);
7019 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, LHS: V);
7020 // OpenMP 5.1 removes the required flush for capture clause.
7021 if (CGF.CGM.getLangOpts().OpenMP < 51) {
7022 // OpenMP, 2.17.7, atomic Construct
7023 // If the write, update, or capture clause is specified and the release,
7024 // acq_rel, or seq_cst clause is specified then the strong flush on entry to
7025 // the atomic operation is also a release flush.
7026 // If the read or capture clause is specified and the acquire, acq_rel, or
7027 // seq_cst clause is specified then the strong flush on exit from the atomic
7028 // operation is also an acquire flush.
7029 switch (AO) {
7030 case llvm::AtomicOrdering::Release:
7031 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7032 AO: llvm::AtomicOrdering::Release);
7033 break;
7034 case llvm::AtomicOrdering::Acquire:
7035 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, Vars: {}, Loc,
7036 AO: llvm::AtomicOrdering::Acquire);
7037 break;
7038 case llvm::AtomicOrdering::AcquireRelease:
7039 case llvm::AtomicOrdering::SequentiallyConsistent:
7040 CGF.CGM.getOpenMPRuntime().emitFlush(
7041 CGF, Vars: {}, Loc, AO: llvm::AtomicOrdering::AcquireRelease);
7042 break;
7043 case llvm::AtomicOrdering::Monotonic:
7044 break;
7045 case llvm::AtomicOrdering::NotAtomic:
7046 case llvm::AtomicOrdering::Unordered:
7047 llvm_unreachable("Unexpected ordering.");
7048 }
7049 }
7050}
7051
7052static void emitOMPAtomicCompareExpr(
7053 CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
7054 const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,
7055 const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,
7056 SourceLocation Loc) {
7057 llvm::OpenMPIRBuilder &OMPBuilder =
7058 CGF.CGM.getOpenMPRuntime().getOMPBuilder();
7059
7060 OMPAtomicCompareOp Op;
7061 assert(isa<BinaryOperator>(CE) && "CE is not a BinaryOperator");
7062 switch (cast<BinaryOperator>(Val: CE)->getOpcode()) {
7063 case BO_EQ:
7064 Op = OMPAtomicCompareOp::EQ;
7065 break;
7066 case BO_LT:
7067 Op = OMPAtomicCompareOp::MIN;
7068 break;
7069 case BO_GT:
7070 Op = OMPAtomicCompareOp::MAX;
7071 break;
7072 default:
7073 llvm_unreachable("unsupported atomic compare binary operator");
7074 }
7075
7076 LValue XLVal = CGF.EmitLValue(E: X);
7077 Address XAddr = XLVal.getAddress();
7078
7079 auto EmitRValueWithCastIfNeeded = [&CGF, Loc](const Expr *X, const Expr *E) {
7080 if (X->getType() == E->getType())
7081 return CGF.EmitScalarExpr(E);
7082 const Expr *NewE = E->IgnoreImplicitAsWritten();
7083 llvm::Value *V = CGF.EmitScalarExpr(E: NewE);
7084 if (NewE->getType() == X->getType())
7085 return V;
7086 return CGF.EmitScalarConversion(Src: V, SrcTy: NewE->getType(), DstTy: X->getType(), Loc);
7087 };
7088
7089 llvm::Value *EVal = EmitRValueWithCastIfNeeded(X, E);
7090 llvm::Value *DVal = D ? EmitRValueWithCastIfNeeded(X, D) : nullptr;
7091 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: EVal))
7092 EVal = CGF.Builder.CreateIntCast(
7093 V: CI, DestTy: XLVal.getAddress().getElementType(),
7094 isSigned: E->getType()->hasSignedIntegerRepresentation());
7095 if (DVal)
7096 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: DVal))
7097 DVal = CGF.Builder.CreateIntCast(
7098 V: CI, DestTy: XLVal.getAddress().getElementType(),
7099 isSigned: D->getType()->hasSignedIntegerRepresentation());
7100
7101 llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
7102 .Var: XAddr.emitRawPointer(CGF), .ElemTy: XAddr.getElementType(),
7103 .IsSigned: X->getType()->hasSignedIntegerRepresentation(),
7104 .IsVolatile: X->getType().isVolatileQualified()};
7105 llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
7106 if (V) {
7107 LValue LV = CGF.EmitLValue(E: V);
7108 Address Addr = LV.getAddress();
7109 VOpVal = {.Var: Addr.emitRawPointer(CGF), .ElemTy: Addr.getElementType(),
7110 .IsSigned: V->getType()->hasSignedIntegerRepresentation(),
7111 .IsVolatile: V->getType().isVolatileQualified()};
7112 }
7113 if (R) {
7114 LValue LV = CGF.EmitLValue(E: R);
7115 Address Addr = LV.getAddress();
7116 ROpVal = {.Var: Addr.emitRawPointer(CGF), .ElemTy: Addr.getElementType(),
7117 .IsSigned: R->getType()->hasSignedIntegerRepresentation(),
7118 .IsVolatile: R->getType().isVolatileQualified()};
7119 }
7120
7121 if (FailAO == llvm::AtomicOrdering::NotAtomic) {
7122 // fail clause was not mentioned on the
7123 // "#pragma omp atomic compare" construct.
7124 CGF.Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
7125 Loc: CGF.Builder, X&: XOpVal, V&: VOpVal, R&: ROpVal, E: EVal, D: DVal, AO, Op, IsXBinopExpr,
7126 IsPostfixUpdate, IsFailOnly));
7127 } else
7128 CGF.Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
7129 Loc: CGF.Builder, X&: XOpVal, V&: VOpVal, R&: ROpVal, E: EVal, D: DVal, AO, Op, IsXBinopExpr,
7130 IsPostfixUpdate, IsFailOnly, Failure: FailAO));
7131}
7132
7133static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
7134 llvm::AtomicOrdering AO,
7135 llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,
7136 const Expr *X, const Expr *V, const Expr *R,
7137 const Expr *E, const Expr *UE, const Expr *D,
7138 const Expr *CE, bool IsXLHSInRHSPart,
7139 bool IsFailOnly, SourceLocation Loc) {
7140 switch (Kind) {
7141 case OMPC_read:
7142 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
7143 break;
7144 case OMPC_write:
7145 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
7146 break;
7147 case OMPC_unknown:
7148 case OMPC_update:
7149 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
7150 break;
7151 case OMPC_capture:
7152 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
7153 IsXLHSInRHSPart, Loc);
7154 break;
7155 case OMPC_compare: {
7156 emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
7157 IsXBinopExpr: IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
7158 break;
7159 }
7160 default:
7161 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
7162 }
7163}
7164
7165void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
7166 llvm::AtomicOrdering AO = CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7167 // Fail Memory Clause Ordering.
7168 llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
7169 bool MemOrderingSpecified = false;
7170 if (S.getSingleClause<OMPSeqCstClause>()) {
7171 AO = llvm::AtomicOrdering::SequentiallyConsistent;
7172 MemOrderingSpecified = true;
7173 } else if (S.getSingleClause<OMPAcqRelClause>()) {
7174 AO = llvm::AtomicOrdering::AcquireRelease;
7175 MemOrderingSpecified = true;
7176 } else if (S.getSingleClause<OMPAcquireClause>()) {
7177 AO = llvm::AtomicOrdering::Acquire;
7178 MemOrderingSpecified = true;
7179 } else if (S.getSingleClause<OMPReleaseClause>()) {
7180 AO = llvm::AtomicOrdering::Release;
7181 MemOrderingSpecified = true;
7182 } else if (S.getSingleClause<OMPRelaxedClause>()) {
7183 AO = llvm::AtomicOrdering::Monotonic;
7184 MemOrderingSpecified = true;
7185 }
7186 llvm::SmallSet<OpenMPClauseKind, 2> KindsEncountered;
7187 OpenMPClauseKind Kind = OMPC_unknown;
7188 for (const OMPClause *C : S.clauses()) {
7189 // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
7190 // if it is first).
7191 OpenMPClauseKind K = C->getClauseKind();
7192 // TBD
7193 if (K == OMPC_weak)
7194 return;
7195 if (K == OMPC_seq_cst || K == OMPC_acq_rel || K == OMPC_acquire ||
7196 K == OMPC_release || K == OMPC_relaxed || K == OMPC_hint)
7197 continue;
7198 Kind = K;
7199 KindsEncountered.insert(V: K);
7200 }
7201 // We just need to correct Kind here. No need to set a bool saying it is
7202 // actually compare capture because we can tell from whether V and R are
7203 // nullptr.
7204 if (KindsEncountered.contains(V: OMPC_compare) &&
7205 KindsEncountered.contains(V: OMPC_capture))
7206 Kind = OMPC_compare;
7207 if (!MemOrderingSpecified) {
7208 llvm::AtomicOrdering DefaultOrder =
7209 CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
7210 if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
7211 DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
7212 (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
7213 Kind == OMPC_capture)) {
7214 AO = DefaultOrder;
7215 } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
7216 if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
7217 AO = llvm::AtomicOrdering::Release;
7218 } else if (Kind == OMPC_read) {
7219 assert(Kind == OMPC_read && "Unexpected atomic kind.");
7220 AO = llvm::AtomicOrdering::Acquire;
7221 }
7222 }
7223 }
7224
7225 if (KindsEncountered.contains(V: OMPC_compare) &&
7226 KindsEncountered.contains(V: OMPC_fail)) {
7227 Kind = OMPC_compare;
7228 const auto *FailClause = S.getSingleClause<OMPFailClause>();
7229 if (FailClause) {
7230 OpenMPClauseKind FailParameter = FailClause->getFailParameter();
7231 if (FailParameter == llvm::omp::OMPC_relaxed)
7232 FailAO = llvm::AtomicOrdering::Monotonic;
7233 else if (FailParameter == llvm::omp::OMPC_acquire)
7234 FailAO = llvm::AtomicOrdering::Acquire;
7235 else if (FailParameter == llvm::omp::OMPC_seq_cst)
7236 FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
7237 }
7238 }
7239
7240 LexicalScope Scope(*this, S.getSourceRange());
7241 EmitStopPoint(S: S.getAssociatedStmt());
7242 emitOMPAtomicExpr(CGF&: *this, Kind, AO, FailAO, IsPostfixUpdate: S.isPostfixUpdate(), X: S.getX(),
7243 V: S.getV(), R: S.getR(), E: S.getExpr(), UE: S.getUpdateExpr(),
7244 D: S.getD(), CE: S.getCondExpr(), IsXLHSInRHSPart: S.isXLHSInRHSPart(),
7245 IsFailOnly: S.isFailOnly(), Loc: S.getBeginLoc());
7246}
7247
7248static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
7249 const OMPExecutableDirective &S,
7250 const RegionCodeGenTy &CodeGen) {
7251 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
7252 CodeGenModule &CGM = CGF.CGM;
7253
7254 // On device emit this construct as inlined code.
7255 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
7256 OMPLexicalScope Scope(CGF, S, OMPD_target);
7257 CGM.getOpenMPRuntime().emitInlinedDirective(
7258 CGF, InnermostKind: OMPD_target, CodeGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7259 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
7260 });
7261 return;
7262 }
7263
7264 auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
7265 llvm::Function *Fn = nullptr;
7266 llvm::Constant *FnID = nullptr;
7267
7268 const Expr *IfCond = nullptr;
7269 // Check for the at most one if clause associated with the target region.
7270 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7271 if (C->getNameModifier() == OMPD_unknown ||
7272 C->getNameModifier() == OMPD_target) {
7273 IfCond = C->getCondition();
7274 break;
7275 }
7276 }
7277
7278 // Check if we have any device clause associated with the directive.
7279 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
7280 nullptr, OMPC_DEVICE_unknown);
7281 if (auto *C = S.getSingleClause<OMPDeviceClause>())
7282 Device.setPointerAndInt(PtrVal: C->getDevice(), IntVal: C->getModifier());
7283
7284 // Check if we have an if clause whose conditional always evaluates to false
7285 // or if we do not have any targets specified. If so the target region is not
7286 // an offload entry point.
7287 bool IsOffloadEntry = true;
7288 if (IfCond) {
7289 bool Val;
7290 if (CGF.ConstantFoldsToSimpleInteger(Cond: IfCond, Result&: Val) && !Val)
7291 IsOffloadEntry = false;
7292 }
7293 if (CGM.getLangOpts().OMPTargetTriples.empty())
7294 IsOffloadEntry = false;
7295
7296 if (CGM.getLangOpts().OpenMPOffloadMandatory && !IsOffloadEntry) {
7297 CGM.getDiags().Report(DiagID: diag::err_missing_mandatory_offloading);
7298 }
7299
7300 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
7301 StringRef ParentName;
7302 // In case we have Ctors/Dtors we use the complete type variant to produce
7303 // the mangling of the device outlined kernel.
7304 if (const auto *D = dyn_cast<CXXConstructorDecl>(Val: CGF.CurFuncDecl))
7305 ParentName = CGM.getMangledName(GD: GlobalDecl(D, Ctor_Complete));
7306 else if (const auto *D = dyn_cast<CXXDestructorDecl>(Val: CGF.CurFuncDecl))
7307 ParentName = CGM.getMangledName(GD: GlobalDecl(D, Dtor_Complete));
7308 else
7309 ParentName =
7310 CGM.getMangledName(GD: GlobalDecl(cast<FunctionDecl>(Val: CGF.CurFuncDecl)));
7311
7312 // Emit target region as a standalone region.
7313 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: FnID,
7314 IsOffloadEntry, CodeGen);
7315 OMPLexicalScope Scope(CGF, S, OMPD_task);
7316 auto &&SizeEmitter =
7317 [IsOffloadEntry](CodeGenFunction &CGF,
7318 const OMPLoopDirective &D) -> llvm::Value * {
7319 if (IsOffloadEntry) {
7320 OMPLoopScope PreInitScope(CGF, D);
7321 // Emit calculation of the iterations count.
7322 llvm::Value *NumIterations = CGF.EmitScalarExpr(E: D.getNumIterations());
7323 NumIterations = CGF.Builder.CreateIntCast(V: NumIterations, DestTy: CGF.Int64Ty,
7324 /*isSigned=*/false);
7325 return NumIterations;
7326 }
7327 return nullptr;
7328 };
7329 CGM.getOpenMPRuntime().emitTargetCall(CGF, D: S, OutlinedFn: Fn, OutlinedFnID: FnID, IfCond, Device,
7330 SizeEmitter);
7331}
7332
7333static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
7334 PrePostActionTy &Action) {
7335 Action.Enter(CGF);
7336 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7337 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7338 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7339 (void)PrivateScope.Privatize();
7340 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
7341 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
7342
7343 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_target)->getCapturedStmt());
7344 CGF.EnsureInsertPoint();
7345}
7346
7347void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
7348 StringRef ParentName,
7349 const OMPTargetDirective &S) {
7350 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7351 emitTargetRegion(CGF, S, Action);
7352 };
7353 llvm::Function *Fn;
7354 llvm::Constant *Addr;
7355 // Emit target region as a standalone region.
7356 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7357 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7358 assert(Fn && Addr && "Target device function emission failed.");
7359}
7360
7361void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
7362 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7363 emitTargetRegion(CGF, S, Action);
7364 };
7365 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7366}
7367
7368static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
7369 const OMPExecutableDirective &S,
7370 OpenMPDirectiveKind InnermostKind,
7371 const RegionCodeGenTy &CodeGen) {
7372 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_teams);
7373 llvm::Function *OutlinedFn =
7374 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
7375 CGF, D: S, ThreadIDVar: *CS->getCapturedDecl()->param_begin(), InnermostKind,
7376 CodeGen);
7377
7378 OMPTeamsScope Scope(CGF, S);
7379 auto ParallelLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7380 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
7381 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7382 if (NT || TL) {
7383 const Expr *NumTeams = NT ? NT->getNumTeams().front() : nullptr;
7384 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7385
7386 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
7387 Loc: S.getBeginLoc());
7388 }
7389 };
7390
7391 const Expr *IfCond = nullptr;
7392 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7393 if (C->getNameModifier() == OMPD_unknown ||
7394 C->getNameModifier() == OMPD_teams) {
7395 IfCond = C->getCondition();
7396 break;
7397 }
7398 }
7399 if (IfCond && CGF.CGM.getLangOpts().OpenMP >= 52) {
7400 auto SerialLeague = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7401 // OpenMP 5.2, 10.2, teams Construct
7402 // When an if clause is present on a teams construct and the if clause
7403 // expression evaluates to false, the number of created teams is one.
7404 const llvm::APInt One(32, 1);
7405 IntegerLiteral NumTeams(
7406 CGF.getContext(), One,
7407 CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
7408 SourceLocation());
7409 // The thread_limit clause is unaffected by the if clause.
7410 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
7411 const Expr *ThreadLimit = TL ? TL->getThreadLimit().front() : nullptr;
7412 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams: &NumTeams, ThreadLimit,
7413 Loc: S.getBeginLoc());
7414 };
7415 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, Cond: IfCond, ThenGen: ParallelLeague,
7416 ElseGen: SerialLeague);
7417 } else {
7418 const RegionCodeGenTy ThenRCG(ParallelLeague);
7419 ThenRCG(CGF);
7420 }
7421
7422 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
7423 CGF.GenerateOpenMPCapturedVars(S: *CS, CapturedVars);
7424 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, D: S, Loc: S.getBeginLoc(), OutlinedFn,
7425 CapturedVars);
7426}
7427
7428void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
7429 // Emit teams region as a standalone region.
7430 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7431 Action.Enter(CGF);
7432 OMPPrivateScope PrivateScope(CGF);
7433 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7434 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7435 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7436 (void)PrivateScope.Privatize();
7437 CGF.EmitStmt(S: S.getCapturedStmt(RegionKind: OMPD_teams)->getCapturedStmt());
7438 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7439 };
7440 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
7441 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7442 CondGen: [](CodeGenFunction &) { return nullptr; });
7443}
7444
7445static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
7446 const OMPTargetTeamsDirective &S) {
7447 auto *CS = S.getCapturedStmt(RegionKind: OMPD_teams);
7448 Action.Enter(CGF);
7449 // Emit teams region as a standalone region.
7450 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
7451 Action.Enter(CGF);
7452 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7453 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
7454 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
7455 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7456 (void)PrivateScope.Privatize();
7457 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
7458 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
7459 CGF.EmitStmt(S: CS->getCapturedStmt());
7460 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7461 };
7462 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_teams, CodeGen);
7463 emitPostUpdateForReductionClause(CGF, D: S,
7464 CondGen: [](CodeGenFunction &) { return nullptr; });
7465}
7466
7467void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
7468 CodeGenModule &CGM, StringRef ParentName,
7469 const OMPTargetTeamsDirective &S) {
7470 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7471 emitTargetTeamsRegion(CGF, Action, S);
7472 };
7473 llvm::Function *Fn;
7474 llvm::Constant *Addr;
7475 // Emit target region as a standalone region.
7476 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7477 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7478 assert(Fn && Addr && "Target device function emission failed.");
7479}
7480
7481void CodeGenFunction::EmitOMPTargetTeamsDirective(
7482 const OMPTargetTeamsDirective &S) {
7483 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7484 emitTargetTeamsRegion(CGF, Action, S);
7485 };
7486 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7487}
7488
7489static void
7490emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
7491 const OMPTargetTeamsDistributeDirective &S) {
7492 Action.Enter(CGF);
7493 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7494 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7495 };
7496
7497 // Emit teams region as a standalone region.
7498 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7499 PrePostActionTy &Action) {
7500 Action.Enter(CGF);
7501 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7502 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7503 (void)PrivateScope.Privatize();
7504 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7505 CodeGen: CodeGenDistribute);
7506 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7507 };
7508 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute, CodeGen);
7509 emitPostUpdateForReductionClause(CGF, D: S,
7510 CondGen: [](CodeGenFunction &) { return nullptr; });
7511}
7512
7513void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
7514 CodeGenModule &CGM, StringRef ParentName,
7515 const OMPTargetTeamsDistributeDirective &S) {
7516 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7517 emitTargetTeamsDistributeRegion(CGF, Action, S);
7518 };
7519 llvm::Function *Fn;
7520 llvm::Constant *Addr;
7521 // Emit target region as a standalone region.
7522 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7523 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7524 assert(Fn && Addr && "Target device function emission failed.");
7525}
7526
7527void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
7528 const OMPTargetTeamsDistributeDirective &S) {
7529 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7530 emitTargetTeamsDistributeRegion(CGF, Action, S);
7531 };
7532 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7533}
7534
7535static void emitTargetTeamsDistributeSimdRegion(
7536 CodeGenFunction &CGF, PrePostActionTy &Action,
7537 const OMPTargetTeamsDistributeSimdDirective &S) {
7538 Action.Enter(CGF);
7539 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7540 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7541 };
7542
7543 // Emit teams region as a standalone region.
7544 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7545 PrePostActionTy &Action) {
7546 Action.Enter(CGF);
7547 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7548 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7549 (void)PrivateScope.Privatize();
7550 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7551 CodeGen: CodeGenDistribute);
7552 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7553 };
7554 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_simd, CodeGen);
7555 emitPostUpdateForReductionClause(CGF, D: S,
7556 CondGen: [](CodeGenFunction &) { return nullptr; });
7557}
7558
7559void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
7560 CodeGenModule &CGM, StringRef ParentName,
7561 const OMPTargetTeamsDistributeSimdDirective &S) {
7562 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7563 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7564 };
7565 llvm::Function *Fn;
7566 llvm::Constant *Addr;
7567 // Emit target region as a standalone region.
7568 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7569 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7570 assert(Fn && Addr && "Target device function emission failed.");
7571}
7572
7573void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
7574 const OMPTargetTeamsDistributeSimdDirective &S) {
7575 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7576 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
7577 };
7578 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7579}
7580
7581void CodeGenFunction::EmitOMPTeamsDistributeDirective(
7582 const OMPTeamsDistributeDirective &S) {
7583
7584 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7585 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7586 };
7587
7588 // Emit teams region as a standalone region.
7589 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7590 PrePostActionTy &Action) {
7591 Action.Enter(CGF);
7592 OMPPrivateScope PrivateScope(CGF);
7593 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7594 (void)PrivateScope.Privatize();
7595 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7596 CodeGen: CodeGenDistribute);
7597 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7598 };
7599 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
7600 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7601 CondGen: [](CodeGenFunction &) { return nullptr; });
7602}
7603
7604void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
7605 const OMPTeamsDistributeSimdDirective &S) {
7606 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7607 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
7608 };
7609
7610 // Emit teams region as a standalone region.
7611 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7612 PrePostActionTy &Action) {
7613 Action.Enter(CGF);
7614 OMPPrivateScope PrivateScope(CGF);
7615 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7616 (void)PrivateScope.Privatize();
7617 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_simd,
7618 CodeGen: CodeGenDistribute);
7619 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7620 };
7621 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_simd, CodeGen);
7622 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7623 CondGen: [](CodeGenFunction &) { return nullptr; });
7624}
7625
7626void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
7627 const OMPTeamsDistributeParallelForDirective &S) {
7628 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7629 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7630 IncExpr: S.getDistInc());
7631 };
7632
7633 // Emit teams region as a standalone region.
7634 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7635 PrePostActionTy &Action) {
7636 Action.Enter(CGF);
7637 OMPPrivateScope PrivateScope(CGF);
7638 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7639 (void)PrivateScope.Privatize();
7640 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
7641 CodeGen: CodeGenDistribute);
7642 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7643 };
7644 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_parallel_for, CodeGen);
7645 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7646 CondGen: [](CodeGenFunction &) { return nullptr; });
7647}
7648
7649void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
7650 const OMPTeamsDistributeParallelForSimdDirective &S) {
7651 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7652 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7653 IncExpr: S.getDistInc());
7654 };
7655
7656 // Emit teams region as a standalone region.
7657 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7658 PrePostActionTy &Action) {
7659 Action.Enter(CGF);
7660 OMPPrivateScope PrivateScope(CGF);
7661 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7662 (void)PrivateScope.Privatize();
7663 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7664 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
7665 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7666 };
7667 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute_parallel_for_simd,
7668 CodeGen);
7669 emitPostUpdateForReductionClause(CGF&: *this, D: S,
7670 CondGen: [](CodeGenFunction &) { return nullptr; });
7671}
7672
7673void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) {
7674 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7675 llvm::Value *Device = nullptr;
7676 llvm::Value *NumDependences = nullptr;
7677 llvm::Value *DependenceList = nullptr;
7678
7679 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
7680 Device = EmitScalarExpr(E: C->getDevice());
7681
7682 // Build list and emit dependences
7683 OMPTaskDataTy Data;
7684 buildDependences(S, Data);
7685 if (!Data.Dependences.empty()) {
7686 Address DependenciesArray = Address::invalid();
7687 std::tie(args&: NumDependences, args&: DependenciesArray) =
7688 CGM.getOpenMPRuntime().emitDependClause(CGF&: *this, Dependencies: Data.Dependences,
7689 Loc: S.getBeginLoc());
7690 DependenceList = DependenciesArray.emitRawPointer(CGF&: *this);
7691 }
7692 Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
7693
7694 assert(!(Data.HasNowaitClause && !(S.getSingleClause<OMPInitClause>() ||
7695 S.getSingleClause<OMPDestroyClause>() ||
7696 S.getSingleClause<OMPUseClause>())) &&
7697 "OMPNowaitClause clause is used separately in OMPInteropDirective.");
7698
7699 auto ItOMPInitClause = S.getClausesOfKind<OMPInitClause>();
7700 if (!ItOMPInitClause.empty()) {
7701 // Look at the multiple init clauses
7702 for (const OMPInitClause *C : ItOMPInitClause) {
7703 llvm::Value *InteropvarPtr =
7704 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
7705 llvm::omp::OMPInteropType InteropType =
7706 llvm::omp::OMPInteropType::Unknown;
7707 if (C->getIsTarget()) {
7708 InteropType = llvm::omp::OMPInteropType::Target;
7709 } else {
7710 assert(C->getIsTargetSync() &&
7711 "Expected interop-type target/targetsync");
7712 InteropType = llvm::omp::OMPInteropType::TargetSync;
7713 }
7714 OMPBuilder.createOMPInteropInit(Loc: Builder, InteropVar: InteropvarPtr, InteropType,
7715 Device, NumDependences, DependenceAddress: DependenceList,
7716 HaveNowaitClause: Data.HasNowaitClause);
7717 }
7718 }
7719 auto ItOMPDestroyClause = S.getClausesOfKind<OMPDestroyClause>();
7720 if (!ItOMPDestroyClause.empty()) {
7721 // Look at the multiple destroy clauses
7722 for (const OMPDestroyClause *C : ItOMPDestroyClause) {
7723 llvm::Value *InteropvarPtr =
7724 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
7725 OMPBuilder.createOMPInteropDestroy(Loc: Builder, InteropVar: InteropvarPtr, Device,
7726 NumDependences, DependenceAddress: DependenceList,
7727 HaveNowaitClause: Data.HasNowaitClause);
7728 }
7729 }
7730 auto ItOMPUseClause = S.getClausesOfKind<OMPUseClause>();
7731 if (!ItOMPUseClause.empty()) {
7732 // Look at the multiple use clauses
7733 for (const OMPUseClause *C : ItOMPUseClause) {
7734 llvm::Value *InteropvarPtr =
7735 EmitLValue(E: C->getInteropVar()).getPointer(CGF&: *this);
7736 OMPBuilder.createOMPInteropUse(Loc: Builder, InteropVar: InteropvarPtr, Device,
7737 NumDependences, DependenceAddress: DependenceList,
7738 HaveNowaitClause: Data.HasNowaitClause);
7739 }
7740 }
7741}
7742
7743static void emitTargetTeamsDistributeParallelForRegion(
7744 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
7745 PrePostActionTy &Action) {
7746 Action.Enter(CGF);
7747 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7748 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7749 IncExpr: S.getDistInc());
7750 };
7751
7752 // Emit teams region as a standalone region.
7753 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7754 PrePostActionTy &Action) {
7755 Action.Enter(CGF);
7756 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7757 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7758 (void)PrivateScope.Privatize();
7759 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7760 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
7761 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7762 };
7763
7764 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for,
7765 CodeGen: CodeGenTeams);
7766 emitPostUpdateForReductionClause(CGF, D: S,
7767 CondGen: [](CodeGenFunction &) { return nullptr; });
7768}
7769
7770void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
7771 CodeGenModule &CGM, StringRef ParentName,
7772 const OMPTargetTeamsDistributeParallelForDirective &S) {
7773 // Emit SPMD target teams distribute parallel for region as a standalone
7774 // region.
7775 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7776 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
7777 };
7778 llvm::Function *Fn;
7779 llvm::Constant *Addr;
7780 // Emit target region as a standalone region.
7781 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7782 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7783 assert(Fn && Addr && "Target device function emission failed.");
7784}
7785
7786void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
7787 const OMPTargetTeamsDistributeParallelForDirective &S) {
7788 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7789 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
7790 };
7791 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7792}
7793
7794static void emitTargetTeamsDistributeParallelForSimdRegion(
7795 CodeGenFunction &CGF,
7796 const OMPTargetTeamsDistributeParallelForSimdDirective &S,
7797 PrePostActionTy &Action) {
7798 Action.Enter(CGF);
7799 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
7800 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
7801 IncExpr: S.getDistInc());
7802 };
7803
7804 // Emit teams region as a standalone region.
7805 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
7806 PrePostActionTy &Action) {
7807 Action.Enter(CGF);
7808 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
7809 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
7810 (void)PrivateScope.Privatize();
7811 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
7812 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
7813 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
7814 };
7815
7816 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for_simd,
7817 CodeGen: CodeGenTeams);
7818 emitPostUpdateForReductionClause(CGF, D: S,
7819 CondGen: [](CodeGenFunction &) { return nullptr; });
7820}
7821
7822void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
7823 CodeGenModule &CGM, StringRef ParentName,
7824 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
7825 // Emit SPMD target teams distribute parallel for simd region as a standalone
7826 // region.
7827 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7828 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
7829 };
7830 llvm::Function *Fn;
7831 llvm::Constant *Addr;
7832 // Emit target region as a standalone region.
7833 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
7834 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
7835 assert(Fn && Addr && "Target device function emission failed.");
7836}
7837
7838void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
7839 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
7840 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
7841 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
7842 };
7843 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
7844}
7845
7846void CodeGenFunction::EmitOMPCancellationPointDirective(
7847 const OMPCancellationPointDirective &S) {
7848 CGM.getOpenMPRuntime().emitCancellationPointCall(CGF&: *this, Loc: S.getBeginLoc(),
7849 CancelRegion: S.getCancelRegion());
7850}
7851
7852void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
7853 const Expr *IfCond = nullptr;
7854 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
7855 if (C->getNameModifier() == OMPD_unknown ||
7856 C->getNameModifier() == OMPD_cancel) {
7857 IfCond = C->getCondition();
7858 break;
7859 }
7860 }
7861 if (CGM.getLangOpts().OpenMPIRBuilder) {
7862 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
7863 // TODO: This check is necessary as we only generate `omp parallel` through
7864 // the OpenMPIRBuilder for now.
7865 if (S.getCancelRegion() == OMPD_parallel ||
7866 S.getCancelRegion() == OMPD_sections ||
7867 S.getCancelRegion() == OMPD_section) {
7868 llvm::Value *IfCondition = nullptr;
7869 if (IfCond)
7870 IfCondition = EmitScalarExpr(E: IfCond,
7871 /*IgnoreResultAssign=*/true);
7872 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(
7873 ValOrErr: OMPBuilder.createCancel(Loc: Builder, IfCondition, CanceledDirective: S.getCancelRegion()));
7874 return Builder.restoreIP(IP: AfterIP);
7875 }
7876 }
7877
7878 CGM.getOpenMPRuntime().emitCancelCall(CGF&: *this, Loc: S.getBeginLoc(), IfCond,
7879 CancelRegion: S.getCancelRegion());
7880}
7881
7882CodeGenFunction::JumpDest
7883CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
7884 if (Kind == OMPD_parallel || Kind == OMPD_task ||
7885 Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
7886 Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
7887 return ReturnBlock;
7888 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
7889 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
7890 Kind == OMPD_distribute_parallel_for ||
7891 Kind == OMPD_target_parallel_for ||
7892 Kind == OMPD_teams_distribute_parallel_for ||
7893 Kind == OMPD_target_teams_distribute_parallel_for);
7894 return OMPCancelStack.getExitBlock();
7895}
7896
7897void CodeGenFunction::EmitOMPUseDevicePtrClause(
7898 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
7899 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7900 CaptureDeviceAddrMap) {
7901 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7902 for (const Expr *OrigVarIt : C.varlist()) {
7903 const auto *OrigVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: OrigVarIt)->getDecl());
7904 if (!Processed.insert(V: OrigVD).second)
7905 continue;
7906
7907 // In order to identify the right initializer we need to match the
7908 // declaration used by the mapping logic. In some cases we may get
7909 // OMPCapturedExprDecl that refers to the original declaration.
7910 const ValueDecl *MatchingVD = OrigVD;
7911 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: MatchingVD)) {
7912 // OMPCapturedExprDecl are used to privative fields of the current
7913 // structure.
7914 const auto *ME = cast<MemberExpr>(Val: OED->getInit());
7915 assert(isa<CXXThisExpr>(ME->getBase()->IgnoreImpCasts()) &&
7916 "Base should be the current struct!");
7917 MatchingVD = ME->getMemberDecl();
7918 }
7919
7920 // If we don't have information about the current list item, move on to
7921 // the next one.
7922 auto InitAddrIt = CaptureDeviceAddrMap.find(Val: MatchingVD);
7923 if (InitAddrIt == CaptureDeviceAddrMap.end())
7924 continue;
7925
7926 llvm::Type *Ty = ConvertTypeForMem(T: OrigVD->getType().getNonReferenceType());
7927
7928 // Return the address of the private variable.
7929 bool IsRegistered = PrivateScope.addPrivate(
7930 LocalVD: OrigVD,
7931 Addr: Address(InitAddrIt->second, Ty,
7932 getContext().getTypeAlignInChars(T: getContext().VoidPtrTy)));
7933 assert(IsRegistered && "firstprivate var already registered as private");
7934 // Silence the warning about unused variable.
7935 (void)IsRegistered;
7936 }
7937}
7938
7939static const VarDecl *getBaseDecl(const Expr *Ref) {
7940 const Expr *Base = Ref->IgnoreParenImpCasts();
7941 while (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Base))
7942 Base = OASE->getBase()->IgnoreParenImpCasts();
7943 while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
7944 Base = ASE->getBase()->IgnoreParenImpCasts();
7945 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Base)->getDecl());
7946}
7947
7948void CodeGenFunction::EmitOMPUseDeviceAddrClause(
7949 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
7950 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
7951 CaptureDeviceAddrMap) {
7952 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
7953 for (const Expr *Ref : C.varlist()) {
7954 const VarDecl *OrigVD = getBaseDecl(Ref);
7955 if (!Processed.insert(V: OrigVD).second)
7956 continue;
7957 // In order to identify the right initializer we need to match the
7958 // declaration used by the mapping logic. In some cases we may get
7959 // OMPCapturedExprDecl that refers to the original declaration.
7960 const ValueDecl *MatchingVD = OrigVD;
7961 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: MatchingVD)) {
7962 // OMPCapturedExprDecl are used to privative fields of the current
7963 // structure.
7964 const auto *ME = cast<MemberExpr>(Val: OED->getInit());
7965 assert(isa<CXXThisExpr>(ME->getBase()) &&
7966 "Base should be the current struct!");
7967 MatchingVD = ME->getMemberDecl();
7968 }
7969
7970 // If we don't have information about the current list item, move on to
7971 // the next one.
7972 auto InitAddrIt = CaptureDeviceAddrMap.find(Val: MatchingVD);
7973 if (InitAddrIt == CaptureDeviceAddrMap.end())
7974 continue;
7975
7976 llvm::Type *Ty = ConvertTypeForMem(T: OrigVD->getType().getNonReferenceType());
7977
7978 Address PrivAddr =
7979 Address(InitAddrIt->second, Ty,
7980 getContext().getTypeAlignInChars(T: getContext().VoidPtrTy));
7981 // For declrefs and variable length array need to load the pointer for
7982 // correct mapping, since the pointer to the data was passed to the runtime.
7983 if (isa<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts()) ||
7984 MatchingVD->getType()->isArrayType()) {
7985 QualType PtrTy = getContext().getPointerType(
7986 T: OrigVD->getType().getNonReferenceType());
7987 PrivAddr =
7988 EmitLoadOfPointer(Ptr: PrivAddr.withElementType(ElemTy: ConvertTypeForMem(T: PtrTy)),
7989 PtrTy: PtrTy->castAs<PointerType>());
7990 }
7991
7992 (void)PrivateScope.addPrivate(LocalVD: OrigVD, Addr: PrivAddr);
7993 }
7994}
7995
7996// Generate the instructions for '#pragma omp target data' directive.
7997void CodeGenFunction::EmitOMPTargetDataDirective(
7998 const OMPTargetDataDirective &S) {
7999 // Emit vtable only from host for target data directive.
8000 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
8001 CGM.getOpenMPRuntime().registerVTable(D: S);
8002
8003 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,
8004 /*SeparateBeginEndCalls=*/true);
8005
8006 // Create a pre/post action to signal the privatization of the device pointer.
8007 // This action can be replaced by the OpenMP runtime code generation to
8008 // deactivate privatization.
8009 bool PrivatizeDevicePointers = false;
8010 class DevicePointerPrivActionTy : public PrePostActionTy {
8011 bool &PrivatizeDevicePointers;
8012
8013 public:
8014 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
8015 : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
8016 void Enter(CodeGenFunction &CGF) override {
8017 PrivatizeDevicePointers = true;
8018 }
8019 };
8020 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
8021
8022 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8023 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8024 CGF.EmitStmt(S: S.getInnermostCapturedStmt()->getCapturedStmt());
8025 };
8026
8027 // Codegen that selects whether to generate the privatization code or not.
8028 auto &&PrivCodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
8029 RegionCodeGenTy RCG(InnermostCodeGen);
8030 PrivatizeDevicePointers = false;
8031
8032 // Call the pre-action to change the status of PrivatizeDevicePointers if
8033 // needed.
8034 Action.Enter(CGF);
8035
8036 if (PrivatizeDevicePointers) {
8037 OMPPrivateScope PrivateScope(CGF);
8038 // Emit all instances of the use_device_ptr clause.
8039 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8040 CGF.EmitOMPUseDevicePtrClause(C: *C, PrivateScope,
8041 CaptureDeviceAddrMap: Info.CaptureDeviceAddrMap);
8042 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8043 CGF.EmitOMPUseDeviceAddrClause(C: *C, PrivateScope,
8044 CaptureDeviceAddrMap: Info.CaptureDeviceAddrMap);
8045 (void)PrivateScope.Privatize();
8046 RCG(CGF);
8047 } else {
8048 // If we don't have target devices, don't bother emitting the data
8049 // mapping code.
8050 std::optional<OpenMPDirectiveKind> CaptureRegion;
8051 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8052 // Emit helper decls of the use_device_ptr/use_device_addr clauses.
8053 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
8054 for (const Expr *E : C->varlist()) {
8055 const Decl *D = cast<DeclRefExpr>(Val: E)->getDecl();
8056 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
8057 CGF.EmitVarDecl(D: *OED);
8058 }
8059 for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
8060 for (const Expr *E : C->varlist()) {
8061 const Decl *D = getBaseDecl(Ref: E);
8062 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(Val: D))
8063 CGF.EmitVarDecl(D: *OED);
8064 }
8065 } else {
8066 CaptureRegion = OMPD_unknown;
8067 }
8068
8069 OMPLexicalScope Scope(CGF, S, CaptureRegion);
8070 RCG(CGF);
8071 }
8072 };
8073
8074 // Forward the provided action to the privatization codegen.
8075 RegionCodeGenTy PrivRCG(PrivCodeGen);
8076 PrivRCG.setAction(Action);
8077
8078 // Notwithstanding the body of the region is emitted as inlined directive,
8079 // we don't use an inline scope as changes in the references inside the
8080 // region are expected to be visible outside, so we do not privative them.
8081 OMPLexicalScope Scope(CGF, S);
8082 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_target_data,
8083 CodeGen: PrivRCG);
8084 };
8085
8086 RegionCodeGenTy RCG(CodeGen);
8087
8088 // If we don't have target devices, don't bother emitting the data mapping
8089 // code.
8090 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
8091 RCG(*this);
8092 return;
8093 }
8094
8095 // Check if we have any if clause associated with the directive.
8096 const Expr *IfCond = nullptr;
8097 if (const auto *C = S.getSingleClause<OMPIfClause>())
8098 IfCond = C->getCondition();
8099
8100 // Check if we have any device clause associated with the directive.
8101 const Expr *Device = nullptr;
8102 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8103 Device = C->getDevice();
8104
8105 // Set the action to signal privatization of device pointers.
8106 RCG.setAction(PrivAction);
8107
8108 // Emit region code.
8109 CGM.getOpenMPRuntime().emitTargetDataCalls(CGF&: *this, D: S, IfCond, Device, CodeGen: RCG,
8110 Info);
8111}
8112
8113void CodeGenFunction::EmitOMPTargetEnterDataDirective(
8114 const OMPTargetEnterDataDirective &S) {
8115 // If we don't have target devices, don't bother emitting the data mapping
8116 // code.
8117 if (CGM.getLangOpts().OMPTargetTriples.empty())
8118 return;
8119
8120 // Check if we have any if clause associated with the directive.
8121 const Expr *IfCond = nullptr;
8122 if (const auto *C = S.getSingleClause<OMPIfClause>())
8123 IfCond = C->getCondition();
8124
8125 // Check if we have any device clause associated with the directive.
8126 const Expr *Device = nullptr;
8127 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8128 Device = C->getDevice();
8129
8130 OMPLexicalScope Scope(*this, S, OMPD_task);
8131 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8132}
8133
8134void CodeGenFunction::EmitOMPTargetExitDataDirective(
8135 const OMPTargetExitDataDirective &S) {
8136 // If we don't have target devices, don't bother emitting the data mapping
8137 // code.
8138 if (CGM.getLangOpts().OMPTargetTriples.empty())
8139 return;
8140
8141 // Check if we have any if clause associated with the directive.
8142 const Expr *IfCond = nullptr;
8143 if (const auto *C = S.getSingleClause<OMPIfClause>())
8144 IfCond = C->getCondition();
8145
8146 // Check if we have any device clause associated with the directive.
8147 const Expr *Device = nullptr;
8148 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8149 Device = C->getDevice();
8150
8151 OMPLexicalScope Scope(*this, S, OMPD_task);
8152 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8153}
8154
8155static void emitTargetParallelRegion(CodeGenFunction &CGF,
8156 const OMPTargetParallelDirective &S,
8157 PrePostActionTy &Action) {
8158 // Get the captured statement associated with the 'parallel' region.
8159 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_parallel);
8160 Action.Enter(CGF);
8161 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
8162 Action.Enter(CGF);
8163 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8164 (void)CGF.EmitOMPFirstprivateClause(D: S, PrivateScope);
8165 CGF.EmitOMPPrivateClause(D: S, PrivateScope);
8166 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8167 (void)PrivateScope.Privatize();
8168 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()))
8169 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, D: S);
8170 // TODO: Add support for clauses.
8171 CGF.EmitStmt(S: CS->getCapturedStmt());
8172 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_parallel);
8173 };
8174 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_parallel, CodeGen,
8175 CodeGenBoundParameters: emitEmptyBoundParameters);
8176 emitPostUpdateForReductionClause(CGF, D: S,
8177 CondGen: [](CodeGenFunction &) { return nullptr; });
8178}
8179
8180void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
8181 CodeGenModule &CGM, StringRef ParentName,
8182 const OMPTargetParallelDirective &S) {
8183 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8184 emitTargetParallelRegion(CGF, S, Action);
8185 };
8186 llvm::Function *Fn;
8187 llvm::Constant *Addr;
8188 // Emit target region as a standalone region.
8189 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8190 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8191 assert(Fn && Addr && "Target device function emission failed.");
8192}
8193
8194void CodeGenFunction::EmitOMPTargetParallelDirective(
8195 const OMPTargetParallelDirective &S) {
8196 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8197 emitTargetParallelRegion(CGF, S, Action);
8198 };
8199 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8200}
8201
8202static void emitTargetParallelForRegion(CodeGenFunction &CGF,
8203 const OMPTargetParallelForDirective &S,
8204 PrePostActionTy &Action) {
8205 Action.Enter(CGF);
8206 // Emit directive as a combined directive that consists of two implicit
8207 // directives: 'parallel' with 'for' directive.
8208 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8209 Action.Enter(CGF);
8210 CodeGenFunction::OMPCancelStackRAII CancelRegion(
8211 CGF, OMPD_target_parallel_for, S.hasCancel());
8212 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
8213 CGDispatchBounds: emitDispatchForLoopBounds);
8214 };
8215 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_for, CodeGen,
8216 CodeGenBoundParameters: emitEmptyBoundParameters);
8217}
8218
8219void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
8220 CodeGenModule &CGM, StringRef ParentName,
8221 const OMPTargetParallelForDirective &S) {
8222 // Emit SPMD target parallel for region as a standalone region.
8223 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8224 emitTargetParallelForRegion(CGF, S, Action);
8225 };
8226 llvm::Function *Fn;
8227 llvm::Constant *Addr;
8228 // Emit target region as a standalone region.
8229 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8230 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8231 assert(Fn && Addr && "Target device function emission failed.");
8232}
8233
8234void CodeGenFunction::EmitOMPTargetParallelForDirective(
8235 const OMPTargetParallelForDirective &S) {
8236 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8237 emitTargetParallelForRegion(CGF, S, Action);
8238 };
8239 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8240}
8241
8242static void
8243emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
8244 const OMPTargetParallelForSimdDirective &S,
8245 PrePostActionTy &Action) {
8246 Action.Enter(CGF);
8247 // Emit directive as a combined directive that consists of two implicit
8248 // directives: 'parallel' with 'for' directive.
8249 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8250 Action.Enter(CGF);
8251 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
8252 CGDispatchBounds: emitDispatchForLoopBounds);
8253 };
8254 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_simd, CodeGen,
8255 CodeGenBoundParameters: emitEmptyBoundParameters);
8256}
8257
8258void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
8259 CodeGenModule &CGM, StringRef ParentName,
8260 const OMPTargetParallelForSimdDirective &S) {
8261 // Emit SPMD target parallel for region as a standalone region.
8262 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8263 emitTargetParallelForSimdRegion(CGF, S, Action);
8264 };
8265 llvm::Function *Fn;
8266 llvm::Constant *Addr;
8267 // Emit target region as a standalone region.
8268 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8269 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8270 assert(Fn && Addr && "Target device function emission failed.");
8271}
8272
8273void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
8274 const OMPTargetParallelForSimdDirective &S) {
8275 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8276 emitTargetParallelForSimdRegion(CGF, S, Action);
8277 };
8278 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8279}
8280
8281/// Emit a helper variable and return corresponding lvalue.
8282static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
8283 const ImplicitParamDecl *PVD,
8284 CodeGenFunction::OMPPrivateScope &Privates) {
8285 const auto *VDecl = cast<VarDecl>(Val: Helper->getDecl());
8286 Privates.addPrivate(LocalVD: VDecl, Addr: CGF.GetAddrOfLocalVar(VD: PVD));
8287}
8288
8289void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
8290 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
8291 // Emit outlined function for task construct.
8292 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: OMPD_taskloop);
8293 Address CapturedStruct = Address::invalid();
8294 {
8295 OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8296 CapturedStruct = GenerateCapturedStmtArgument(S: *CS);
8297 }
8298 CanQualType SharedsTy =
8299 getContext().getCanonicalTagType(TD: CS->getCapturedRecordDecl());
8300 const Expr *IfCond = nullptr;
8301 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
8302 if (C->getNameModifier() == OMPD_unknown ||
8303 C->getNameModifier() == OMPD_taskloop) {
8304 IfCond = C->getCondition();
8305 break;
8306 }
8307 }
8308
8309 OMPTaskDataTy Data;
8310 // Check if taskloop must be emitted without taskgroup.
8311 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
8312 // TODO: Check if we should emit tied or untied task.
8313 Data.Tied = true;
8314 // Set scheduling for taskloop
8315 if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
8316 // grainsize clause
8317 Data.Schedule.setInt(/*IntVal=*/false);
8318 Data.Schedule.setPointer(EmitScalarExpr(E: Clause->getGrainsize()));
8319 Data.HasModifier =
8320 (Clause->getModifier() == OMPC_GRAINSIZE_strict) ? true : false;
8321 } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
8322 // num_tasks clause
8323 Data.Schedule.setInt(/*IntVal=*/true);
8324 Data.Schedule.setPointer(EmitScalarExpr(E: Clause->getNumTasks()));
8325 Data.HasModifier =
8326 (Clause->getModifier() == OMPC_NUMTASKS_strict) ? true : false;
8327 }
8328
8329 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
8330 // if (PreCond) {
8331 // for (IV in 0..LastIteration) BODY;
8332 // <Final counter/linear vars updates>;
8333 // }
8334 //
8335
8336 // Emit: if (PreCond) - begin.
8337 // If the condition constant folds and can be elided, avoid emitting the
8338 // whole loop.
8339 bool CondConstant;
8340 llvm::BasicBlock *ContBlock = nullptr;
8341 OMPLoopScope PreInitScope(CGF, S);
8342 if (CGF.ConstantFoldsToSimpleInteger(Cond: S.getPreCond(), Result&: CondConstant)) {
8343 if (!CondConstant)
8344 return;
8345 } else {
8346 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "taskloop.if.then");
8347 ContBlock = CGF.createBasicBlock(name: "taskloop.if.end");
8348 emitPreCond(CGF, S, Cond: S.getPreCond(), TrueBlock: ThenBlock, FalseBlock: ContBlock,
8349 TrueCount: CGF.getProfileCount(S: &S));
8350 CGF.EmitBlock(BB: ThenBlock);
8351 CGF.incrementProfileCounter(S: &S);
8352 }
8353
8354 (void)CGF.EmitOMPLinearClauseInit(D: S);
8355
8356 OMPPrivateScope LoopScope(CGF);
8357 // Emit helper vars inits.
8358 enum { LowerBound = 5, UpperBound, Stride, LastIter };
8359 auto *I = CS->getCapturedDecl()->param_begin();
8360 auto *LBP = std::next(x: I, n: LowerBound);
8361 auto *UBP = std::next(x: I, n: UpperBound);
8362 auto *STP = std::next(x: I, n: Stride);
8363 auto *LIP = std::next(x: I, n: LastIter);
8364 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getLowerBoundVariable()), PVD: *LBP,
8365 Privates&: LoopScope);
8366 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getUpperBoundVariable()), PVD: *UBP,
8367 Privates&: LoopScope);
8368 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getStrideVariable()), PVD: *STP, Privates&: LoopScope);
8369 mapParam(CGF, Helper: cast<DeclRefExpr>(Val: S.getIsLastIterVariable()), PVD: *LIP,
8370 Privates&: LoopScope);
8371 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8372 CGF.EmitOMPLinearClause(D: S, PrivateScope&: LoopScope);
8373 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(D: S, PrivateScope&: LoopScope);
8374 (void)LoopScope.Privatize();
8375 // Emit the loop iteration variable.
8376 const Expr *IVExpr = S.getIterationVariable();
8377 const auto *IVDecl = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IVExpr)->getDecl());
8378 CGF.EmitVarDecl(D: *IVDecl);
8379 CGF.EmitIgnoredExpr(E: S.getInit());
8380
8381 // Emit the iterations count variable.
8382 // If it is not a variable, Sema decided to calculate iterations count on
8383 // each iteration (e.g., it is foldable into a constant).
8384 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(Val: S.getLastIteration())) {
8385 CGF.EmitVarDecl(D: *cast<VarDecl>(Val: LIExpr->getDecl()));
8386 // Emit calculation of the iterations count.
8387 CGF.EmitIgnoredExpr(E: S.getCalcLastIteration());
8388 }
8389
8390 {
8391 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
8392 emitCommonSimdLoop(
8393 CGF, S,
8394 SimdInitGen: [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8395 if (isOpenMPSimdDirective(DKind: S.getDirectiveKind()))
8396 CGF.EmitOMPSimdInit(D: S);
8397 },
8398 BodyCodeGen: [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
8399 CGF.EmitOMPInnerLoop(
8400 S, RequiresCleanup: LoopScope.requiresCleanups(), LoopCond: S.getCond(), IncExpr: S.getInc(),
8401 BodyGen: [&S](CodeGenFunction &CGF) {
8402 emitOMPLoopBodyWithStopPoint(CGF, S,
8403 LoopExit: CodeGenFunction::JumpDest());
8404 },
8405 PostIncGen: [](CodeGenFunction &) {});
8406 });
8407 }
8408 // Emit: if (PreCond) - end.
8409 if (ContBlock) {
8410 CGF.EmitBranch(Block: ContBlock);
8411 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
8412 }
8413 // Emit final copy of the lastprivate variables if IsLastIter != 0.
8414 if (HasLastprivateClause) {
8415 CGF.EmitOMPLastprivateClauseFinal(
8416 D: S, NoFinals: isOpenMPSimdDirective(DKind: S.getDirectiveKind()),
8417 IsLastIterCond: CGF.Builder.CreateIsNotNull(Arg: CGF.EmitLoadOfScalar(
8418 Addr: CGF.GetAddrOfLocalVar(VD: *LIP), /*Volatile=*/false,
8419 Ty: (*LIP)->getType(), Loc: S.getBeginLoc())));
8420 }
8421 LoopScope.restoreMap();
8422 CGF.EmitOMPLinearClauseFinal(D: S, CondGen: [LIP, &S](CodeGenFunction &CGF) {
8423 return CGF.Builder.CreateIsNotNull(
8424 Arg: CGF.EmitLoadOfScalar(Addr: CGF.GetAddrOfLocalVar(VD: *LIP), /*Volatile=*/false,
8425 Ty: (*LIP)->getType(), Loc: S.getBeginLoc()));
8426 });
8427 };
8428 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
8429 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
8430 const OMPTaskDataTy &Data) {
8431 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
8432 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
8433 OMPLoopScope PreInitScope(CGF, S);
8434 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, Loc: S.getBeginLoc(), D: S,
8435 TaskFunction: OutlinedFn, SharedsTy,
8436 Shareds: CapturedStruct, IfCond, Data);
8437 };
8438 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_taskloop,
8439 CodeGen);
8440 };
8441 if (Data.Nogroup) {
8442 EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_taskloop, BodyGen, TaskGen, Data);
8443 } else {
8444 CGM.getOpenMPRuntime().emitTaskgroupRegion(
8445 CGF&: *this,
8446 TaskgroupOpGen: [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
8447 PrePostActionTy &Action) {
8448 Action.Enter(CGF);
8449 CGF.EmitOMPTaskBasedDirective(S, CapturedRegion: OMPD_taskloop, BodyGen, TaskGen,
8450 Data);
8451 },
8452 Loc: S.getBeginLoc());
8453 }
8454}
8455
8456void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
8457 auto LPCRegion =
8458 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8459 EmitOMPTaskLoopBasedDirective(S);
8460}
8461
8462void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
8463 const OMPTaskLoopSimdDirective &S) {
8464 auto LPCRegion =
8465 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8466 OMPLexicalScope Scope(*this, S);
8467 EmitOMPTaskLoopBasedDirective(S);
8468}
8469
8470void CodeGenFunction::EmitOMPMasterTaskLoopDirective(
8471 const OMPMasterTaskLoopDirective &S) {
8472 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8473 Action.Enter(CGF);
8474 EmitOMPTaskLoopBasedDirective(S);
8475 };
8476 auto LPCRegion =
8477 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8478 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8479 CGM.getOpenMPRuntime().emitMasterRegion(CGF&: *this, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
8480}
8481
8482void CodeGenFunction::EmitOMPMaskedTaskLoopDirective(
8483 const OMPMaskedTaskLoopDirective &S) {
8484 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8485 Action.Enter(CGF);
8486 EmitOMPTaskLoopBasedDirective(S);
8487 };
8488 auto LPCRegion =
8489 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8490 OMPLexicalScope Scope(*this, S, std::nullopt, /*EmitPreInitStmt=*/false);
8491 CGM.getOpenMPRuntime().emitMaskedRegion(CGF&: *this, MaskedOpGen: CodeGen, Loc: S.getBeginLoc());
8492}
8493
8494void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(
8495 const OMPMasterTaskLoopSimdDirective &S) {
8496 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8497 Action.Enter(CGF);
8498 EmitOMPTaskLoopBasedDirective(S);
8499 };
8500 auto LPCRegion =
8501 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8502 OMPLexicalScope Scope(*this, S);
8503 CGM.getOpenMPRuntime().emitMasterRegion(CGF&: *this, MasterOpGen: CodeGen, Loc: S.getBeginLoc());
8504}
8505
8506void CodeGenFunction::EmitOMPMaskedTaskLoopSimdDirective(
8507 const OMPMaskedTaskLoopSimdDirective &S) {
8508 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8509 Action.Enter(CGF);
8510 EmitOMPTaskLoopBasedDirective(S);
8511 };
8512 auto LPCRegion =
8513 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8514 OMPLexicalScope Scope(*this, S);
8515 CGM.getOpenMPRuntime().emitMaskedRegion(CGF&: *this, MaskedOpGen: CodeGen, Loc: S.getBeginLoc());
8516}
8517
8518void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(
8519 const OMPParallelMasterTaskLoopDirective &S) {
8520 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8521 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8522 PrePostActionTy &Action) {
8523 Action.Enter(CGF);
8524 CGF.EmitOMPTaskLoopBasedDirective(S);
8525 };
8526 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8527 CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: TaskLoopCodeGen,
8528 Loc: S.getBeginLoc());
8529 };
8530 auto LPCRegion =
8531 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8532 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master_taskloop, CodeGen,
8533 CodeGenBoundParameters: emitEmptyBoundParameters);
8534}
8535
8536void CodeGenFunction::EmitOMPParallelMaskedTaskLoopDirective(
8537 const OMPParallelMaskedTaskLoopDirective &S) {
8538 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8539 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8540 PrePostActionTy &Action) {
8541 Action.Enter(CGF);
8542 CGF.EmitOMPTaskLoopBasedDirective(S);
8543 };
8544 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8545 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: TaskLoopCodeGen,
8546 Loc: S.getBeginLoc());
8547 };
8548 auto LPCRegion =
8549 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8550 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked_taskloop, CodeGen,
8551 CodeGenBoundParameters: emitEmptyBoundParameters);
8552}
8553
8554void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(
8555 const OMPParallelMasterTaskLoopSimdDirective &S) {
8556 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8557 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8558 PrePostActionTy &Action) {
8559 Action.Enter(CGF);
8560 CGF.EmitOMPTaskLoopBasedDirective(S);
8561 };
8562 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8563 CGM.getOpenMPRuntime().emitMasterRegion(CGF, MasterOpGen: TaskLoopCodeGen,
8564 Loc: S.getBeginLoc());
8565 };
8566 auto LPCRegion =
8567 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8568 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_master_taskloop_simd, CodeGen,
8569 CodeGenBoundParameters: emitEmptyBoundParameters);
8570}
8571
8572void CodeGenFunction::EmitOMPParallelMaskedTaskLoopSimdDirective(
8573 const OMPParallelMaskedTaskLoopSimdDirective &S) {
8574 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8575 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
8576 PrePostActionTy &Action) {
8577 Action.Enter(CGF);
8578 CGF.EmitOMPTaskLoopBasedDirective(S);
8579 };
8580 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
8581 CGM.getOpenMPRuntime().emitMaskedRegion(CGF, MaskedOpGen: TaskLoopCodeGen,
8582 Loc: S.getBeginLoc());
8583 };
8584 auto LPCRegion =
8585 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8586 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_masked_taskloop_simd, CodeGen,
8587 CodeGenBoundParameters: emitEmptyBoundParameters);
8588}
8589
8590// Generate the instructions for '#pragma omp target update' directive.
8591void CodeGenFunction::EmitOMPTargetUpdateDirective(
8592 const OMPTargetUpdateDirective &S) {
8593 // If we don't have target devices, don't bother emitting the data mapping
8594 // code.
8595 if (CGM.getLangOpts().OMPTargetTriples.empty())
8596 return;
8597
8598 // Check if we have any if clause associated with the directive.
8599 const Expr *IfCond = nullptr;
8600 if (const auto *C = S.getSingleClause<OMPIfClause>())
8601 IfCond = C->getCondition();
8602
8603 // Check if we have any device clause associated with the directive.
8604 const Expr *Device = nullptr;
8605 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
8606 Device = C->getDevice();
8607
8608 OMPLexicalScope Scope(*this, S, OMPD_task);
8609 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF&: *this, D: S, IfCond, Device);
8610}
8611
8612void CodeGenFunction::EmitOMPGenericLoopDirective(
8613 const OMPGenericLoopDirective &S) {
8614 // Always expect a bind clause on the loop directive. It it wasn't
8615 // in the source, it should have been added in sema.
8616
8617 OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;
8618 if (const auto *C = S.getSingleClause<OMPBindClause>())
8619 BindKind = C->getBindKind();
8620
8621 switch (BindKind) {
8622 case OMPC_BIND_parallel: // for
8623 return emitOMPForDirective(S, CGF&: *this, CGM, /*HasCancel=*/false);
8624 case OMPC_BIND_teams: // distribute
8625 return emitOMPDistributeDirective(S, CGF&: *this, CGM);
8626 case OMPC_BIND_thread: // simd
8627 return emitOMPSimdDirective(S, CGF&: *this, CGM);
8628 case OMPC_BIND_unknown:
8629 break;
8630 }
8631
8632 // Unimplemented, just inline the underlying statement for now.
8633 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8634 // Emit the loop iteration variable.
8635 const Stmt *CS =
8636 cast<CapturedStmt>(Val: S.getAssociatedStmt())->getCapturedStmt();
8637 const auto *ForS = dyn_cast<ForStmt>(Val: CS);
8638 if (ForS && !isa<DeclStmt>(Val: ForS->getInit())) {
8639 OMPPrivateScope LoopScope(CGF);
8640 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
8641 (void)LoopScope.Privatize();
8642 CGF.EmitStmt(S: CS);
8643 LoopScope.restoreMap();
8644 } else {
8645 CGF.EmitStmt(S: CS);
8646 }
8647 };
8648 OMPLexicalScope Scope(*this, S, OMPD_unknown);
8649 CGM.getOpenMPRuntime().emitInlinedDirective(CGF&: *this, InnermostKind: OMPD_loop, CodeGen);
8650}
8651
8652void CodeGenFunction::EmitOMPParallelGenericLoopDirective(
8653 const OMPLoopDirective &S) {
8654 // Emit combined directive as if its constituent constructs are 'parallel'
8655 // and 'for'.
8656 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8657 Action.Enter(CGF);
8658 emitOMPCopyinClause(CGF, S);
8659 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
8660 };
8661 {
8662 auto LPCRegion =
8663 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S);
8664 emitCommonOMPParallelDirective(CGF&: *this, S, InnermostKind: OMPD_for, CodeGen,
8665 CodeGenBoundParameters: emitEmptyBoundParameters);
8666 }
8667 // Check for outer lastprivate conditional update.
8668 checkForLastprivateConditionalUpdate(CGF&: *this, S);
8669}
8670
8671void CodeGenFunction::EmitOMPTeamsGenericLoopDirective(
8672 const OMPTeamsGenericLoopDirective &S) {
8673 // To be consistent with current behavior of 'target teams loop', emit
8674 // 'teams loop' as if its constituent constructs are 'teams' and 'distribute'.
8675 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8676 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
8677 };
8678
8679 // Emit teams region as a standalone region.
8680 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8681 PrePostActionTy &Action) {
8682 Action.Enter(CGF);
8683 OMPPrivateScope PrivateScope(CGF);
8684 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8685 (void)PrivateScope.Privatize();
8686 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, InnermostKind: OMPD_distribute,
8687 CodeGen: CodeGenDistribute);
8688 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8689 };
8690 emitCommonOMPTeamsDirective(CGF&: *this, S, InnermostKind: OMPD_distribute, CodeGen);
8691 emitPostUpdateForReductionClause(CGF&: *this, D: S,
8692 CondGen: [](CodeGenFunction &) { return nullptr; });
8693}
8694
8695#ifndef NDEBUG
8696static void emitTargetTeamsLoopCodegenStatus(CodeGenFunction &CGF,
8697 std::string StatusMsg,
8698 const OMPExecutableDirective &D) {
8699 bool IsDevice = CGF.CGM.getLangOpts().OpenMPIsTargetDevice;
8700 if (IsDevice)
8701 StatusMsg += ": DEVICE";
8702 else
8703 StatusMsg += ": HOST";
8704 SourceLocation L = D.getBeginLoc();
8705 auto &SM = CGF.getContext().getSourceManager();
8706 PresumedLoc PLoc = SM.getPresumedLoc(L);
8707 const char *FileName = PLoc.isValid() ? PLoc.getFilename() : nullptr;
8708 unsigned LineNo =
8709 PLoc.isValid() ? PLoc.getLine() : SM.getExpansionLineNumber(L);
8710 llvm::dbgs() << StatusMsg << ": " << FileName << ": " << LineNo << "\n";
8711}
8712#endif
8713
8714static void emitTargetTeamsGenericLoopRegionAsParallel(
8715 CodeGenFunction &CGF, PrePostActionTy &Action,
8716 const OMPTargetTeamsGenericLoopDirective &S) {
8717 Action.Enter(CGF);
8718 // Emit 'teams loop' as if its constituent constructs are 'distribute,
8719 // 'parallel, and 'for'.
8720 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8721 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitInnerParallelForWhenCombined,
8722 IncExpr: S.getDistInc());
8723 };
8724
8725 // Emit teams region as a standalone region.
8726 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8727 PrePostActionTy &Action) {
8728 Action.Enter(CGF);
8729 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8730 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8731 (void)PrivateScope.Privatize();
8732 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8733 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
8734 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8735 };
8736 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
8737 emitTargetTeamsLoopCodegenStatus(
8738 CGF, TTL_CODEGEN_TYPE " as parallel for", S));
8739 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute_parallel_for,
8740 CodeGen: CodeGenTeams);
8741 emitPostUpdateForReductionClause(CGF, D: S,
8742 CondGen: [](CodeGenFunction &) { return nullptr; });
8743}
8744
8745static void emitTargetTeamsGenericLoopRegionAsDistribute(
8746 CodeGenFunction &CGF, PrePostActionTy &Action,
8747 const OMPTargetTeamsGenericLoopDirective &S) {
8748 Action.Enter(CGF);
8749 // Emit 'teams loop' as if its constituent construct is 'distribute'.
8750 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
8751 CGF.EmitOMPDistributeLoop(S, CodeGenLoop: emitOMPLoopBodyWithStopPoint, IncExpr: S.getInc());
8752 };
8753
8754 // Emit teams region as a standalone region.
8755 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
8756 PrePostActionTy &Action) {
8757 Action.Enter(CGF);
8758 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
8759 CGF.EmitOMPReductionClauseInit(D: S, PrivateScope);
8760 (void)PrivateScope.Privatize();
8761 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
8762 CGF, InnermostKind: OMPD_distribute, CodeGen: CodeGenDistribute, /*HasCancel=*/false);
8763 CGF.EmitOMPReductionClauseFinal(D: S, /*ReductionKind=*/OMPD_teams);
8764 };
8765 DEBUG_WITH_TYPE(TTL_CODEGEN_TYPE,
8766 emitTargetTeamsLoopCodegenStatus(
8767 CGF, TTL_CODEGEN_TYPE " as distribute", S));
8768 emitCommonOMPTeamsDirective(CGF, S, InnermostKind: OMPD_distribute, CodeGen);
8769 emitPostUpdateForReductionClause(CGF, D: S,
8770 CondGen: [](CodeGenFunction &) { return nullptr; });
8771}
8772
8773void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDirective(
8774 const OMPTargetTeamsGenericLoopDirective &S) {
8775 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8776 if (S.canBeParallelFor())
8777 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
8778 else
8779 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
8780 };
8781 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8782}
8783
8784void CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
8785 CodeGenModule &CGM, StringRef ParentName,
8786 const OMPTargetTeamsGenericLoopDirective &S) {
8787 // Emit SPMD target parallel loop region as a standalone region.
8788 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8789 if (S.canBeParallelFor())
8790 emitTargetTeamsGenericLoopRegionAsParallel(CGF, Action, S);
8791 else
8792 emitTargetTeamsGenericLoopRegionAsDistribute(CGF, Action, S);
8793 };
8794 llvm::Function *Fn;
8795 llvm::Constant *Addr;
8796 // Emit target region as a standalone region.
8797 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8798 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8799 assert(Fn && Addr &&
8800 "Target device function emission failed for 'target teams loop'.");
8801}
8802
8803static void emitTargetParallelGenericLoopRegion(
8804 CodeGenFunction &CGF, const OMPTargetParallelGenericLoopDirective &S,
8805 PrePostActionTy &Action) {
8806 Action.Enter(CGF);
8807 // Emit as 'parallel for'.
8808 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8809 Action.Enter(CGF);
8810 CodeGenFunction::OMPCancelStackRAII CancelRegion(
8811 CGF, OMPD_target_parallel_loop, /*hasCancel=*/false);
8812 CGF.EmitOMPWorksharingLoop(S, EUB: S.getEnsureUpperBound(), CodeGenLoopBounds: emitForLoopBounds,
8813 CGDispatchBounds: emitDispatchForLoopBounds);
8814 };
8815 emitCommonOMPParallelDirective(CGF, S, InnermostKind: OMPD_for, CodeGen,
8816 CodeGenBoundParameters: emitEmptyBoundParameters);
8817}
8818
8819void CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
8820 CodeGenModule &CGM, StringRef ParentName,
8821 const OMPTargetParallelGenericLoopDirective &S) {
8822 // Emit target parallel loop region as a standalone region.
8823 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8824 emitTargetParallelGenericLoopRegion(CGF, S, Action);
8825 };
8826 llvm::Function *Fn;
8827 llvm::Constant *Addr;
8828 // Emit target region as a standalone region.
8829 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
8830 D: S, ParentName, OutlinedFn&: Fn, OutlinedFnID&: Addr, /*IsOffloadEntry=*/true, CodeGen);
8831 assert(Fn && Addr && "Target device function emission failed.");
8832}
8833
8834/// Emit combined directive 'target parallel loop' as if its constituent
8835/// constructs are 'target', 'parallel', and 'for'.
8836void CodeGenFunction::EmitOMPTargetParallelGenericLoopDirective(
8837 const OMPTargetParallelGenericLoopDirective &S) {
8838 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
8839 emitTargetParallelGenericLoopRegion(CGF, S, Action);
8840 };
8841 emitCommonOMPTargetDirective(CGF&: *this, S, CodeGen);
8842}
8843
8844void CodeGenFunction::EmitSimpleOMPExecutableDirective(
8845 const OMPExecutableDirective &D) {
8846 if (const auto *SD = dyn_cast<OMPScanDirective>(Val: &D)) {
8847 EmitOMPScanDirective(S: *SD);
8848 return;
8849 }
8850 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
8851 return;
8852 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
8853 OMPPrivateScope GlobalsScope(CGF);
8854 if (isOpenMPTaskingDirective(Kind: D.getDirectiveKind())) {
8855 // Capture global firstprivates to avoid crash.
8856 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
8857 for (const Expr *Ref : C->varlist()) {
8858 const auto *DRE = cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
8859 if (!DRE)
8860 continue;
8861 const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl());
8862 if (!VD || VD->hasLocalStorage())
8863 continue;
8864 if (!CGF.LocalDeclMap.count(Val: VD)) {
8865 LValue GlobLVal = CGF.EmitLValue(E: Ref);
8866 GlobalsScope.addPrivate(LocalVD: VD, Addr: GlobLVal.getAddress());
8867 }
8868 }
8869 }
8870 }
8871 if (isOpenMPSimdDirective(DKind: D.getDirectiveKind())) {
8872 (void)GlobalsScope.Privatize();
8873 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
8874 emitOMPSimdRegion(CGF, S: cast<OMPLoopDirective>(Val: D), Action);
8875 } else {
8876 if (const auto *LD = dyn_cast<OMPLoopDirective>(Val: &D)) {
8877 for (const Expr *E : LD->counters()) {
8878 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
8879 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(Val: VD)) {
8880 LValue GlobLVal = CGF.EmitLValue(E);
8881 GlobalsScope.addPrivate(LocalVD: VD, Addr: GlobLVal.getAddress());
8882 }
8883 if (isa<OMPCapturedExprDecl>(Val: VD)) {
8884 // Emit only those that were not explicitly referenced in clauses.
8885 if (!CGF.LocalDeclMap.count(Val: VD))
8886 CGF.EmitVarDecl(D: *VD);
8887 }
8888 }
8889 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
8890 if (!C->getNumForLoops())
8891 continue;
8892 for (unsigned I = LD->getLoopsNumber(),
8893 E = C->getLoopNumIterations().size();
8894 I < E; ++I) {
8895 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
8896 Val: cast<DeclRefExpr>(Val: C->getLoopCounter(NumLoop: I))->getDecl())) {
8897 // Emit only those that were not explicitly referenced in clauses.
8898 if (!CGF.LocalDeclMap.count(Val: VD))
8899 CGF.EmitVarDecl(D: *VD);
8900 }
8901 }
8902 }
8903 }
8904 (void)GlobalsScope.Privatize();
8905 CGF.EmitStmt(S: D.getInnermostCapturedStmt()->getCapturedStmt());
8906 }
8907 };
8908 if (D.getDirectiveKind() == OMPD_atomic ||
8909 D.getDirectiveKind() == OMPD_critical ||
8910 D.getDirectiveKind() == OMPD_section ||
8911 D.getDirectiveKind() == OMPD_master ||
8912 D.getDirectiveKind() == OMPD_masked ||
8913 D.getDirectiveKind() == OMPD_unroll ||
8914 D.getDirectiveKind() == OMPD_assume) {
8915 EmitStmt(S: D.getAssociatedStmt());
8916 } else {
8917 auto LPCRegion =
8918 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF&: *this, S: D);
8919 OMPSimdLexicalScope Scope(*this, D);
8920 CGM.getOpenMPRuntime().emitInlinedDirective(
8921 CGF&: *this,
8922 InnermostKind: isOpenMPSimdDirective(DKind: D.getDirectiveKind()) ? OMPD_simd
8923 : D.getDirectiveKind(),
8924 CodeGen);
8925 }
8926 // Check for outer lastprivate conditional update.
8927 checkForLastprivateConditionalUpdate(CGF&: *this, S: D);
8928}
8929
8930void CodeGenFunction::EmitOMPAssumeDirective(const OMPAssumeDirective &S) {
8931 for (const auto *C : S.getClausesOfKind<OMPHoldsClause>()) {
8932 const Expr *E = C->getExpr();
8933 assert(E && "holds clause requires an expression");
8934 if (!E->HasSideEffects(Ctx: getContext()))
8935 Builder.CreateAssumption(Cond: EvaluateExprAsBool(E));
8936 }
8937 EmitStmt(S: S.getAssociatedStmt());
8938}
8939