1//===---- CGOpenMPRuntimeGPU.cpp - Interface to OpenMP GPU Runtimes ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a generalized class for OpenMP runtime code generation
10// specialized by GPU targets NVPTX, AMDGCN and SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGOpenMPRuntimeGPU.h"
15#include "CGDebugInfo.h"
16#include "CodeGenFunction.h"
17#include "TargetInfo.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/DeclOpenMP.h"
20#include "clang/AST/OpenMPClause.h"
21#include "clang/AST/StmtOpenMP.h"
22#include "clang/AST/StmtVisitor.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
25#include "llvm/Frontend/OpenMP/OMPGridValues.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/TargetParser/NVPTXTargetParser.h"
29
30using namespace clang;
31using namespace CodeGen;
32using namespace llvm::omp;
33
34namespace {
35/// Pre(post)-action for different OpenMP constructs specialized for NVPTX.
36class NVPTXActionTy final : public PrePostActionTy {
37 llvm::FunctionCallee EnterCallee = nullptr;
38 ArrayRef<llvm::Value *> EnterArgs;
39 llvm::FunctionCallee ExitCallee = nullptr;
40 ArrayRef<llvm::Value *> ExitArgs;
41 bool Conditional = false;
42 llvm::BasicBlock *ContBlock = nullptr;
43
44public:
45 NVPTXActionTy(llvm::FunctionCallee EnterCallee,
46 ArrayRef<llvm::Value *> EnterArgs,
47 llvm::FunctionCallee ExitCallee,
48 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
49 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
50 ExitArgs(ExitArgs), Conditional(Conditional) {}
51 void Enter(CodeGenFunction &CGF) override {
52 llvm::Value *EnterRes = CGF.EmitRuntimeCall(callee: EnterCallee, args: EnterArgs);
53 if (Conditional) {
54 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(Arg: EnterRes);
55 auto *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
56 ContBlock = CGF.createBasicBlock(name: "omp_if.end");
57 // Generate the branch (If-stmt)
58 CGF.Builder.CreateCondBr(Cond: CallBool, True: ThenBlock, False: ContBlock);
59 CGF.EmitBlock(BB: ThenBlock);
60 }
61 }
62 void Done(CodeGenFunction &CGF) {
63 // Emit the rest of blocks/branches
64 CGF.EmitBranch(Block: ContBlock);
65 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
66 }
67 void Exit(CodeGenFunction &CGF) override {
68 CGF.EmitRuntimeCall(callee: ExitCallee, args: ExitArgs);
69 }
70};
71
72/// A class to track the execution mode when codegening directives within
73/// a target region. The appropriate mode (SPMD|NON-SPMD) is set on entry
74/// to the target region and used by containing directives such as 'parallel'
75/// to emit optimized code.
76class ExecutionRuntimeModesRAII {
77private:
78 CGOpenMPRuntimeGPU::ExecutionMode SavedExecMode =
79 CGOpenMPRuntimeGPU::EM_Unknown;
80 CGOpenMPRuntimeGPU::ExecutionMode &ExecMode;
81
82public:
83 ExecutionRuntimeModesRAII(CGOpenMPRuntimeGPU::ExecutionMode &ExecMode,
84 CGOpenMPRuntimeGPU::ExecutionMode EntryMode)
85 : ExecMode(ExecMode) {
86 SavedExecMode = ExecMode;
87 ExecMode = EntryMode;
88 }
89 ~ExecutionRuntimeModesRAII() { ExecMode = SavedExecMode; }
90};
91
92static const ValueDecl *getPrivateItem(const Expr *RefExpr) {
93 RefExpr = RefExpr->IgnoreParens();
94 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: RefExpr)) {
95 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
96 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
97 Base = TempASE->getBase()->IgnoreParenImpCasts();
98 RefExpr = Base;
99 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(Val: RefExpr)) {
100 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
101 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Val: Base))
102 Base = TempOASE->getBase()->IgnoreParenImpCasts();
103 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
104 Base = TempASE->getBase()->IgnoreParenImpCasts();
105 RefExpr = Base;
106 }
107 RefExpr = RefExpr->IgnoreParenImpCasts();
108 if (const auto *DE = dyn_cast<DeclRefExpr>(Val: RefExpr))
109 return cast<ValueDecl>(Val: DE->getDecl()->getCanonicalDecl());
110 const auto *ME = cast<MemberExpr>(Val: RefExpr);
111 return cast<ValueDecl>(Val: ME->getMemberDecl()->getCanonicalDecl());
112}
113
114static RecordDecl *buildRecordForGlobalizedVars(
115 ASTContext &C, ArrayRef<const ValueDecl *> EscapedDecls,
116 ArrayRef<const ValueDecl *> EscapedDeclsForTeams,
117 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
118 &MappedDeclsFields,
119 int BufSize) {
120 using VarsDataTy = std::pair<CharUnits /*Align*/, const ValueDecl *>;
121 if (EscapedDecls.empty() && EscapedDeclsForTeams.empty())
122 return nullptr;
123 SmallVector<VarsDataTy, 4> GlobalizedVars;
124 for (const ValueDecl *D : EscapedDecls)
125 GlobalizedVars.emplace_back(Args: C.getDeclAlign(D), Args&: D);
126 for (const ValueDecl *D : EscapedDeclsForTeams)
127 GlobalizedVars.emplace_back(Args: C.getDeclAlign(D), Args&: D);
128
129 // Build struct _globalized_locals_ty {
130 // /* globalized vars */[WarSize] align (decl_align)
131 // /* globalized vars */ for EscapedDeclsForTeams
132 // };
133 RecordDecl *GlobalizedRD = C.buildImplicitRecord(Name: "_globalized_locals_ty");
134 GlobalizedRD->startDefinition();
135 llvm::SmallPtrSet<const ValueDecl *, 16> SingleEscaped(llvm::from_range,
136 EscapedDeclsForTeams);
137 for (const auto &Pair : GlobalizedVars) {
138 const ValueDecl *VD = Pair.second;
139 QualType Type = VD->getType();
140 if (Type->isLValueReferenceType())
141 Type = C.getPointerType(T: Type.getNonReferenceType());
142 else
143 Type = Type.getNonReferenceType();
144 SourceLocation Loc = VD->getLocation();
145 FieldDecl *Field;
146 if (SingleEscaped.count(Ptr: VD)) {
147 Field = FieldDecl::Create(
148 C, DC: GlobalizedRD, StartLoc: Loc, IdLoc: Loc, Id: VD->getIdentifier(), T: Type,
149 TInfo: C.getTrivialTypeSourceInfo(T: Type, Loc: SourceLocation()),
150 /*BW=*/nullptr, /*Mutable=*/false,
151 /*InitStyle=*/ICIS_NoInit);
152 Field->setAccess(AS_public);
153 if (VD->hasAttrs()) {
154 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
155 E(VD->getAttrs().end());
156 I != E; ++I)
157 Field->addAttr(A: *I);
158 }
159 } else {
160 if (BufSize > 1) {
161 llvm::APInt ArraySize(32, BufSize);
162 Type = C.getConstantArrayType(EltTy: Type, ArySize: ArraySize, SizeExpr: nullptr,
163 ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
164 }
165 Field = FieldDecl::Create(
166 C, DC: GlobalizedRD, StartLoc: Loc, IdLoc: Loc, Id: VD->getIdentifier(), T: Type,
167 TInfo: C.getTrivialTypeSourceInfo(T: Type, Loc: SourceLocation()),
168 /*BW=*/nullptr, /*Mutable=*/false,
169 /*InitStyle=*/ICIS_NoInit);
170 Field->setAccess(AS_public);
171 llvm::APInt Align(32, Pair.first.getQuantity());
172 Field->addAttr(A: AlignedAttr::CreateImplicit(
173 Ctx&: C, /*IsAlignmentExpr=*/true,
174 Alignment: IntegerLiteral::Create(C, V: Align,
175 type: C.getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
176 l: SourceLocation()),
177 Range: {}, S: AlignedAttr::GNU_aligned));
178 }
179 GlobalizedRD->addDecl(D: Field);
180 MappedDeclsFields.try_emplace(Key: VD, Args&: Field);
181 }
182 GlobalizedRD->completeDefinition();
183 return GlobalizedRD;
184}
185
186/// Get the list of variables that can escape their declaration context.
187class CheckVarsEscapingDeclContext final
188 : public ConstStmtVisitor<CheckVarsEscapingDeclContext> {
189 CodeGenFunction &CGF;
190 llvm::SetVector<const ValueDecl *> EscapedDecls;
191 llvm::SetVector<const ValueDecl *> EscapedVariableLengthDecls;
192 llvm::SetVector<const ValueDecl *> DelayedVariableLengthDecls;
193 llvm::SmallPtrSet<const Decl *, 4> EscapedParameters;
194 RecordDecl *GlobalizedRD = nullptr;
195 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> MappedDeclsFields;
196 bool AllEscaped = false;
197 bool IsForCombinedParallelRegion = false;
198
199 void markAsEscaped(const ValueDecl *VD) {
200 // Do not globalize declare target variables.
201 if (!isa<VarDecl>(Val: VD) ||
202 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD))
203 return;
204 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
205 // Use user-specified allocation.
206 if (VD->hasAttrs() && VD->hasAttr<OMPAllocateDeclAttr>())
207 return;
208 // Variables captured by value must be globalized.
209 bool IsCaptured = false;
210 if (auto *CSI = CGF.CapturedStmtInfo) {
211 if (const FieldDecl *FD = CSI->lookup(VD: cast<VarDecl>(Val: VD))) {
212 // Check if need to capture the variable that was already captured by
213 // value in the outer region.
214 IsCaptured = true;
215 if (!IsForCombinedParallelRegion) {
216 if (!FD->hasAttrs())
217 return;
218 const auto *Attr = FD->getAttr<OMPCaptureKindAttr>();
219 if (!Attr)
220 return;
221 if (((Attr->getCaptureKind() != OMPC_map) &&
222 !isOpenMPPrivate(Kind: Attr->getCaptureKind())) ||
223 ((Attr->getCaptureKind() == OMPC_map) &&
224 !FD->getType()->isAnyPointerType()))
225 return;
226 }
227 if (!FD->getType()->isReferenceType()) {
228 assert(!VD->getType()->isVariablyModifiedType() &&
229 "Parameter captured by value with variably modified type");
230 EscapedParameters.insert(Ptr: VD);
231 } else if (!IsForCombinedParallelRegion) {
232 return;
233 }
234 }
235 }
236 if ((!CGF.CapturedStmtInfo ||
237 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) &&
238 VD->getType()->isReferenceType())
239 // Do not globalize variables with reference type.
240 return;
241 if (VD->getType()->isVariablyModifiedType()) {
242 // If not captured at the target region level then mark the escaped
243 // variable as delayed.
244 if (IsCaptured)
245 EscapedVariableLengthDecls.insert(X: VD);
246 else
247 DelayedVariableLengthDecls.insert(X: VD);
248 } else
249 EscapedDecls.insert(X: VD);
250 }
251
252 void VisitValueDecl(const ValueDecl *VD) {
253 if (VD->getType()->isLValueReferenceType())
254 markAsEscaped(VD);
255 if (const auto *VarD = dyn_cast<VarDecl>(Val: VD)) {
256 if (!isa<ParmVarDecl>(Val: VarD) && VarD->hasInit()) {
257 const bool SavedAllEscaped = AllEscaped;
258 AllEscaped = VD->getType()->isLValueReferenceType();
259 Visit(S: VarD->getInit());
260 AllEscaped = SavedAllEscaped;
261 }
262 }
263 }
264 void VisitOpenMPCapturedStmt(const CapturedStmt *S,
265 ArrayRef<OMPClause *> Clauses,
266 bool IsCombinedParallelRegion) {
267 if (!S)
268 return;
269 for (const CapturedStmt::Capture &C : S->captures()) {
270 if (C.capturesVariable() && !C.capturesVariableByCopy()) {
271 const ValueDecl *VD = C.getCapturedVar();
272 bool SavedIsForCombinedParallelRegion = IsForCombinedParallelRegion;
273 if (IsCombinedParallelRegion) {
274 // Check if the variable is privatized in the combined construct and
275 // those private copies must be shared in the inner parallel
276 // directive.
277 IsForCombinedParallelRegion = false;
278 for (const OMPClause *C : Clauses) {
279 if (!isOpenMPPrivate(Kind: C->getClauseKind()) ||
280 C->getClauseKind() == OMPC_reduction ||
281 C->getClauseKind() == OMPC_linear ||
282 C->getClauseKind() == OMPC_private)
283 continue;
284 ArrayRef<const Expr *> Vars;
285 if (const auto *PC = dyn_cast<OMPFirstprivateClause>(Val: C))
286 Vars = PC->getVarRefs();
287 else if (const auto *PC = dyn_cast<OMPLastprivateClause>(Val: C))
288 Vars = PC->getVarRefs();
289 else
290 llvm_unreachable("Unexpected clause.");
291 for (const auto *E : Vars) {
292 const Decl *D =
293 cast<DeclRefExpr>(Val: E)->getDecl()->getCanonicalDecl();
294 if (D == VD->getCanonicalDecl()) {
295 IsForCombinedParallelRegion = true;
296 break;
297 }
298 }
299 if (IsForCombinedParallelRegion)
300 break;
301 }
302 }
303 markAsEscaped(VD);
304 if (isa<OMPCapturedExprDecl>(Val: VD))
305 VisitValueDecl(VD);
306 IsForCombinedParallelRegion = SavedIsForCombinedParallelRegion;
307 }
308 }
309 }
310
311 void buildRecordForGlobalizedVars(bool IsInTTDRegion) {
312 assert(!GlobalizedRD &&
313 "Record for globalized variables is built already.");
314 ArrayRef<const ValueDecl *> EscapedDeclsForParallel, EscapedDeclsForTeams;
315 unsigned WarpSize = CGF.getTarget().getGridValue().GV_Warp_Size;
316 if (IsInTTDRegion)
317 EscapedDeclsForTeams = EscapedDecls.getArrayRef();
318 else
319 EscapedDeclsForParallel = EscapedDecls.getArrayRef();
320 GlobalizedRD = ::buildRecordForGlobalizedVars(
321 C&: CGF.getContext(), EscapedDecls: EscapedDeclsForParallel, EscapedDeclsForTeams,
322 MappedDeclsFields, BufSize: WarpSize);
323 }
324
325public:
326 CheckVarsEscapingDeclContext(CodeGenFunction &CGF,
327 ArrayRef<const ValueDecl *> TeamsReductions)
328 : CGF(CGF), EscapedDecls(llvm::from_range, TeamsReductions) {}
329 ~CheckVarsEscapingDeclContext() = default;
330 void VisitDeclStmt(const DeclStmt *S) {
331 if (!S)
332 return;
333 for (const Decl *D : S->decls())
334 if (const auto *VD = dyn_cast_or_null<ValueDecl>(Val: D))
335 VisitValueDecl(VD);
336 }
337 void VisitOMPExecutableDirective(const OMPExecutableDirective *D) {
338 if (!D)
339 return;
340 if (!D->hasAssociatedStmt())
341 return;
342 if (const auto *S =
343 dyn_cast_or_null<CapturedStmt>(Val: D->getAssociatedStmt())) {
344 // Do not analyze directives that do not actually require capturing,
345 // like `omp for` or `omp simd` directives.
346 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
347 getOpenMPCaptureRegions(CaptureRegions, DKind: D->getDirectiveKind());
348 if (CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown) {
349 VisitStmt(S: S->getCapturedStmt());
350 return;
351 }
352 VisitOpenMPCapturedStmt(
353 S, Clauses: D->clauses(),
354 IsCombinedParallelRegion: CaptureRegions.back() == OMPD_parallel &&
355 isOpenMPDistributeDirective(DKind: D->getDirectiveKind()));
356 }
357 }
358 // A metadirective has no associated statement; visit its variants and
359 // conditions.
360 void VisitOMPMetaDirective(const OMPMetaDirective *D) { VisitStmt(S: D); }
361 void VisitCapturedStmt(const CapturedStmt *S) {
362 if (!S)
363 return;
364 for (const CapturedStmt::Capture &C : S->captures()) {
365 if (C.capturesVariable() && !C.capturesVariableByCopy()) {
366 const ValueDecl *VD = C.getCapturedVar();
367 markAsEscaped(VD);
368 if (isa<OMPCapturedExprDecl>(Val: VD))
369 VisitValueDecl(VD);
370 }
371 }
372 }
373 void VisitLambdaExpr(const LambdaExpr *E) {
374 if (!E)
375 return;
376 for (const LambdaCapture &C : E->captures()) {
377 if (C.capturesVariable()) {
378 if (C.getCaptureKind() == LCK_ByRef) {
379 const ValueDecl *VD = C.getCapturedVar();
380 markAsEscaped(VD);
381 if (E->isInitCapture(Capture: &C) || isa<OMPCapturedExprDecl>(Val: VD))
382 VisitValueDecl(VD);
383 }
384 }
385 }
386 }
387 void VisitBlockExpr(const BlockExpr *E) {
388 if (!E)
389 return;
390 for (const BlockDecl::Capture &C : E->getBlockDecl()->captures()) {
391 if (C.isByRef()) {
392 const VarDecl *VD = C.getVariable();
393 markAsEscaped(VD);
394 if (isa<OMPCapturedExprDecl>(Val: VD) || VD->isInitCapture())
395 VisitValueDecl(VD);
396 }
397 }
398 }
399 void VisitCallExpr(const CallExpr *E) {
400 if (!E)
401 return;
402 for (const Expr *Arg : E->arguments()) {
403 if (!Arg)
404 continue;
405 if (Arg->isLValue()) {
406 const bool SavedAllEscaped = AllEscaped;
407 AllEscaped = true;
408 Visit(S: Arg);
409 AllEscaped = SavedAllEscaped;
410 } else {
411 Visit(S: Arg);
412 }
413 }
414 Visit(S: E->getCallee());
415 }
416 void VisitDeclRefExpr(const DeclRefExpr *E) {
417 if (!E)
418 return;
419 const ValueDecl *VD = E->getDecl();
420 if (AllEscaped)
421 markAsEscaped(VD);
422 if (isa<OMPCapturedExprDecl>(Val: VD))
423 VisitValueDecl(VD);
424 else if (VD->isInitCapture())
425 VisitValueDecl(VD);
426 }
427 void VisitUnaryOperator(const UnaryOperator *E) {
428 if (!E)
429 return;
430 if (E->getOpcode() == UO_AddrOf) {
431 const bool SavedAllEscaped = AllEscaped;
432 AllEscaped = true;
433 Visit(S: E->getSubExpr());
434 AllEscaped = SavedAllEscaped;
435 } else {
436 Visit(S: E->getSubExpr());
437 }
438 }
439 void VisitImplicitCastExpr(const ImplicitCastExpr *E) {
440 if (!E)
441 return;
442 if (E->getCastKind() == CK_ArrayToPointerDecay) {
443 const bool SavedAllEscaped = AllEscaped;
444 AllEscaped = true;
445 Visit(S: E->getSubExpr());
446 AllEscaped = SavedAllEscaped;
447 } else {
448 Visit(S: E->getSubExpr());
449 }
450 }
451 void VisitExpr(const Expr *E) {
452 if (!E)
453 return;
454 bool SavedAllEscaped = AllEscaped;
455 if (!E->isLValue())
456 AllEscaped = false;
457 for (const Stmt *Child : E->children())
458 if (Child)
459 Visit(S: Child);
460 AllEscaped = SavedAllEscaped;
461 }
462 void VisitStmt(const Stmt *S) {
463 if (!S)
464 return;
465 for (const Stmt *Child : S->children())
466 if (Child)
467 Visit(S: Child);
468 }
469
470 /// Returns the record that handles all the escaped local variables and used
471 /// instead of their original storage.
472 const RecordDecl *getGlobalizedRecord(bool IsInTTDRegion) {
473 if (!GlobalizedRD)
474 buildRecordForGlobalizedVars(IsInTTDRegion);
475 return GlobalizedRD;
476 }
477
478 /// Returns the field in the globalized record for the escaped variable.
479 const FieldDecl *getFieldForGlobalizedVar(const ValueDecl *VD) const {
480 assert(GlobalizedRD &&
481 "Record for globalized variables must be generated already.");
482 return MappedDeclsFields.lookup(Val: VD);
483 }
484
485 /// Returns the list of the escaped local variables/parameters.
486 ArrayRef<const ValueDecl *> getEscapedDecls() const {
487 return EscapedDecls.getArrayRef();
488 }
489
490 /// Checks if the escaped local variable is actually a parameter passed by
491 /// value.
492 const llvm::SmallPtrSetImpl<const Decl *> &getEscapedParameters() const {
493 return EscapedParameters;
494 }
495
496 /// Returns the list of the escaped variables with the variably modified
497 /// types.
498 ArrayRef<const ValueDecl *> getEscapedVariableLengthDecls() const {
499 return EscapedVariableLengthDecls.getArrayRef();
500 }
501
502 /// Returns the list of the delayed variables with the variably modified
503 /// types.
504 ArrayRef<const ValueDecl *> getDelayedVariableLengthDecls() const {
505 return DelayedVariableLengthDecls.getArrayRef();
506 }
507};
508} // anonymous namespace
509
510CGOpenMPRuntimeGPU::ExecutionMode
511CGOpenMPRuntimeGPU::getExecutionMode() const {
512 return CurrentExecutionMode;
513}
514
515CGOpenMPRuntimeGPU::DataSharingMode
516CGOpenMPRuntimeGPU::getDataSharingMode() const {
517 return CurrentDataSharingMode;
518}
519
520/// Check for inner (nested) SPMD construct, if any
521static bool hasNestedSPMDDirective(ASTContext &Ctx,
522 const OMPExecutableDirective &D) {
523 const auto *CS = D.getInnermostCapturedStmt();
524 const auto *Body =
525 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
526 const Stmt *ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
527
528 if (const auto *NestedDir =
529 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
530 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
531 switch (D.getDirectiveKind()) {
532 case OMPD_target:
533 if (isOpenMPParallelDirective(DKind))
534 return true;
535 if (DKind == OMPD_teams) {
536 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
537 /*IgnoreCaptured=*/true);
538 if (!Body)
539 return false;
540 ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
541 if (const auto *NND =
542 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
543 DKind = NND->getDirectiveKind();
544 if (isOpenMPParallelDirective(DKind))
545 return true;
546 }
547 }
548 return false;
549 case OMPD_target_teams:
550 return isOpenMPParallelDirective(DKind);
551 case OMPD_target_simd:
552 case OMPD_target_parallel:
553 case OMPD_target_parallel_for:
554 case OMPD_target_parallel_for_simd:
555 case OMPD_target_teams_distribute:
556 case OMPD_target_teams_distribute_simd:
557 case OMPD_target_teams_distribute_parallel_for:
558 case OMPD_target_teams_distribute_parallel_for_simd:
559 case OMPD_parallel:
560 case OMPD_for:
561 case OMPD_parallel_for:
562 case OMPD_parallel_master:
563 case OMPD_parallel_sections:
564 case OMPD_for_simd:
565 case OMPD_parallel_for_simd:
566 case OMPD_cancel:
567 case OMPD_cancellation_point:
568 case OMPD_ordered_standalone:
569 case OMPD_ordered_blockassoc:
570 case OMPD_threadprivate:
571 case OMPD_allocate:
572 case OMPD_task:
573 case OMPD_simd:
574 case OMPD_sections:
575 case OMPD_section:
576 case OMPD_single:
577 case OMPD_master:
578 case OMPD_critical:
579 case OMPD_taskyield:
580 case OMPD_barrier:
581 case OMPD_taskwait:
582 case OMPD_taskgroup:
583 case OMPD_atomic:
584 case OMPD_flush:
585 case OMPD_depobj:
586 case OMPD_scan:
587 case OMPD_teams:
588 case OMPD_target_data:
589 case OMPD_target_exit_data:
590 case OMPD_target_enter_data:
591 case OMPD_distribute:
592 case OMPD_distribute_simd:
593 case OMPD_distribute_parallel_for:
594 case OMPD_distribute_parallel_for_simd:
595 case OMPD_teams_distribute:
596 case OMPD_teams_distribute_simd:
597 case OMPD_teams_distribute_parallel_for:
598 case OMPD_teams_distribute_parallel_for_simd:
599 case OMPD_target_update:
600 case OMPD_declare_simd:
601 case OMPD_declare_variant:
602 case OMPD_begin_declare_variant:
603 case OMPD_end_declare_variant:
604 case OMPD_declare_target:
605 case OMPD_end_declare_target:
606 case OMPD_declare_reduction:
607 case OMPD_declare_mapper:
608 case OMPD_taskloop:
609 case OMPD_taskloop_simd:
610 case OMPD_master_taskloop:
611 case OMPD_master_taskloop_simd:
612 case OMPD_parallel_master_taskloop:
613 case OMPD_parallel_master_taskloop_simd:
614 case OMPD_requires:
615 case OMPD_unknown:
616 default:
617 llvm_unreachable("Unexpected directive.");
618 }
619 }
620
621 return false;
622}
623
624static bool supportsSPMDExecutionMode(ASTContext &Ctx,
625 const OMPExecutableDirective &D) {
626 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
627 switch (DirectiveKind) {
628 case OMPD_target:
629 case OMPD_target_teams:
630 return hasNestedSPMDDirective(Ctx, D);
631 case OMPD_target_parallel_loop:
632 case OMPD_target_parallel:
633 case OMPD_target_parallel_for:
634 case OMPD_target_parallel_for_simd:
635 case OMPD_target_teams_distribute_parallel_for:
636 case OMPD_target_teams_distribute_parallel_for_simd:
637 case OMPD_target_simd:
638 case OMPD_target_loop:
639 case OMPD_target_teams_distribute_simd:
640 return true;
641 case OMPD_target_teams_distribute:
642 return false;
643 case OMPD_target_teams_loop:
644 // Whether this is true or not depends on how the directive will
645 // eventually be emitted.
646 if (auto *TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(Val: &D))
647 return TTLD->canBeParallelFor();
648 return false;
649 case OMPD_parallel:
650 case OMPD_for:
651 case OMPD_parallel_for:
652 case OMPD_parallel_master:
653 case OMPD_parallel_sections:
654 case OMPD_for_simd:
655 case OMPD_parallel_for_simd:
656 case OMPD_cancel:
657 case OMPD_cancellation_point:
658 case OMPD_ordered_standalone:
659 case OMPD_ordered_blockassoc:
660 case OMPD_threadprivate:
661 case OMPD_allocate:
662 case OMPD_task:
663 case OMPD_simd:
664 case OMPD_sections:
665 case OMPD_section:
666 case OMPD_single:
667 case OMPD_master:
668 case OMPD_critical:
669 case OMPD_taskyield:
670 case OMPD_barrier:
671 case OMPD_taskwait:
672 case OMPD_taskgroup:
673 case OMPD_atomic:
674 case OMPD_flush:
675 case OMPD_depobj:
676 case OMPD_scan:
677 case OMPD_teams:
678 case OMPD_target_data:
679 case OMPD_target_exit_data:
680 case OMPD_target_enter_data:
681 case OMPD_distribute:
682 case OMPD_distribute_simd:
683 case OMPD_distribute_parallel_for:
684 case OMPD_distribute_parallel_for_simd:
685 case OMPD_teams_distribute:
686 case OMPD_teams_distribute_simd:
687 case OMPD_teams_distribute_parallel_for:
688 case OMPD_teams_distribute_parallel_for_simd:
689 case OMPD_target_update:
690 case OMPD_declare_simd:
691 case OMPD_declare_variant:
692 case OMPD_begin_declare_variant:
693 case OMPD_end_declare_variant:
694 case OMPD_declare_target:
695 case OMPD_end_declare_target:
696 case OMPD_declare_reduction:
697 case OMPD_declare_mapper:
698 case OMPD_taskloop:
699 case OMPD_taskloop_simd:
700 case OMPD_master_taskloop:
701 case OMPD_master_taskloop_simd:
702 case OMPD_parallel_master_taskloop:
703 case OMPD_parallel_master_taskloop_simd:
704 case OMPD_requires:
705 case OMPD_unknown:
706 default:
707 break;
708 }
709 llvm_unreachable(
710 "Unknown programming model for OpenMP directive on NVPTX target.");
711}
712
713void CGOpenMPRuntimeGPU::emitNonSPMDKernel(const OMPExecutableDirective &D,
714 StringRef ParentName,
715 llvm::Function *&OutlinedFn,
716 llvm::Constant *&OutlinedFnID,
717 bool IsOffloadEntry,
718 const RegionCodeGenTy &CodeGen) {
719 ExecutionRuntimeModesRAII ModeRAII(CurrentExecutionMode, EM_NonSPMD);
720 EntryFunctionState EST;
721 WrapperFunctionsMap.clear();
722
723 [[maybe_unused]] bool IsBareKernel = D.getSingleClause<OMPXBareClause>();
724 assert(!IsBareKernel && "bare kernel should not be at generic mode");
725
726 // Emit target region as a standalone region.
727 class NVPTXPrePostActionTy : public PrePostActionTy {
728 CGOpenMPRuntimeGPU::EntryFunctionState &EST;
729 const OMPExecutableDirective &D;
730
731 public:
732 NVPTXPrePostActionTy(CGOpenMPRuntimeGPU::EntryFunctionState &EST,
733 const OMPExecutableDirective &D)
734 : EST(EST), D(D) {}
735 void Enter(CodeGenFunction &CGF) override {
736 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
737 RT.emitKernelInit(D, CGF, EST, /* IsSPMD */ false);
738 // Skip target region initialization.
739 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
740 }
741 void Exit(CodeGenFunction &CGF) override {
742 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
743 RT.clearLocThreadIdInsertPt(CGF);
744 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ false);
745 }
746 } Action(EST, D);
747 CodeGen.setAction(Action);
748 IsInTTDRegion = true;
749 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
750 IsOffloadEntry, CodeGen);
751 IsInTTDRegion = false;
752}
753
754void CGOpenMPRuntimeGPU::emitBareKernelEnvironment(
755 const OMPExecutableDirective &D, CodeGenFunction &CGF) {
756 // Bare kernels manage their own initialization and never call
757 // __kmpc_target_init, but the runtime still needs a
758 // '<kernel>_kernel_environment' global to know how the kernel was
759 // configured, so emit it directly here.
760 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs Attrs;
761 Attrs.ExecFlags = llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_BARE;
762 CGBuilderTy &Bld = CGF.Builder;
763 OMPBuilder.emitKernelEnvironment(Loc: Bld, Attrs);
764}
765
766void CGOpenMPRuntimeGPU::emitKernelInit(const OMPExecutableDirective &D,
767 CodeGenFunction &CGF,
768 EntryFunctionState &EST, bool IsSPMD) {
769 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs Attrs;
770 Attrs.ExecFlags =
771 IsSPMD ? llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_SPMD
772 : llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC;
773 computeMinAndMaxThreadsAndTeams(D, CGF, Attrs);
774
775 CGBuilderTy &Bld = CGF.Builder;
776 Bld.restoreIP(IP: OMPBuilder.createTargetInit(Loc: Bld, Attrs));
777 if (!IsSPMD)
778 emitGenericVarsProlog(CGF, Loc: EST.Loc);
779}
780
781void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF,
782 EntryFunctionState &EST,
783 bool IsSPMD) {
784 if (!IsSPMD)
785 emitGenericVarsEpilog(CGF);
786
787 // This is temporary until we remove the fixed sized buffer.
788 ASTContext &C = CGM.getContext();
789 RecordDecl *StaticRD = C.buildImplicitRecord(
790 Name: "_openmp_teams_reduction_type_$_", TK: RecordDecl::TagKind::Union);
791 StaticRD->startDefinition();
792 for (const RecordDecl *TeamReductionRec : TeamsReductions) {
793 CanQualType RecTy = C.getCanonicalTagType(TD: TeamReductionRec);
794 auto *Field = FieldDecl::Create(
795 C, DC: StaticRD, StartLoc: SourceLocation(), IdLoc: SourceLocation(), Id: nullptr, T: RecTy,
796 TInfo: C.getTrivialTypeSourceInfo(T: RecTy, Loc: SourceLocation()),
797 /*BW=*/nullptr, /*Mutable=*/false,
798 /*InitStyle=*/ICIS_NoInit);
799 Field->setAccess(AS_public);
800 StaticRD->addDecl(D: Field);
801 }
802 StaticRD->completeDefinition();
803 CanQualType StaticTy = C.getCanonicalTagType(TD: StaticRD);
804 llvm::Type *LLVMReductionsBufferTy =
805 CGM.getTypes().ConvertTypeForMem(T: StaticTy);
806 const auto &DL = CGM.getModule().getDataLayout();
807 uint64_t ReductionDataSize =
808 TeamsReductions.empty()
809 ? 0
810 : DL.getTypeAllocSize(Ty: LLVMReductionsBufferTy).getFixedValue();
811 CGBuilderTy &Bld = CGF.Builder;
812 OMPBuilder.createTargetDeinit(Loc: Bld, TeamsReductionDataSize: ReductionDataSize);
813 TeamsReductions.clear();
814}
815
816void CGOpenMPRuntimeGPU::emitSPMDKernel(const OMPExecutableDirective &D,
817 StringRef ParentName,
818 llvm::Function *&OutlinedFn,
819 llvm::Constant *&OutlinedFnID,
820 bool IsOffloadEntry,
821 const RegionCodeGenTy &CodeGen) {
822 ExecutionRuntimeModesRAII ModeRAII(CurrentExecutionMode, EM_SPMD);
823 EntryFunctionState EST;
824
825 bool IsBareKernel = D.getSingleClause<OMPXBareClause>();
826
827 // Emit target region as a standalone region.
828 class NVPTXPrePostActionTy : public PrePostActionTy {
829 CGOpenMPRuntimeGPU &RT;
830 CGOpenMPRuntimeGPU::EntryFunctionState &EST;
831 bool IsBareKernel;
832 DataSharingMode Mode;
833 const OMPExecutableDirective &D;
834
835 public:
836 NVPTXPrePostActionTy(CGOpenMPRuntimeGPU &RT,
837 CGOpenMPRuntimeGPU::EntryFunctionState &EST,
838 bool IsBareKernel, const OMPExecutableDirective &D)
839 : RT(RT), EST(EST), IsBareKernel(IsBareKernel),
840 Mode(RT.CurrentDataSharingMode), D(D) {}
841 void Enter(CodeGenFunction &CGF) override {
842 if (IsBareKernel) {
843 RT.CurrentDataSharingMode = DataSharingMode::DS_CUDA;
844 RT.emitBareKernelEnvironment(D, CGF);
845 return;
846 }
847 RT.emitKernelInit(D, CGF, EST, /* IsSPMD */ true);
848 // Skip target region initialization.
849 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
850 }
851 void Exit(CodeGenFunction &CGF) override {
852 if (IsBareKernel) {
853 RT.CurrentDataSharingMode = Mode;
854 return;
855 }
856 RT.clearLocThreadIdInsertPt(CGF);
857 RT.emitKernelDeinit(CGF, EST, /* IsSPMD */ true);
858 }
859 } Action(*this, EST, IsBareKernel, D);
860 CodeGen.setAction(Action);
861 IsInTTDRegion = true;
862 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
863 IsOffloadEntry, CodeGen);
864 IsInTTDRegion = false;
865}
866
867void CGOpenMPRuntimeGPU::emitTargetOutlinedFunction(
868 const OMPExecutableDirective &D, StringRef ParentName,
869 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
870 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
871 if (!IsOffloadEntry) // Nothing to do.
872 return;
873
874 assert(!ParentName.empty() && "Invalid target region parent name!");
875
876 bool Mode = supportsSPMDExecutionMode(Ctx&: CGM.getContext(), D);
877 bool IsBareKernel = D.getSingleClause<OMPXBareClause>();
878 if (Mode || IsBareKernel)
879 emitSPMDKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
880 CodeGen);
881 else
882 emitNonSPMDKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
883 CodeGen);
884}
885
886CGOpenMPRuntimeGPU::CGOpenMPRuntimeGPU(CodeGenModule &CGM)
887 : CGOpenMPRuntime(CGM) {
888 llvm::OpenMPIRBuilderConfig Config(
889 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),
890 CGM.getLangOpts().OpenMPOffloadMandatory,
891 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,
892 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);
893 Config.setDefaultTargetAS(
894 CGM.getContext().getTargetInfo().getTargetAddressSpace(AS: LangAS::Default));
895 Config.setRuntimeCC(CGM.getRuntimeCC());
896
897 OMPBuilder.setConfig(Config);
898
899 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
900 llvm_unreachable("OpenMP can only handle device code.");
901
902 if (CGM.getLangOpts().OpenMPCUDAMode)
903 CurrentDataSharingMode = CGOpenMPRuntimeGPU::DS_CUDA;
904
905 llvm::OpenMPIRBuilder &OMPBuilder = getOMPBuilder();
906 if (CGM.getLangOpts().NoGPULib || CGM.getLangOpts().OMPHostIRFile.empty())
907 return;
908
909 OMPBuilder.createGlobalFlag(Value: CGM.getLangOpts().OpenMPTargetDebug,
910 Name: "__omp_rtl_debug_kind");
911 OMPBuilder.createGlobalFlag(Value: CGM.getLangOpts().OpenMPTeamSubscription,
912 Name: "__omp_rtl_assume_teams_oversubscription");
913 OMPBuilder.createGlobalFlag(Value: CGM.getLangOpts().OpenMPThreadSubscription,
914 Name: "__omp_rtl_assume_threads_oversubscription");
915 OMPBuilder.createGlobalFlag(Value: CGM.getLangOpts().OpenMPNoThreadState,
916 Name: "__omp_rtl_assume_no_thread_state");
917 OMPBuilder.createGlobalFlag(Value: CGM.getLangOpts().OpenMPNoNestedParallelism,
918 Name: "__omp_rtl_assume_no_nested_parallelism");
919}
920
921void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF,
922 ProcBindKind ProcBind,
923 SourceLocation Loc) {
924 // Nothing to do.
925}
926
927llvm::Value *CGOpenMPRuntimeGPU::emitMessageClause(CodeGenFunction &CGF,
928 const Expr *Message,
929 SourceLocation Loc) {
930 CGM.getDiags().Report(Loc, DiagID: diag::warn_omp_gpu_unsupported_clause)
931 << getOpenMPClauseName(C: OMPC_message);
932 return nullptr;
933}
934
935llvm::Value *
936CGOpenMPRuntimeGPU::emitSeverityClause(OpenMPSeverityClauseKind Severity,
937 SourceLocation Loc) {
938 CGM.getDiags().Report(Loc, DiagID: diag::warn_omp_gpu_unsupported_clause)
939 << getOpenMPClauseName(C: OMPC_severity);
940 return nullptr;
941}
942
943void CGOpenMPRuntimeGPU::emitNumThreadsClause(
944 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
945 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
946 SourceLocation SeverityLoc, const Expr *Message,
947 SourceLocation MessageLoc) {
948 if (Modifier == OMPC_NUMTHREADS_strict) {
949 CGM.getDiags().Report(Loc,
950 DiagID: diag::warn_omp_gpu_unsupported_modifier_for_clause)
951 << "strict" << getOpenMPClauseName(C: OMPC_num_threads);
952 return;
953 }
954
955 // Nothing to do.
956}
957
958void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF,
959 const Expr *NumTeams,
960 const Expr *ThreadLimit,
961 SourceLocation Loc) {}
962
963llvm::Function *CGOpenMPRuntimeGPU::emitParallelOutlinedFunction(
964 CodeGenFunction &CGF, const OMPExecutableDirective &D,
965 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
966 const RegionCodeGenTy &CodeGen) {
967 // Emit target region as a standalone region.
968 bool PrevIsInTTDRegion = IsInTTDRegion;
969 IsInTTDRegion = false;
970 auto *OutlinedFun =
971 cast<llvm::Function>(Val: CGOpenMPRuntime::emitParallelOutlinedFunction(
972 CGF, D, ThreadIDVar, InnermostKind, CodeGen));
973 IsInTTDRegion = PrevIsInTTDRegion;
974 if (getExecutionMode() != CGOpenMPRuntimeGPU::EM_SPMD) {
975 llvm::Function *WrapperFun =
976 createParallelDataSharingWrapper(OutlinedParallelFn: OutlinedFun, D);
977 WrapperFunctionsMap[OutlinedFun] = WrapperFun;
978 }
979
980 return OutlinedFun;
981}
982
983/// Get list of lastprivate variables from the teams distribute ... or
984/// teams {distribute ...} directives.
985static void
986getDistributeLastprivateVars(ASTContext &Ctx, const OMPExecutableDirective &D,
987 llvm::SmallVectorImpl<const ValueDecl *> &Vars) {
988 assert(isOpenMPTeamsDirective(D.getDirectiveKind()) &&
989 "expected teams directive.");
990 const OMPExecutableDirective *Dir = &D;
991 if (!isOpenMPDistributeDirective(DKind: D.getDirectiveKind())) {
992 if (const Stmt *S = CGOpenMPRuntime::getSingleCompoundChild(
993 Ctx,
994 Body: D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers(
995 /*IgnoreCaptured=*/true))) {
996 Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: S);
997 if (Dir && !isOpenMPDistributeDirective(DKind: Dir->getDirectiveKind()))
998 Dir = nullptr;
999 }
1000 }
1001 if (!Dir)
1002 return;
1003 for (const auto *C : Dir->getClausesOfKind<OMPLastprivateClause>()) {
1004 for (const Expr *E : C->getVarRefs())
1005 Vars.push_back(Elt: getPrivateItem(RefExpr: E));
1006 }
1007}
1008
1009/// Get list of reduction variables from the teams ... directives.
1010static void
1011getTeamsReductionVars(ASTContext &Ctx, const OMPExecutableDirective &D,
1012 llvm::SmallVectorImpl<const ValueDecl *> &Vars) {
1013 assert(isOpenMPTeamsDirective(D.getDirectiveKind()) &&
1014 "expected teams directive.");
1015 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1016 for (const Expr *E : C->privates())
1017 Vars.push_back(Elt: getPrivateItem(RefExpr: E));
1018 }
1019}
1020
1021llvm::Function *CGOpenMPRuntimeGPU::emitTeamsOutlinedFunction(
1022 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1023 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1024 const RegionCodeGenTy &CodeGen) {
1025 SourceLocation Loc = D.getBeginLoc();
1026
1027 const RecordDecl *GlobalizedRD = nullptr;
1028 llvm::SmallVector<const ValueDecl *, 4> LastPrivatesReductions;
1029 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> MappedDeclsFields;
1030 unsigned WarpSize = CGM.getTarget().getGridValue().GV_Warp_Size;
1031 // Globalize team reductions variable unconditionally in all modes.
1032 if (getExecutionMode() != CGOpenMPRuntimeGPU::EM_SPMD)
1033 getTeamsReductionVars(Ctx&: CGM.getContext(), D, Vars&: LastPrivatesReductions);
1034 if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD) {
1035 getDistributeLastprivateVars(Ctx&: CGM.getContext(), D, Vars&: LastPrivatesReductions);
1036 if (!LastPrivatesReductions.empty()) {
1037 GlobalizedRD = ::buildRecordForGlobalizedVars(
1038 C&: CGM.getContext(), EscapedDecls: {}, EscapedDeclsForTeams: LastPrivatesReductions, MappedDeclsFields,
1039 BufSize: WarpSize);
1040 }
1041 } else if (!LastPrivatesReductions.empty()) {
1042 assert(!TeamAndReductions.first &&
1043 "Previous team declaration is not expected.");
1044 TeamAndReductions.first = D.getCapturedStmt(RegionKind: OMPD_teams)->getCapturedDecl();
1045 std::swap(LHS&: TeamAndReductions.second, RHS&: LastPrivatesReductions);
1046 }
1047
1048 // Emit target region as a standalone region.
1049 class NVPTXPrePostActionTy : public PrePostActionTy {
1050 SourceLocation &Loc;
1051 const RecordDecl *GlobalizedRD;
1052 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
1053 &MappedDeclsFields;
1054
1055 public:
1056 NVPTXPrePostActionTy(
1057 SourceLocation &Loc, const RecordDecl *GlobalizedRD,
1058 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
1059 &MappedDeclsFields)
1060 : Loc(Loc), GlobalizedRD(GlobalizedRD),
1061 MappedDeclsFields(MappedDeclsFields) {}
1062 void Enter(CodeGenFunction &CGF) override {
1063 auto &Rt =
1064 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1065 if (GlobalizedRD) {
1066 auto I = Rt.FunctionGlobalizedDecls.try_emplace(Key: CGF.CurFn).first;
1067 I->getSecond().MappedParams =
1068 std::make_unique<CodeGenFunction::OMPMapVars>();
1069 DeclToAddrMapTy &Data = I->getSecond().LocalVarData;
1070 for (const auto &Pair : MappedDeclsFields) {
1071 assert(Pair.getFirst()->isCanonicalDecl() &&
1072 "Expected canonical declaration");
1073 Data.try_emplace(Key: Pair.getFirst());
1074 }
1075 }
1076 Rt.emitGenericVarsProlog(CGF, Loc);
1077 }
1078 void Exit(CodeGenFunction &CGF) override {
1079 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime())
1080 .emitGenericVarsEpilog(CGF);
1081 }
1082 } Action(Loc, GlobalizedRD, MappedDeclsFields);
1083 CodeGen.setAction(Action);
1084 llvm::Function *OutlinedFun = CGOpenMPRuntime::emitTeamsOutlinedFunction(
1085 CGF, D, ThreadIDVar, InnermostKind, CodeGen);
1086
1087 return OutlinedFun;
1088}
1089
1090void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF,
1091 SourceLocation Loc) {
1092 if (getDataSharingMode() != CGOpenMPRuntimeGPU::DS_Generic)
1093 return;
1094
1095 CGBuilderTy &Bld = CGF.Builder;
1096
1097 const auto I = FunctionGlobalizedDecls.find(Val: CGF.CurFn);
1098 if (I == FunctionGlobalizedDecls.end())
1099 return;
1100
1101 for (auto &Rec : I->getSecond().LocalVarData) {
1102 const auto *VD = cast<VarDecl>(Val: Rec.first);
1103 bool EscapedParam = I->getSecond().EscapedParameters.count(Ptr: Rec.first);
1104 QualType VarTy = VD->getType();
1105
1106 // Get the local allocation of a firstprivate variable before sharing
1107 llvm::Value *ParValue;
1108 if (EscapedParam) {
1109 LValue ParLVal =
1110 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD), T: VD->getType());
1111 ParValue = CGF.EmitLoadOfScalar(lvalue: ParLVal, Loc);
1112 }
1113
1114 // Allocate space for the variable to be globalized
1115 llvm::Value *AllocArgs[] = {CGF.getTypeSize(Ty: VD->getType())};
1116 llvm::CallBase *VoidPtr =
1117 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1118 M&: CGM.getModule(), FnID: OMPRTL___kmpc_alloc_shared),
1119 args: AllocArgs, name: VD->getName());
1120 // FIXME: We should use the variables actual alignment as an argument.
1121 VoidPtr->addRetAttr(Attr: llvm::Attribute::get(
1122 Context&: CGM.getLLVMContext(), Kind: llvm::Attribute::Alignment,
1123 Val: CGM.getContext().getTargetInfo().getNewAlign() / 8));
1124
1125 // Cast the void pointer and get the address of the globalized variable.
1126 llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1127 V: VoidPtr, DestTy: Bld.getPtrTy(AddrSpace: 0), Name: VD->getName() + "_on_stack");
1128 LValue VarAddr =
1129 CGF.MakeNaturalAlignPointeeRawAddrLValue(V: CastedVoidPtr, T: VarTy);
1130 Rec.second.PrivateAddr = VarAddr.getAddress();
1131 Rec.second.GlobalizedVal = VoidPtr;
1132
1133 // Assign the local allocation to the newly globalized location.
1134 if (EscapedParam) {
1135 CGF.EmitStoreOfScalar(value: ParValue, lvalue: VarAddr);
1136 I->getSecond().MappedParams->setVarAddr(CGF, LocalVD: VD, TempAddr: VarAddr.getAddress());
1137 }
1138 if (auto *DI = CGF.getDebugInfo())
1139 VoidPtr->setDebugLoc(DI->SourceLocToDebugLoc(Loc: VD->getLocation()));
1140 }
1141
1142 for (const auto *ValueD : I->getSecond().EscapedVariableLengthDecls) {
1143 const auto *VD = cast<VarDecl>(Val: ValueD);
1144 std::pair<llvm::Value *, llvm::Value *> AddrSizePair =
1145 getKmpcAllocShared(CGF, VD);
1146 I->getSecond().EscapedVariableLengthDeclsAddrs.emplace_back(Args&: AddrSizePair);
1147 LValue Base = CGF.MakeAddrLValue(V: AddrSizePair.first, T: VD->getType(),
1148 Alignment: CGM.getContext().getDeclAlign(D: VD),
1149 Source: AlignmentSource::Decl);
1150 I->getSecond().MappedParams->setVarAddr(CGF, LocalVD: VD, TempAddr: Base.getAddress());
1151 }
1152 I->getSecond().MappedParams->apply(CGF);
1153}
1154
1155bool CGOpenMPRuntimeGPU::isDelayedVariableLengthDecl(CodeGenFunction &CGF,
1156 const VarDecl *VD) const {
1157 const auto I = FunctionGlobalizedDecls.find(Val: CGF.CurFn);
1158 if (I == FunctionGlobalizedDecls.end())
1159 return false;
1160
1161 // Check variable declaration is delayed:
1162 return llvm::is_contained(Range: I->getSecond().DelayedVariableLengthDecls, Element: VD);
1163}
1164
1165std::pair<llvm::Value *, llvm::Value *>
1166CGOpenMPRuntimeGPU::getKmpcAllocShared(CodeGenFunction &CGF,
1167 const VarDecl *VD) {
1168 CGBuilderTy &Bld = CGF.Builder;
1169
1170 // Compute size and alignment.
1171 llvm::Value *Size = CGF.getTypeSize(Ty: VD->getType());
1172 CharUnits Align = CGM.getContext().getDeclAlign(D: VD);
1173 Size = Bld.CreateNUWAdd(
1174 LHS: Size, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Align.getQuantity() - 1));
1175 llvm::Value *AlignVal =
1176 llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Align.getQuantity());
1177 Size = Bld.CreateUDiv(LHS: Size, RHS: AlignVal);
1178 Size = Bld.CreateNUWMul(LHS: Size, RHS: AlignVal);
1179
1180 // Allocate space for this VLA object to be globalized.
1181 llvm::Value *AllocArgs[] = {Size};
1182 llvm::CallBase *VoidPtr =
1183 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1184 M&: CGM.getModule(), FnID: OMPRTL___kmpc_alloc_shared),
1185 args: AllocArgs, name: VD->getName());
1186 VoidPtr->addRetAttr(Attr: llvm::Attribute::get(
1187 Context&: CGM.getLLVMContext(), Kind: llvm::Attribute::Alignment, Val: Align.getQuantity()));
1188
1189 return std::make_pair(x&: VoidPtr, y&: Size);
1190}
1191
1192void CGOpenMPRuntimeGPU::getKmpcFreeShared(
1193 CodeGenFunction &CGF,
1194 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
1195 // Deallocate the memory for each globalized VLA object
1196 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1197 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free_shared),
1198 args: {AddrSizePair.first, AddrSizePair.second});
1199}
1200
1201void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF) {
1202 if (getDataSharingMode() != CGOpenMPRuntimeGPU::DS_Generic)
1203 return;
1204
1205 const auto I = FunctionGlobalizedDecls.find(Val: CGF.CurFn);
1206 if (I != FunctionGlobalizedDecls.end()) {
1207 // Deallocate the memory for each globalized VLA object that was
1208 // globalized in the prolog (i.e. emitGenericVarsProlog).
1209 for (const auto &AddrSizePair :
1210 llvm::reverse(C&: I->getSecond().EscapedVariableLengthDeclsAddrs)) {
1211 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1212 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free_shared),
1213 args: {AddrSizePair.first, AddrSizePair.second});
1214 }
1215 // Deallocate the memory for each globalized value
1216 for (auto &Rec : llvm::reverse(C&: I->getSecond().LocalVarData)) {
1217 const auto *VD = cast<VarDecl>(Val: Rec.first);
1218 I->getSecond().MappedParams->restore(CGF);
1219
1220 llvm::Value *FreeArgs[] = {Rec.second.GlobalizedVal,
1221 CGF.getTypeSize(Ty: VD->getType())};
1222 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1223 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free_shared),
1224 args: FreeArgs);
1225 }
1226 }
1227}
1228
1229void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF,
1230 const OMPExecutableDirective &D,
1231 SourceLocation Loc,
1232 llvm::Function *OutlinedFn,
1233 ArrayRef<llvm::Value *> CapturedVars) {
1234 if (!CGF.HaveInsertPoint())
1235 return;
1236
1237 bool IsBareKernel = D.getSingleClause<OMPXBareClause>();
1238
1239 RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(Ty: CGF.Int32Ty,
1240 /*Name=*/".zero.addr");
1241 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(/*C*/ 0), Addr: ZeroAddr);
1242 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
1243 // We don't emit any thread id function call in bare kernel, but because the
1244 // outlined function has a pointer argument, we emit a nullptr here.
1245 if (IsBareKernel)
1246 OutlinedFnArgs.push_back(Elt: llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy));
1247 else
1248 OutlinedFnArgs.push_back(Elt: emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF));
1249 OutlinedFnArgs.push_back(Elt: ZeroAddr.getPointer());
1250 OutlinedFnArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
1251 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, Args: OutlinedFnArgs);
1252}
1253
1254void CGOpenMPRuntimeGPU::emitParallelCall(
1255 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
1256 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
1257 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
1258 OpenMPSeverityClauseKind Severity, const Expr *Message) {
1259 if (!CGF.HaveInsertPoint())
1260 return;
1261
1262 auto &&ParallelGen = [this, Loc, OutlinedFn, CapturedVars, IfCond,
1263 NumThreads](CodeGenFunction &CGF,
1264 PrePostActionTy &Action) {
1265 CGBuilderTy &Bld = CGF.Builder;
1266 llvm::Value *NumThreadsVal = NumThreads;
1267 llvm::Function *WFn = WrapperFunctionsMap[OutlinedFn];
1268 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1269 C&: CGF.getLLVMContext(), AddressSpace: CGM.getDataLayout().getProgramAddressSpace());
1270
1271 llvm::Value *ID = llvm::ConstantPointerNull::get(T: FnPtrTy);
1272 if (WFn)
1273 ID = Bld.CreateBitOrPointerCast(V: WFn, DestTy: FnPtrTy);
1274
1275 llvm::Value *FnPtr = Bld.CreateBitOrPointerCast(V: OutlinedFn, DestTy: FnPtrTy);
1276
1277 // Create a private scope that will globalize the arguments
1278 // passed from the outside of the target region.
1279 // TODO: Is that needed?
1280 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF);
1281
1282 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca(
1283 Ty: llvm::ArrayType::get(ElementType: CGM.VoidPtrTy, NumElements: CapturedVars.size()),
1284 Name: "captured_vars_addrs");
1285 // There's something to share.
1286 if (!CapturedVars.empty()) {
1287 // Prepare for parallel region. Indicate the outlined function.
1288 ASTContext &Ctx = CGF.getContext();
1289 unsigned Idx = 0;
1290 for (llvm::Value *V : CapturedVars) {
1291 Address Dst = Bld.CreateConstArrayGEP(Addr: CapturedVarsAddrs, Index: Idx);
1292 llvm::Value *PtrV;
1293 if (V->getType()->isIntegerTy())
1294 PtrV = Bld.CreateIntToPtr(V, DestTy: CGF.VoidPtrTy);
1295 else
1296 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, DestTy: CGF.VoidPtrTy);
1297 CGF.EmitStoreOfScalar(Value: PtrV, Addr: Dst, /*Volatile=*/false,
1298 Ty: Ctx.getPointerType(T: Ctx.VoidPtrTy));
1299 ++Idx;
1300 }
1301 }
1302
1303 llvm::Value *IfCondVal = nullptr;
1304 if (IfCond)
1305 IfCondVal = Bld.CreateIntCast(V: CGF.EvaluateExprAsBool(E: IfCond), DestTy: CGF.Int32Ty,
1306 /* isSigned */ false);
1307 else
1308 IfCondVal = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 1);
1309
1310 if (!NumThreadsVal)
1311 NumThreadsVal = llvm::ConstantInt::getAllOnesValue(Ty: CGF.Int32Ty);
1312 else
1313 NumThreadsVal = Bld.CreateZExtOrTrunc(V: NumThreadsVal, DestTy: CGF.Int32Ty);
1314
1315 // No strict prescriptiveness for the number of threads.
1316 llvm::Value *StrictNumThreadsVal = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0);
1317
1318 assert(IfCondVal && "Expected a value");
1319 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1320 llvm::Value *Args[] = {
1321 RTLoc,
1322 getThreadID(CGF, Loc),
1323 IfCondVal,
1324 NumThreadsVal,
1325 llvm::ConstantInt::getAllOnesValue(Ty: CGF.Int32Ty),
1326 FnPtr,
1327 ID,
1328 Bld.CreateBitOrPointerCast(V: CapturedVarsAddrs.emitRawPointer(CGF),
1329 DestTy: CGF.VoidPtrPtrTy),
1330 llvm::ConstantInt::get(Ty: CGM.SizeTy, V: CapturedVars.size()),
1331 StrictNumThreadsVal};
1332
1333 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1334 M&: CGM.getModule(), FnID: OMPRTL___kmpc_parallel_60),
1335 args: Args);
1336 };
1337
1338 RegionCodeGenTy RCG(ParallelGen);
1339 RCG(CGF);
1340}
1341
1342void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) {
1343 // Always emit simple barriers!
1344 if (!CGF.HaveInsertPoint())
1345 return;
1346 // Build call __kmpc_barrier_simple_spmd(nullptr, 0);
1347 // This function does not use parameters, so we can emit just default values.
1348 llvm::Value *Args[] = {
1349 llvm::ConstantPointerNull::get(
1350 T: cast<llvm::PointerType>(Val: getIdentTyPointerTy())),
1351 llvm::ConstantInt::get(Ty: CGF.Int32Ty, /*V=*/0, /*isSigned=*/IsSigned: true)};
1352 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1353 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier_simple_spmd),
1354 args: Args);
1355}
1356
1357void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF,
1358 SourceLocation Loc,
1359 OpenMPDirectiveKind Kind, bool,
1360 bool) {
1361 // Always emit simple barriers!
1362 if (!CGF.HaveInsertPoint())
1363 return;
1364 // Build call __kmpc_cancel_barrier(loc, thread_id);
1365 unsigned Flags = getDefaultFlagsForBarriers(Kind);
1366 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
1367 getThreadID(CGF, Loc)};
1368
1369 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1370 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
1371 args: Args);
1372}
1373
1374void CGOpenMPRuntimeGPU::emitCriticalRegion(
1375 CodeGenFunction &CGF, StringRef CriticalName,
1376 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
1377 const Expr *Hint) {
1378 llvm::BasicBlock *LoopBB = CGF.createBasicBlock(name: "omp.critical.loop");
1379 llvm::BasicBlock *TestBB = CGF.createBasicBlock(name: "omp.critical.test");
1380 llvm::BasicBlock *SyncBB = CGF.createBasicBlock(name: "omp.critical.sync");
1381 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.critical.body");
1382 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "omp.critical.exit");
1383
1384 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1385
1386 // Get the mask of active threads in the warp.
1387 llvm::Value *Mask = CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1388 M&: CGM.getModule(), FnID: OMPRTL___kmpc_warp_active_thread_mask));
1389 // Fetch team-local id of the thread.
1390 llvm::Value *ThreadID = RT.getGPUThreadID(CGF);
1391
1392 // Get the width of the team.
1393 llvm::Value *TeamWidth = RT.getGPUNumThreads(CGF);
1394
1395 // Initialize the counter variable for the loop.
1396 QualType Int32Ty =
1397 CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/0);
1398 Address Counter = CGF.CreateMemTempWithoutCast(T: Int32Ty, Name: "critical_counter");
1399 LValue CounterLVal = CGF.MakeAddrLValue(Addr: Counter, T: Int32Ty);
1400 CGF.EmitStoreOfScalar(value: llvm::Constant::getNullValue(Ty: CGM.Int32Ty), lvalue: CounterLVal,
1401 /*isInit=*/true);
1402
1403 // Block checks if loop counter exceeds upper bound.
1404 CGF.EmitBlock(BB: LoopBB);
1405 llvm::Value *CounterVal = CGF.EmitLoadOfScalar(lvalue: CounterLVal, Loc);
1406 llvm::Value *CmpLoopBound = CGF.Builder.CreateICmpSLT(LHS: CounterVal, RHS: TeamWidth);
1407 CGF.Builder.CreateCondBr(Cond: CmpLoopBound, True: TestBB, False: ExitBB);
1408
1409 // Block tests which single thread should execute region, and which threads
1410 // should go straight to synchronisation point.
1411 CGF.EmitBlock(BB: TestBB);
1412 CounterVal = CGF.EmitLoadOfScalar(lvalue: CounterLVal, Loc);
1413 llvm::Value *CmpThreadToCounter =
1414 CGF.Builder.CreateICmpEQ(LHS: ThreadID, RHS: CounterVal);
1415 CGF.Builder.CreateCondBr(Cond: CmpThreadToCounter, True: BodyBB, False: SyncBB);
1416
1417 // Block emits the body of the critical region.
1418 CGF.EmitBlock(BB: BodyBB);
1419
1420 // Output the critical statement.
1421 CGOpenMPRuntime::emitCriticalRegion(CGF, CriticalName, CriticalOpGen, Loc,
1422 Hint);
1423
1424 // After the body surrounded by the critical region, the single executing
1425 // thread will jump to the synchronisation point.
1426 // Block waits for all threads in current team to finish then increments the
1427 // counter variable and returns to the loop.
1428 CGF.EmitBlock(BB: SyncBB);
1429 // Reconverge active threads in the warp.
1430 (void)CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1431 M&: CGM.getModule(), FnID: OMPRTL___kmpc_syncwarp),
1432 args: Mask);
1433
1434 llvm::Value *IncCounterVal =
1435 CGF.Builder.CreateNSWAdd(LHS: CounterVal, RHS: CGF.Builder.getInt32(C: 1));
1436 CGF.EmitStoreOfScalar(value: IncCounterVal, lvalue: CounterLVal);
1437 CGF.EmitBranch(Block: LoopBB);
1438
1439 // Block that is reached when all threads in the team complete the region.
1440 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1441}
1442
1443/// Cast value to the specified type.
1444static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val,
1445 QualType ValTy, QualType CastTy,
1446 SourceLocation Loc) {
1447 assert(!CGF.getContext().getTypeSizeInChars(CastTy).isZero() &&
1448 "Cast type must sized.");
1449 assert(!CGF.getContext().getTypeSizeInChars(ValTy).isZero() &&
1450 "Val type must sized.");
1451 llvm::Type *LLVMCastTy = CGF.ConvertTypeForMem(T: CastTy);
1452 if (ValTy == CastTy)
1453 return Val;
1454 if (CGF.getContext().getTypeSizeInChars(T: ValTy) ==
1455 CGF.getContext().getTypeSizeInChars(T: CastTy))
1456 return CGF.Builder.CreateBitCast(V: Val, DestTy: LLVMCastTy);
1457 if (CastTy->isIntegerType() && ValTy->isIntegerType())
1458 return CGF.Builder.CreateIntCast(V: Val, DestTy: LLVMCastTy,
1459 isSigned: CastTy->hasSignedIntegerRepresentation());
1460 Address CastItem = CGF.CreateMemTempWithoutCast(T: CastTy);
1461 Address ValCastItem = CastItem.withElementType(ElemTy: Val->getType());
1462 CGF.EmitStoreOfScalar(Value: Val, Addr: ValCastItem, /*Volatile=*/false, Ty: ValTy,
1463 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
1464 TBAAInfo: TBAAAccessInfo());
1465 return CGF.EmitLoadOfScalar(Addr: CastItem, /*Volatile=*/false, Ty: CastTy, Loc,
1466 BaseInfo: LValueBaseInfo(AlignmentSource::Type),
1467 TBAAInfo: TBAAAccessInfo());
1468}
1469
1470/// Extracts the built-in reduction operator from a combiner of the form `x = x
1471/// <op> rhs` (or the min/max conditional), or nullopt if the shape is not
1472/// recognized (e.g. user-defined reductions).
1473static std::optional<BinaryOperatorKind>
1474getReductionBinOpKind(const Expr *ReductionOp) {
1475 const auto *Assign = dyn_cast<BinaryOperator>(Val: ReductionOp);
1476 if (!Assign || Assign->getOpcode() != BO_Assign)
1477 return std::nullopt;
1478 const Expr *RHS = Assign->getRHS();
1479 // min/max are lowered as `x <cmp> rhs ? x : rhs`; the comparison identifies
1480 // it.
1481 if (const auto *ACO =
1482 dyn_cast<AbstractConditionalOperator>(Val: RHS->IgnoreParenImpCasts()))
1483 RHS = ACO->getCond();
1484 if (const auto *BO = dyn_cast<BinaryOperator>(Val: RHS->IgnoreParenImpCasts()))
1485 return BO->getOpcode();
1486 return std::nullopt;
1487}
1488
1489/// Maps a built-in reduction operator to an atomicrmw opcode for the atomic
1490/// cross-team reduction fast path, or nullopt if there is no direct atomicrmw
1491/// (e.g. user-defined, complex, fp min/max) so the buffer path is used instead.
1492static std::optional<llvm::AtomicRMWInst::BinOp>
1493getReductionAtomicRMWOp(BinaryOperatorKind BOK, QualType Ty) {
1494 bool IsInt = Ty->isIntegerType();
1495 bool IsSigned = Ty->hasSignedIntegerRepresentation();
1496 switch (BOK) {
1497 case BO_Add:
1498 case BO_Sub: // A `-` reduction sums the partials, so it accumulates with add.
1499 if (IsInt)
1500 return llvm::AtomicRMWInst::Add;
1501 if (Ty->isFloatingType())
1502 return llvm::AtomicRMWInst::FAdd;
1503 return std::nullopt;
1504 case BO_And:
1505 return IsInt ? std::optional(llvm::AtomicRMWInst::And) : std::nullopt;
1506 case BO_Or:
1507 return IsInt ? std::optional(llvm::AtomicRMWInst::Or) : std::nullopt;
1508 case BO_Xor:
1509 return IsInt ? std::optional(llvm::AtomicRMWInst::Xor) : std::nullopt;
1510 case BO_LT: // min
1511 if (IsInt)
1512 return IsSigned ? llvm::AtomicRMWInst::Min : llvm::AtomicRMWInst::UMin;
1513 return std::nullopt;
1514 case BO_GT: // max
1515 if (IsInt)
1516 return IsSigned ? llvm::AtomicRMWInst::Max : llvm::AtomicRMWInst::UMax;
1517 return std::nullopt;
1518 default:
1519 return std::nullopt;
1520 }
1521}
1522
1523///
1524/// Design of OpenMP reductions on the GPU
1525///
1526/// Consider a typical OpenMP program with one or more reduction
1527/// clauses:
1528///
1529/// float foo;
1530/// double bar;
1531/// #pragma omp target teams distribute parallel for \
1532/// reduction(+:foo) reduction(*:bar)
1533/// for (int i = 0; i < N; i++) {
1534/// foo += A[i]; bar *= B[i];
1535/// }
1536///
1537/// where 'foo' and 'bar' are reduced across all OpenMP threads in
1538/// all teams. In our OpenMP implementation on the NVPTX device an
1539/// OpenMP team is mapped to a CUDA threadblock and OpenMP threads
1540/// within a team are mapped to CUDA threads within a threadblock.
1541/// Our goal is to efficiently aggregate values across all OpenMP
1542/// threads such that:
1543///
1544/// - the compiler and runtime are logically concise, and
1545/// - the reduction is performed efficiently in a hierarchical
1546/// manner as follows: within OpenMP threads in the same warp,
1547/// across warps in a threadblock, and finally across teams on
1548/// the NVPTX device.
1549///
1550/// Introduction to Decoupling
1551///
1552/// We would like to decouple the compiler and the runtime so that the
1553/// latter is ignorant of the reduction variables (number, data types)
1554/// and the reduction operators. This allows a simpler interface
1555/// and implementation while still attaining good performance.
1556///
1557/// Pseudocode for the aforementioned OpenMP program generated by the
1558/// compiler is as follows:
1559///
1560/// 1. Create private copies of reduction variables on each OpenMP
1561/// thread: 'foo_private', 'bar_private'
1562/// 2. Each OpenMP thread reduces the chunk of 'A' and 'B' assigned
1563/// to it and writes the result in 'foo_private' and 'bar_private'
1564/// respectively.
1565/// 3. Call the OpenMP runtime on the GPU to reduce within a team
1566/// and store the result on the team master:
1567///
1568/// __kmpc_nvptx_parallel_reduce_nowait_v2(...,
1569/// reduceData, shuffleReduceFn, interWarpCpyFn)
1570///
1571/// where:
1572/// struct ReduceData {
1573/// double *foo;
1574/// double *bar;
1575/// } reduceData
1576/// reduceData.foo = &foo_private
1577/// reduceData.bar = &bar_private
1578///
1579/// 'shuffleReduceFn' and 'interWarpCpyFn' are pointers to two
1580/// auxiliary functions generated by the compiler that operate on
1581/// variables of type 'ReduceData'. They aid the runtime perform
1582/// algorithmic steps in a data agnostic manner.
1583///
1584/// 'shuffleReduceFn' is a pointer to a function that reduces data
1585/// of type 'ReduceData' across two OpenMP threads (lanes) in the
1586/// same warp. It takes the following arguments as input:
1587///
1588/// a. variable of type 'ReduceData' on the calling lane,
1589/// b. its lane_id,
1590/// c. an offset relative to the current lane_id to generate a
1591/// remote_lane_id. The remote lane contains the second
1592/// variable of type 'ReduceData' that is to be reduced.
1593/// d. an algorithm version parameter determining which reduction
1594/// algorithm to use.
1595///
1596/// 'shuffleReduceFn' retrieves data from the remote lane using
1597/// efficient GPU shuffle intrinsics and reduces, using the
1598/// algorithm specified by the 4th parameter, the two operands
1599/// element-wise. The result is written to the first operand.
1600///
1601/// Different reduction algorithms are implemented in different
1602/// runtime functions, all calling 'shuffleReduceFn' to perform
1603/// the essential reduction step. Therefore, based on the 4th
1604/// parameter, this function behaves slightly differently to
1605/// cooperate with the runtime to ensure correctness under
1606/// different circumstances.
1607///
1608/// 'InterWarpCpyFn' is a pointer to a function that transfers
1609/// reduced variables across warps. It tunnels, through CUDA
1610/// shared memory, the thread-private data of type 'ReduceData'
1611/// from lane 0 of each warp to a lane in the first warp.
1612/// 4. Call the OpenMP runtime on the GPU to reduce across teams.
1613/// The last team writes the global reduced value to memory.
1614///
1615/// ret = __kmpc_nvptx_teams_reduce_nowait(...,
1616/// reduceData, shuffleReduceFn, interWarpCpyFn,
1617/// scratchpadCopyFn, loadAndReduceFn)
1618///
1619/// 'scratchpadCopyFn' is a helper that stores reduced
1620/// data from the team master to a scratchpad array in
1621/// global memory.
1622///
1623/// 'loadAndReduceFn' is a helper that loads data from
1624/// the scratchpad array and reduces it with the input
1625/// operand.
1626///
1627/// These compiler generated functions hide address
1628/// calculation and alignment information from the runtime.
1629/// 5. if ret == 1:
1630/// The team master of the last team stores the reduced
1631/// result to the globals in memory.
1632/// foo += reduceData.foo; bar *= reduceData.bar
1633///
1634///
1635/// Warp Reduction Algorithms
1636///
1637/// On the warp level, we have three algorithms implemented in the
1638/// OpenMP runtime depending on the number of active lanes:
1639///
1640/// Full Warp Reduction
1641///
1642/// The reduce algorithm within a warp where all lanes are active
1643/// is implemented in the runtime as follows:
1644///
1645/// full_warp_reduce(void *reduce_data,
1646/// kmp_ShuffleReductFctPtr ShuffleReduceFn) {
1647/// for (int offset = WARPSIZE/2; offset > 0; offset /= 2)
1648/// ShuffleReduceFn(reduce_data, 0, offset, 0);
1649/// }
1650///
1651/// The algorithm completes in log(2, WARPSIZE) steps.
1652///
1653/// 'ShuffleReduceFn' is used here with lane_id set to 0 because it is
1654/// not used therefore we save instructions by not retrieving lane_id
1655/// from the corresponding special registers. The 4th parameter, which
1656/// represents the version of the algorithm being used, is set to 0 to
1657/// signify full warp reduction.
1658///
1659/// In this version, 'ShuffleReduceFn' behaves, per element, as follows:
1660///
1661/// #reduce_elem refers to an element in the local lane's data structure
1662/// #remote_elem is retrieved from a remote lane
1663/// remote_elem = shuffle_down(reduce_elem, offset, WARPSIZE);
1664/// reduce_elem = reduce_elem REDUCE_OP remote_elem;
1665///
1666/// Contiguous Partial Warp Reduction
1667///
1668/// This reduce algorithm is used within a warp where only the first
1669/// 'n' (n <= WARPSIZE) lanes are active. It is typically used when the
1670/// number of OpenMP threads in a parallel region is not a multiple of
1671/// WARPSIZE. The algorithm is implemented in the runtime as follows:
1672///
1673/// void
1674/// contiguous_partial_reduce(void *reduce_data,
1675/// kmp_ShuffleReductFctPtr ShuffleReduceFn,
1676/// int size, int lane_id) {
1677/// int curr_size;
1678/// int offset;
1679/// curr_size = size;
1680/// mask = curr_size/2;
1681/// while (offset>0) {
1682/// ShuffleReduceFn(reduce_data, lane_id, offset, 1);
1683/// curr_size = (curr_size+1)/2;
1684/// offset = curr_size/2;
1685/// }
1686/// }
1687///
1688/// In this version, 'ShuffleReduceFn' behaves, per element, as follows:
1689///
1690/// remote_elem = shuffle_down(reduce_elem, offset, WARPSIZE);
1691/// if (lane_id < offset)
1692/// reduce_elem = reduce_elem REDUCE_OP remote_elem
1693/// else
1694/// reduce_elem = remote_elem
1695///
1696/// This algorithm assumes that the data to be reduced are located in a
1697/// contiguous subset of lanes starting from the first. When there is
1698/// an odd number of active lanes, the data in the last lane is not
1699/// aggregated with any other lane's dat but is instead copied over.
1700///
1701/// Dispersed Partial Warp Reduction
1702///
1703/// This algorithm is used within a warp when any discontiguous subset of
1704/// lanes are active. It is used to implement the reduction operation
1705/// across lanes in an OpenMP simd region or in a nested parallel region.
1706///
1707/// void
1708/// dispersed_partial_reduce(void *reduce_data,
1709/// kmp_ShuffleReductFctPtr ShuffleReduceFn) {
1710/// int size, remote_id;
1711/// int logical_lane_id = number_of_active_lanes_before_me() * 2;
1712/// do {
1713/// remote_id = next_active_lane_id_right_after_me();
1714/// # the above function returns 0 of no active lane
1715/// # is present right after the current lane.
1716/// size = number_of_active_lanes_in_this_warp();
1717/// logical_lane_id /= 2;
1718/// ShuffleReduceFn(reduce_data, logical_lane_id,
1719/// remote_id-1-threadIdx.x, 2);
1720/// } while (logical_lane_id % 2 == 0 && size > 1);
1721/// }
1722///
1723/// There is no assumption made about the initial state of the reduction.
1724/// Any number of lanes (>=1) could be active at any position. The reduction
1725/// result is returned in the first active lane.
1726///
1727/// In this version, 'ShuffleReduceFn' behaves, per element, as follows:
1728///
1729/// remote_elem = shuffle_down(reduce_elem, offset, WARPSIZE);
1730/// if (lane_id % 2 == 0 && offset > 0)
1731/// reduce_elem = reduce_elem REDUCE_OP remote_elem
1732/// else
1733/// reduce_elem = remote_elem
1734///
1735///
1736/// Intra-Team Reduction
1737///
1738/// This function, as implemented in the runtime call
1739/// '__kmpc_nvptx_parallel_reduce_nowait_v2', aggregates data across OpenMP
1740/// threads in a team. It first reduces within a warp using the
1741/// aforementioned algorithms. We then proceed to gather all such
1742/// reduced values at the first warp.
1743///
1744/// The runtime makes use of the function 'InterWarpCpyFn', which copies
1745/// data from each of the "warp master" (zeroth lane of each warp, where
1746/// warp-reduced data is held) to the zeroth warp. This step reduces (in
1747/// a mathematical sense) the problem of reduction across warp masters in
1748/// a block to the problem of warp reduction.
1749///
1750///
1751/// Inter-Team Reduction
1752///
1753/// Once a team has reduced its data to a single value, it is stored in
1754/// a global scratchpad array. Since each team has a distinct slot, this
1755/// can be done without locking.
1756///
1757/// The last team to write to the scratchpad array proceeds to reduce the
1758/// scratchpad array. One or more workers in the last team use the helper
1759/// 'loadAndReduceDataFn' to load and reduce values from the array, i.e.,
1760/// the k'th worker reduces every k'th element.
1761///
1762/// Finally, a call is made to '__kmpc_nvptx_parallel_reduce_nowait_v2' to
1763/// reduce across workers and compute a globally reduced value.
1764///
1765void CGOpenMPRuntimeGPU::emitReduction(
1766 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
1767 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
1768 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
1769 if (!CGF.HaveInsertPoint())
1770 return;
1771
1772 bool ParallelReduction = isOpenMPParallelDirective(DKind: Options.ReductionKind);
1773 bool TeamsReduction = isOpenMPTeamsDirective(DKind: Options.ReductionKind);
1774
1775 if (Options.SimpleReduction) {
1776 assert(!TeamsReduction && !ParallelReduction &&
1777 "Invalid reduction selection in emitReduction.");
1778 (void)ParallelReduction;
1779 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
1780 ReductionOps, Options);
1781 return;
1782 }
1783
1784 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> VarFieldMap;
1785 llvm::SmallVector<const ValueDecl *, 4> PrivatesReductions(Privates.size());
1786 int Cnt = 0;
1787 for (const Expr *DRE : Privates) {
1788 PrivatesReductions[Cnt] = cast<DeclRefExpr>(Val: DRE)->getDecl();
1789 ++Cnt;
1790 }
1791 const RecordDecl *ReductionRec = ::buildRecordForGlobalizedVars(
1792 C&: CGM.getContext(), EscapedDecls: PrivatesReductions, EscapedDeclsForTeams: {}, MappedDeclsFields&: VarFieldMap, BufSize: 1);
1793
1794 // The atomic cross-team reduction fast path is opt-in. Hand each eligible
1795 // scalar reduction an atomic combiner; createReductionsGPU uses the atomic
1796 // path only if every reduction in the set has one. Track whether that holds
1797 // so we can skip the (then unused) per-team buffer registration.
1798 bool UseAtomicReduction =
1799 TeamsReduction && CGM.getLangOpts().OpenMPTargetAtomicReduction;
1800 bool AllAtomicable = UseAtomicReduction;
1801
1802 // Source location for the ident struct
1803 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1804
1805 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1806 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
1807 InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
1808 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(
1809 CodeGenIP, CGF.SourceLocToDebugLoc(Location: Loc));
1810 llvm::SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> ReductionInfos;
1811
1812 CodeGenFunction::OMPPrivateScope Scope(CGF);
1813 unsigned Idx = 0;
1814 for (const Expr *Private : Privates) {
1815 llvm::Type *ElementType;
1816 llvm::Value *Variable;
1817 llvm::Value *PrivateVariable;
1818 llvm::OpenMPIRBuilder::ReductionGenAtomicCBTy AtomicReductionGen = nullptr;
1819 ElementType = CGF.ConvertTypeForMem(T: Private->getType());
1820 const auto *RHSVar =
1821 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSExprs[Idx])->getDecl());
1822 PrivateVariable = CGF.GetAddrOfLocalVar(VD: RHSVar).emitRawPointer(CGF);
1823 const auto *LHSVar =
1824 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSExprs[Idx])->getDecl());
1825 Variable = CGF.GetAddrOfLocalVar(VD: LHSVar).emitRawPointer(CGF);
1826 llvm::OpenMPIRBuilder::EvalKind EvalKind;
1827 switch (CGF.getEvaluationKind(T: Private->getType())) {
1828 case TEK_Scalar:
1829 EvalKind = llvm::OpenMPIRBuilder::EvalKind::Scalar;
1830 break;
1831 case TEK_Complex:
1832 EvalKind = llvm::OpenMPIRBuilder::EvalKind::Complex;
1833 break;
1834 case TEK_Aggregate:
1835 EvalKind = llvm::OpenMPIRBuilder::EvalKind::Aggregate;
1836 break;
1837 }
1838 auto ReductionGen = [&](InsertPointTy CodeGenIP, unsigned I,
1839 llvm::Value **LHSPtr, llvm::Value **RHSPtr,
1840 llvm::Function *NewFunc) {
1841 CGF.Builder.restoreIP(IP: CodeGenIP);
1842 auto *CurFn = CGF.CurFn;
1843 CGF.CurFn = NewFunc;
1844
1845 const bool DisableDebugInfo = !NewFunc->getSubprogram();
1846 llvm::DebugLoc SavedDebugLoc;
1847 if (DisableDebugInfo) {
1848 // The helper has no DISubprogram of its own, so a debug location here
1849 // would name the enclosing function's scope, which is invalid IR.
1850 // Suppress them, as the other OpenMPIRBuilder-generated helpers do.
1851 SavedDebugLoc = CGF.Builder.getCurrentDebugLocation();
1852 CGF.Builder.SetCurrentDebugLocation({});
1853 CGF.disableDebugInfo();
1854 }
1855
1856 *LHSPtr = CGF.GetAddrOfLocalVar(
1857 VD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSExprs[I])->getDecl()))
1858 .emitRawPointer(CGF);
1859 *RHSPtr = CGF.GetAddrOfLocalVar(
1860 VD: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSExprs[I])->getDecl()))
1861 .emitRawPointer(CGF);
1862
1863 emitSingleReductionCombiner(CGF, ReductionOp: ReductionOps[I], PrivateRef: Privates[I],
1864 LHS: cast<DeclRefExpr>(Val: LHSExprs[I]),
1865 RHS: cast<DeclRefExpr>(Val: RHSExprs[I]));
1866
1867 if (DisableDebugInfo) {
1868 CGF.enableDebugInfo();
1869 CGF.Builder.SetCurrentDebugLocation(SavedDebugLoc);
1870 }
1871
1872 CGF.CurFn = CurFn;
1873 return CGF.Builder.GetInsertPoint();
1874 };
1875
1876 // For the atomic fast path, hand this reduction an atomic combiner if it is
1877 // a scalar with a direct atomicrmw; otherwise the set is not fully
1878 // atomicable and falls back to the buffer path.
1879 if (UseAtomicReduction) {
1880 std::optional<llvm::AtomicRMWInst::BinOp> AtomicOp;
1881 if (EvalKind == llvm::OpenMPIRBuilder::EvalKind::Scalar) {
1882 if (std::optional<BinaryOperatorKind> BOK =
1883 getReductionBinOpKind(ReductionOp: ReductionOps[Idx]))
1884 AtomicOp = getReductionAtomicRMWOp(BOK: *BOK, Ty: Private->getType());
1885 }
1886 if (!AtomicOp) {
1887 AllAtomicable = false;
1888 } else {
1889 llvm::AtomicRMWInst::BinOp Op = *AtomicOp;
1890 llvm::Align Alignment =
1891 CGM.getModule().getDataLayout().getPrefTypeAlign(Ty: ElementType);
1892 // Device (agent) scope suffices: all teams accumulate on-device and the
1893 // host reads the result only after the kernel (via map-back), so the
1894 // far costlier system scope is unnecessary. The
1895 // no.fine.grained/no.remote memory metadata is omitted so the atomic
1896 // stays correct under USM.
1897 llvm::SyncScope::ID SSID = CGF.getTargetHooks().getLLVMSyncScopeID(
1898 LangOpts: CGF.getLangOpts(), Scope: SyncScope::DeviceScope,
1899 Ordering: llvm::AtomicOrdering::Monotonic, Ctx&: CGF.getLLVMContext());
1900 AtomicReductionGen = [Op, Alignment,
1901 SSID](InsertPointTy IP, llvm::Type *EltTy,
1902 llvm::Value *LHS, llvm::Value *RHS)
1903 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
1904 llvm::IRBuilder<> Builder(IP);
1905 llvm::Value *Val = Builder.CreateLoad(Ty: EltTy, Ptr: RHS);
1906 Builder.CreateAtomicRMW(Op, Ptr: LHS, Val, Align: Alignment,
1907 Ordering: llvm::AtomicOrdering::Monotonic, SSID);
1908 return Builder.GetInsertPoint();
1909 };
1910 }
1911 }
1912
1913 ReductionInfos.emplace_back(Args: llvm::OpenMPIRBuilder::ReductionInfo(
1914 ElementType, Variable, PrivateVariable, EvalKind,
1915 /*ReductionGen=*/nullptr, ReductionGen, AtomicReductionGen,
1916 /*DataPtrPtrGen=*/nullptr));
1917 Idx++;
1918 }
1919
1920 // The atomic path folds directly into the mapped variable and needs no
1921 // per-team buffer; register the record for buffer allocation otherwise.
1922 if (TeamsReduction && !AllAtomicable)
1923 TeamsReductions.push_back(Elt: ReductionRec);
1924
1925 bool IsSPMD = getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD;
1926 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
1927 cantFail(ValOrErr: OMPBuilder.createReductionsGPU(
1928 Loc: OmpLoc, AllocaIP, CodeGenIP, ReductionInfos, /*IsByRef=*/{}, IsNoWait: false,
1929 IsTeamsReduction: TeamsReduction, IsSPMD,
1930 ReductionGenCBKind: llvm::OpenMPIRBuilder::ReductionGenCBKind::Clang,
1931 GridValue: CGF.getTarget().getGridValue(), SrcLocInfo: RTLoc));
1932 CGF.Builder.restoreIP(IP: AfterIP);
1933}
1934
1935const VarDecl *
1936CGOpenMPRuntimeGPU::translateParameter(const FieldDecl *FD,
1937 const VarDecl *NativeParam) const {
1938 if (!NativeParam->getType()->isReferenceType())
1939 return NativeParam;
1940 QualType ArgType = NativeParam->getType();
1941 QualifierCollector QC;
1942 const Type *NonQualTy = QC.strip(type: ArgType);
1943 QualType PointeeTy = cast<ReferenceType>(Val: NonQualTy)->getPointeeType();
1944 if (const auto *Attr = FD->getAttr<OMPCaptureKindAttr>()) {
1945 if (Attr->getCaptureKind() == OMPC_map) {
1946 PointeeTy = CGM.getContext().getAddrSpaceQualType(T: PointeeTy,
1947 AddressSpace: LangAS::opencl_global);
1948 }
1949 }
1950 ArgType = CGM.getContext().getPointerType(T: PointeeTy);
1951 QC.addRestrict();
1952 ArgType = QC.apply(Context: CGM.getContext(), QT: ArgType);
1953 if (isa<ImplicitParamDecl>(Val: NativeParam))
1954 return ImplicitParamDecl::Create(
1955 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: NativeParam->getLocation(),
1956 Id: NativeParam->getIdentifier(), T: ArgType, ParamKind: ImplicitParamKind::Other);
1957 return ParmVarDecl::Create(
1958 C&: CGM.getContext(),
1959 DC: const_cast<DeclContext *>(NativeParam->getDeclContext()),
1960 StartLoc: NativeParam->getBeginLoc(), IdLoc: NativeParam->getLocation(),
1961 Id: NativeParam->getIdentifier(), T: ArgType,
1962 /*TInfo=*/nullptr, S: SC_None, /*DefArg=*/nullptr);
1963}
1964
1965Address
1966CGOpenMPRuntimeGPU::getParameterAddress(CodeGenFunction &CGF,
1967 const VarDecl *NativeParam,
1968 const VarDecl *TargetParam) const {
1969 assert(NativeParam != TargetParam &&
1970 NativeParam->getType()->isReferenceType() &&
1971 "Native arg must not be the same as target arg.");
1972 Address LocalAddr = CGF.GetAddrOfLocalVar(VD: TargetParam);
1973 QualType NativeParamType = NativeParam->getType();
1974 QualifierCollector QC;
1975 const Type *NonQualTy = QC.strip(type: NativeParamType);
1976 QualType NativePointeeTy = cast<ReferenceType>(Val: NonQualTy)->getPointeeType();
1977 unsigned NativePointeeAddrSpace =
1978 CGF.getTypes().getTargetAddressSpace(T: NativePointeeTy);
1979 QualType TargetTy = TargetParam->getType();
1980 llvm::Value *TargetAddr = CGF.EmitLoadOfScalar(Addr: LocalAddr, /*Volatile=*/false,
1981 Ty: TargetTy, Loc: SourceLocation());
1982 // Cast to native address space.
1983 TargetAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1984 V: TargetAddr,
1985 DestTy: llvm::PointerType::get(C&: CGF.getLLVMContext(), AddressSpace: NativePointeeAddrSpace));
1986 Address NativeParamAddr = CGF.CreateMemTemp(T: NativeParamType);
1987 CGF.EmitStoreOfScalar(Value: TargetAddr, Addr: NativeParamAddr, /*Volatile=*/false,
1988 Ty: NativeParamType);
1989 return NativeParamAddr;
1990}
1991
1992void CGOpenMPRuntimeGPU::emitOutlinedFunctionCall(
1993 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
1994 ArrayRef<llvm::Value *> Args) const {
1995 SmallVector<llvm::Value *, 4> TargetArgs;
1996 TargetArgs.reserve(N: Args.size());
1997 auto *FnType = OutlinedFn.getFunctionType();
1998 for (unsigned I = 0, E = Args.size(); I < E; ++I) {
1999 if (FnType->isVarArg() && FnType->getNumParams() <= I) {
2000 TargetArgs.append(in_start: std::next(x: Args.begin(), n: I), in_end: Args.end());
2001 break;
2002 }
2003 llvm::Type *TargetType = FnType->getParamType(i: I);
2004 llvm::Value *NativeArg = Args[I];
2005 if (!TargetType->isPointerTy()) {
2006 TargetArgs.emplace_back(Args&: NativeArg);
2007 continue;
2008 }
2009 TargetArgs.emplace_back(
2010 Args: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: NativeArg, DestTy: TargetType));
2011 }
2012 CGOpenMPRuntime::emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, Args: TargetArgs);
2013}
2014
2015/// Emit function which wraps the outline parallel region
2016/// and controls the arguments which are passed to this function.
2017/// The wrapper ensures that the outlined function is called
2018/// with the correct arguments when data is shared.
2019llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper(
2020 llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D) {
2021 ASTContext &Ctx = CGM.getContext();
2022 const auto &CS = *D.getCapturedStmt(RegionKind: OMPD_parallel);
2023
2024 // Create a function that takes as argument the source thread.
2025 FunctionArgList WrapperArgs;
2026 QualType Int16QTy =
2027 Ctx.getIntTypeForBitwidth(/*DestWidth=*/16, /*Signed=*/false);
2028 QualType Int32QTy =
2029 Ctx.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false);
2030 auto *ParallelLevelArg = ImplicitParamDecl::Create(
2031 C&: Ctx, /*DC=*/nullptr, IdLoc: D.getBeginLoc(),
2032 /*Id=*/nullptr, T: Int16QTy, ParamKind: ImplicitParamKind::Other);
2033 auto *WrapperArg = ImplicitParamDecl::Create(
2034 C&: Ctx, /*DC=*/nullptr, IdLoc: D.getBeginLoc(),
2035 /*Id=*/nullptr, T: Int32QTy, ParamKind: ImplicitParamKind::Other);
2036 WrapperArgs.emplace_back(Args&: ParallelLevelArg);
2037 WrapperArgs.emplace_back(Args&: WrapperArg);
2038
2039 const CGFunctionInfo &CGFI =
2040 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: WrapperArgs);
2041
2042 auto *Fn = llvm::Function::Create(
2043 Ty: CGM.getTypes().GetFunctionType(Info: CGFI), Linkage: llvm::GlobalValue::InternalLinkage,
2044 N: Twine(OutlinedParallelFn->getName(), "_wrapper"), M: &CGM.getModule());
2045
2046 // Ensure we do not inline the function. This is trivially true for the ones
2047 // passed to __kmpc_fork_call but the ones calles in serialized regions
2048 // could be inlined. This is not a perfect but it is closer to the invariant
2049 // we want, namely, every data environment starts with a new function.
2050 // TODO: We should pass the if condition to the runtime function and do the
2051 // handling there. Much cleaner code.
2052 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
2053
2054 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
2055 Fn->setLinkage(llvm::GlobalValue::InternalLinkage);
2056
2057 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
2058 CGF.StartFunction(GD: GlobalDecl(), RetTy: Ctx.VoidTy, Fn, FnInfo: CGFI, Args: WrapperArgs,
2059 Loc: D.getBeginLoc(), StartLoc: D.getBeginLoc());
2060
2061 const auto *RD = CS.getCapturedRecordDecl();
2062 auto CurField = RD->field_begin();
2063
2064 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(Ty: CGF.Int32Ty,
2065 /*Name=*/".zero.addr");
2066 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(/*C*/ 0), Addr: ZeroAddr);
2067 // Get the array of arguments.
2068 SmallVector<llvm::Value *, 8> Args;
2069
2070 Args.emplace_back(Args: CGF.GetAddrOfLocalVar(VD: WrapperArg).emitRawPointer(CGF));
2071 Args.emplace_back(Args: ZeroAddr.emitRawPointer(CGF));
2072
2073 CGBuilderTy &Bld = CGF.Builder;
2074 auto CI = CS.capture_begin();
2075
2076 // Use global memory for data sharing.
2077 // Handle passing of global args to workers.
2078 RawAddress GlobalArgs =
2079 CGF.CreateDefaultAlignTempAlloca(Ty: CGF.VoidPtrPtrTy, Name: "global_args");
2080 llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer();
2081 llvm::Value *DataSharingArgs[] = {GlobalArgsPtr};
2082 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2083 M&: CGM.getModule(), FnID: OMPRTL___kmpc_get_shared_variables),
2084 args: DataSharingArgs);
2085
2086 // Retrieve the shared variables from the list of references returned
2087 // by the runtime. Pass the variables to the outlined function.
2088 Address SharedArgListAddress = Address::invalid();
2089 if (CS.capture_size() > 0 ||
2090 isOpenMPLoopBoundSharingDirective(Kind: D.getDirectiveKind())) {
2091 SharedArgListAddress = CGF.EmitLoadOfPointer(
2092 Ptr: GlobalArgs, PtrTy: CGF.getContext()
2093 .getPointerType(T: CGF.getContext().VoidPtrTy)
2094 .castAs<PointerType>());
2095 }
2096 unsigned Idx = 0;
2097 if (isOpenMPLoopBoundSharingDirective(Kind: D.getDirectiveKind())) {
2098 Address Src = Bld.CreateConstInBoundsGEP(Addr: SharedArgListAddress, Index: Idx);
2099 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
2100 Addr: Src, Ty: Bld.getPtrTy(AddrSpace: 0), ElementTy: CGF.SizeTy);
2101 llvm::Value *LB = CGF.EmitLoadOfScalar(
2102 Addr: TypedAddress,
2103 /*Volatile=*/false,
2104 Ty: CGF.getContext().getPointerType(T: CGF.getContext().getSizeType()),
2105 Loc: cast<OMPLoopDirective>(Val: D).getLowerBoundVariable()->getExprLoc());
2106 Args.emplace_back(Args&: LB);
2107 ++Idx;
2108 Src = Bld.CreateConstInBoundsGEP(Addr: SharedArgListAddress, Index: Idx);
2109 TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(Addr: Src, Ty: Bld.getPtrTy(AddrSpace: 0),
2110 ElementTy: CGF.SizeTy);
2111 llvm::Value *UB = CGF.EmitLoadOfScalar(
2112 Addr: TypedAddress,
2113 /*Volatile=*/false,
2114 Ty: CGF.getContext().getPointerType(T: CGF.getContext().getSizeType()),
2115 Loc: cast<OMPLoopDirective>(Val: D).getUpperBoundVariable()->getExprLoc());
2116 Args.emplace_back(Args&: UB);
2117 ++Idx;
2118 }
2119 if (CS.capture_size() > 0) {
2120 ASTContext &CGFContext = CGF.getContext();
2121 for (unsigned I = 0, E = CS.capture_size(); I < E; ++I, ++CI, ++CurField) {
2122 QualType ElemTy = CurField->getType();
2123 Address Src = Bld.CreateConstInBoundsGEP(Addr: SharedArgListAddress, Index: I + Idx);
2124 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
2125 Addr: Src, Ty: CGF.ConvertTypeForMem(T: CGFContext.getPointerType(T: ElemTy)),
2126 ElementTy: CGF.ConvertTypeForMem(T: ElemTy));
2127 llvm::Value *Arg = CGF.EmitLoadOfScalar(Addr: TypedAddress,
2128 /*Volatile=*/false,
2129 Ty: CGFContext.getPointerType(T: ElemTy),
2130 Loc: CI->getLocation());
2131 if (CI->capturesVariableByCopy() &&
2132 !CI->getCapturedVar()->getType()->isAnyPointerType()) {
2133 Arg = castValueToType(CGF, Val: Arg, ValTy: ElemTy, CastTy: CGFContext.getUIntPtrType(),
2134 Loc: CI->getLocation());
2135 }
2136 Args.emplace_back(Args&: Arg);
2137 }
2138 }
2139
2140 emitOutlinedFunctionCall(CGF, Loc: D.getBeginLoc(), OutlinedFn: OutlinedParallelFn, Args);
2141 CGF.FinishFunction();
2142 return Fn;
2143}
2144
2145void CGOpenMPRuntimeGPU::emitFunctionProlog(CodeGenFunction &CGF,
2146 const Decl *D) {
2147 if (getDataSharingMode() != CGOpenMPRuntimeGPU::DS_Generic)
2148 return;
2149
2150 assert(D && "Expected function or captured|block decl.");
2151 assert(FunctionGlobalizedDecls.count(CGF.CurFn) == 0 &&
2152 "Function is registered already.");
2153 assert((!TeamAndReductions.first || TeamAndReductions.first == D) &&
2154 "Team is set but not processed.");
2155 const Stmt *Body = nullptr;
2156 bool NeedToDelayGlobalization = false;
2157 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
2158 Body = FD->getBody();
2159 } else if (const auto *BD = dyn_cast<BlockDecl>(Val: D)) {
2160 Body = BD->getBody();
2161 } else if (const auto *CD = dyn_cast<CapturedDecl>(Val: D)) {
2162 Body = CD->getBody();
2163 NeedToDelayGlobalization = CGF.CapturedStmtInfo->getKind() == CR_OpenMP;
2164 if (NeedToDelayGlobalization &&
2165 getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD)
2166 return;
2167 }
2168 if (!Body)
2169 return;
2170 CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second);
2171 VarChecker.Visit(S: Body);
2172 const RecordDecl *GlobalizedVarsRecord =
2173 VarChecker.getGlobalizedRecord(IsInTTDRegion);
2174 TeamAndReductions.first = nullptr;
2175 TeamAndReductions.second.clear();
2176 ArrayRef<const ValueDecl *> EscapedVariableLengthDecls =
2177 VarChecker.getEscapedVariableLengthDecls();
2178 ArrayRef<const ValueDecl *> DelayedVariableLengthDecls =
2179 VarChecker.getDelayedVariableLengthDecls();
2180 if (!GlobalizedVarsRecord && EscapedVariableLengthDecls.empty() &&
2181 DelayedVariableLengthDecls.empty())
2182 return;
2183 auto I = FunctionGlobalizedDecls.try_emplace(Key: CGF.CurFn).first;
2184 I->getSecond().MappedParams =
2185 std::make_unique<CodeGenFunction::OMPMapVars>();
2186 I->getSecond().EscapedParameters.insert(
2187 I: VarChecker.getEscapedParameters().begin(),
2188 E: VarChecker.getEscapedParameters().end());
2189 I->getSecond().EscapedVariableLengthDecls.append(
2190 in_start: EscapedVariableLengthDecls.begin(), in_end: EscapedVariableLengthDecls.end());
2191 I->getSecond().DelayedVariableLengthDecls.append(
2192 in_start: DelayedVariableLengthDecls.begin(), in_end: DelayedVariableLengthDecls.end());
2193 DeclToAddrMapTy &Data = I->getSecond().LocalVarData;
2194 for (const ValueDecl *VD : VarChecker.getEscapedDecls()) {
2195 assert(VD->isCanonicalDecl() && "Expected canonical declaration");
2196 Data.try_emplace(Key: VD);
2197 }
2198 if (!NeedToDelayGlobalization) {
2199 emitGenericVarsProlog(CGF, Loc: D->getBeginLoc());
2200 struct GlobalizationScope final : EHScopeStack::Cleanup {
2201 GlobalizationScope() = default;
2202
2203 void Emit(CodeGenFunction &CGF, Flags flags) override {
2204 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime())
2205 .emitGenericVarsEpilog(CGF);
2206 }
2207 };
2208 CGF.EHStack.pushCleanup<GlobalizationScope>(Kind: NormalAndEHCleanup);
2209 }
2210}
2211
2212Address CGOpenMPRuntimeGPU::getAddressOfLocalVariable(CodeGenFunction &CGF,
2213 const VarDecl *VD) {
2214 if (VD && VD->hasAttr<OMPAllocateDeclAttr>()) {
2215 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
2216 auto AS = LangAS::Default;
2217 switch (A->getAllocatorType()) {
2218 case OMPAllocateDeclAttr::OMPNullMemAlloc:
2219 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
2220 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
2221 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
2222 break;
2223 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
2224 return Address::invalid();
2225 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
2226 // TODO: implement aupport for user-defined allocators.
2227 return Address::invalid();
2228 case OMPAllocateDeclAttr::OMPConstMemAlloc:
2229 AS = LangAS::cuda_constant;
2230 break;
2231 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
2232 AS = LangAS::cuda_shared;
2233 break;
2234 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
2235 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
2236 break;
2237 }
2238 llvm::Type *VarTy = CGF.ConvertTypeForMem(T: VD->getType());
2239 auto *GV = new llvm::GlobalVariable(
2240 CGM.getModule(), VarTy, /*isConstant=*/false,
2241 llvm::GlobalValue::InternalLinkage, llvm::PoisonValue::get(T: VarTy),
2242 VD->getName(),
2243 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
2244 CGM.getContext().getTargetAddressSpace(AS));
2245 CharUnits Align = CGM.getContext().getDeclAlign(D: VD);
2246 GV->setAlignment(Align.getAsAlign());
2247 return Address(
2248 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2249 V: GV, DestTy: CGF.Builder.getPtrTy(AddrSpace: CGM.getContext().getTargetAddressSpace(
2250 AS: VD->getType().getAddressSpace()))),
2251 VarTy, Align);
2252 }
2253
2254 if (getDataSharingMode() != CGOpenMPRuntimeGPU::DS_Generic)
2255 return Address::invalid();
2256
2257 VD = VD->getCanonicalDecl();
2258 auto I = FunctionGlobalizedDecls.find(Val: CGF.CurFn);
2259 if (I == FunctionGlobalizedDecls.end())
2260 return Address::invalid();
2261 auto VDI = I->getSecond().LocalVarData.find(Key: VD);
2262 if (VDI != I->getSecond().LocalVarData.end())
2263 return VDI->second.PrivateAddr;
2264 if (VD->hasAttrs()) {
2265 for (specific_attr_iterator<OMPReferencedVarAttr> IT(VD->attr_begin()),
2266 E(VD->attr_end());
2267 IT != E; ++IT) {
2268 auto VDI = I->getSecond().LocalVarData.find(
2269 Key: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: IT->getRef())->getDecl())
2270 ->getCanonicalDecl());
2271 if (VDI != I->getSecond().LocalVarData.end())
2272 return VDI->second.PrivateAddr;
2273 }
2274 }
2275
2276 return Address::invalid();
2277}
2278
2279void CGOpenMPRuntimeGPU::functionFinished(CodeGenFunction &CGF) {
2280 FunctionGlobalizedDecls.erase(Val: CGF.CurFn);
2281 CGOpenMPRuntime::functionFinished(CGF);
2282}
2283
2284void CGOpenMPRuntimeGPU::getDefaultDistScheduleAndChunk(
2285 CodeGenFunction &CGF, const OMPLoopDirective &S,
2286 OpenMPDistScheduleClauseKind &ScheduleKind,
2287 llvm::Value *&Chunk) const {
2288 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
2289 if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD) {
2290 ScheduleKind = OMPC_DIST_SCHEDULE_static;
2291 Chunk = CGF.EmitScalarConversion(
2292 Src: RT.getGPUNumThreads(CGF),
2293 SrcTy: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
2294 DstTy: S.getIterationVariable()->getType(), Loc: S.getBeginLoc());
2295 return;
2296 }
2297 CGOpenMPRuntime::getDefaultDistScheduleAndChunk(
2298 CGF, S, ScheduleKind, Chunk);
2299}
2300
2301void CGOpenMPRuntimeGPU::getDefaultScheduleAndChunk(
2302 CodeGenFunction &CGF, const OMPLoopDirective &S,
2303 OpenMPScheduleClauseKind &ScheduleKind,
2304 const Expr *&ChunkExpr) const {
2305 ScheduleKind = OMPC_SCHEDULE_static;
2306 // Chunk size is 1 in this case.
2307 llvm::APInt ChunkSize(32, 1);
2308 ChunkExpr = IntegerLiteral::Create(C: CGF.getContext(), V: ChunkSize,
2309 type: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
2310 l: SourceLocation());
2311}
2312
2313void CGOpenMPRuntimeGPU::adjustTargetSpecificDataForLambdas(
2314 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
2315 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
2316 " Expected target-based directive.");
2317 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_target);
2318 for (const CapturedStmt::Capture &C : CS->captures()) {
2319 // Capture variables captured by reference in lambdas for target-based
2320 // directives.
2321 if (!C.capturesVariable())
2322 continue;
2323 const VarDecl *VD = C.getCapturedVar();
2324 const auto *RD = VD->getType()
2325 .getCanonicalType()
2326 .getNonReferenceType()
2327 ->getAsCXXRecordDecl();
2328 if (!RD || !RD->isLambda())
2329 continue;
2330 Address VDAddr = CGF.GetAddrOfLocalVar(VD);
2331 LValue VDLVal;
2332 if (VD->getType().getCanonicalType()->isReferenceType())
2333 VDLVal = CGF.EmitLoadOfReferenceLValue(RefAddr: VDAddr, RefTy: VD->getType());
2334 else
2335 VDLVal = CGF.MakeAddrLValue(
2336 Addr: VDAddr, T: VD->getType().getCanonicalType().getNonReferenceType());
2337 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
2338 FieldDecl *ThisCapture = nullptr;
2339 RD->getCaptureFields(Captures, ThisCapture);
2340 if (ThisCapture && CGF.CapturedStmtInfo->isCXXThisExprCaptured()) {
2341 LValue ThisLVal =
2342 CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: ThisCapture);
2343 llvm::Value *CXXThis = CGF.LoadCXXThis();
2344 CGF.EmitStoreOfScalar(value: CXXThis, lvalue: ThisLVal);
2345 }
2346 for (const LambdaCapture &LC : RD->captures()) {
2347 if (LC.getCaptureKind() != LCK_ByRef)
2348 continue;
2349 const ValueDecl *VD = LC.getCapturedVar();
2350 // FIXME: For now VD is always a VarDecl because OpenMP does not support
2351 // capturing structured bindings in lambdas yet.
2352 if (!CS->capturesVariable(Var: cast<VarDecl>(Val: VD)))
2353 continue;
2354 auto It = Captures.find(Val: VD);
2355 assert(It != Captures.end() && "Found lambda capture without field.");
2356 LValue VarLVal = CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: It->second);
2357 Address VDAddr = CGF.GetAddrOfLocalVar(VD: cast<VarDecl>(Val: VD));
2358 if (VD->getType().getCanonicalType()->isReferenceType())
2359 VDAddr = CGF.EmitLoadOfReferenceLValue(RefAddr: VDAddr,
2360 RefTy: VD->getType().getCanonicalType())
2361 .getAddress();
2362 CGF.EmitStoreOfScalar(value: VDAddr.emitRawPointer(CGF), lvalue: VarLVal);
2363 }
2364 }
2365}
2366
2367bool CGOpenMPRuntimeGPU::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
2368 LangAS &AS) {
2369 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
2370 return false;
2371 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
2372 switch(A->getAllocatorType()) {
2373 case OMPAllocateDeclAttr::OMPNullMemAlloc:
2374 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
2375 // Not supported, fallback to the default mem space.
2376 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
2377 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
2378 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
2379 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
2380 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
2381 AS = LangAS::Default;
2382 return true;
2383 case OMPAllocateDeclAttr::OMPConstMemAlloc:
2384 AS = LangAS::cuda_constant;
2385 return true;
2386 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
2387 AS = LangAS::cuda_shared;
2388 return true;
2389 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
2390 llvm_unreachable("Expected predefined allocator for the variables with the "
2391 "static storage.");
2392 }
2393 return false;
2394}
2395
2396/// Check to see if target architecture supports unified addressing which is
2397/// a restriction for OpenMP requires clause "unified_shared_memory".
2398void CGOpenMPRuntimeGPU::processRequiresDirective(const OMPRequiresDecl *D) {
2399 StringRef CPU = CGM.getTarget().getTargetOpts().CPU;
2400 if (CGM.getTarget().getTriple().isNVPTX() &&
2401 !llvm::NVPTX::supportsUnifiedAddressing(Kind: llvm::NVPTX::parseArch(CPU))) {
2402 for (const OMPClause *Clause : D->clauselists()) {
2403 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
2404 CGM.getDiags().Report(Loc: Clause->getBeginLoc(),
2405 DiagID: diag::err_omp_unified_shared_memory_unsupported)
2406 << CPU;
2407 return;
2408 }
2409 }
2410 }
2411
2412 CGOpenMPRuntime::processRequiresDirective(D);
2413}
2414
2415llvm::Value *CGOpenMPRuntimeGPU::getGPUNumThreads(CodeGenFunction &CGF) {
2416 CGBuilderTy &Bld = CGF.Builder;
2417 llvm::Module *M = &CGF.CGM.getModule();
2418 const char *LocSize = "__kmpc_get_hardware_num_threads_in_block";
2419 llvm::Function *F = M->getFunction(Name: LocSize);
2420 if (!F) {
2421 F = llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGF.Int32Ty, Params: {}, isVarArg: false),
2422 Linkage: llvm::GlobalVariable::ExternalLinkage, N: LocSize,
2423 M: &CGF.CGM.getModule());
2424 }
2425 return Bld.CreateCall(Callee: F, Args: {}, Name: "nvptx_num_threads");
2426}
2427
2428llvm::Value *CGOpenMPRuntimeGPU::getGPUThreadID(CodeGenFunction &CGF) {
2429 ArrayRef<llvm::Value *> Args{};
2430 return CGF.EmitRuntimeCall(
2431 callee: OMPBuilder.getOrCreateRuntimeFunction(
2432 M&: CGM.getModule(), FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block),
2433 args: Args);
2434}
2435