1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "ABIInfoImpl.h"
15#include "CGCXXABI.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGRecordLayout.h"
19#include "CodeGenFunction.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/Attr.h"
23#include "clang/AST/Decl.h"
24#include "clang/AST/OpenMPClause.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/SourceManager.h"
30#include "clang/CodeGen/ConstantInitBuilder.h"
31#include "clang/CodeGenUtils/RecordLayoutUtils.h"
32#include "llvm/ADT/ArrayRef.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/Bitcode/BitcodeReader.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/InstrTypes.h"
43#include "llvm/IR/Value.h"
44#include "llvm/Support/AtomicOrdering.h"
45#include "llvm/Support/VirtualFileSystem.h"
46#include "llvm/Support/raw_ostream.h"
47#include <cassert>
48#include <cstdint>
49#include <numeric>
50#include <optional>
51
52using namespace clang;
53using namespace CodeGen;
54using namespace llvm::omp;
55
56namespace {
57/// Base class for handling code generation inside OpenMP regions.
58class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
59public:
60 /// Kinds of OpenMP regions used in codegen.
61 enum CGOpenMPRegionKind {
62 /// Region with outlined function for standalone 'parallel'
63 /// directive.
64 ParallelOutlinedRegion,
65 /// Region with outlined function for standalone 'task' directive.
66 TaskOutlinedRegion,
67 /// Region for constructs that do not require function outlining,
68 /// like 'for', 'sections', 'atomic' etc. directives.
69 InlinedRegion,
70 /// Region with outlined function for standalone 'target' directive.
71 TargetRegion,
72 };
73
74 CGOpenMPRegionInfo(const CapturedStmt &CS,
75 const CGOpenMPRegionKind RegionKind,
76 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
77 bool HasCancel)
78 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
79 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
80
81 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
82 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
83 bool HasCancel)
84 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 Kind(Kind), HasCancel(HasCancel) {}
86
87 /// Get a variable or parameter for storing global thread id
88 /// inside OpenMP construct.
89 virtual const VarDecl *getThreadIDVariable() const = 0;
90
91 /// Emit the captured statement body.
92 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
93
94 /// Get an LValue for the current ThreadID variable.
95 /// \return LValue for thread id variable. This LValue always has type int32*.
96 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
97
98 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
99
100 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
101
102 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
103
104 bool hasCancel() const { return HasCancel; }
105
106 static bool classof(const CGCapturedStmtInfo *Info) {
107 return Info->getKind() == CR_OpenMP;
108 }
109
110 ~CGOpenMPRegionInfo() override = default;
111
112protected:
113 CGOpenMPRegionKind RegionKind;
114 RegionCodeGenTy CodeGen;
115 OpenMPDirectiveKind Kind;
116 bool HasCancel;
117};
118
119/// API for captured statement code generation in OpenMP constructs.
120class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
121public:
122 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
123 const RegionCodeGenTy &CodeGen,
124 OpenMPDirectiveKind Kind, bool HasCancel,
125 StringRef HelperName)
126 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
127 HasCancel),
128 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
129 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
130 }
131
132 /// Get a variable or parameter for storing global thread id
133 /// inside OpenMP construct.
134 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
135
136 /// Get the name of the capture helper.
137 StringRef getHelperName() const override { return HelperName; }
138
139 static bool classof(const CGCapturedStmtInfo *Info) {
140 return CGOpenMPRegionInfo::classof(Info) &&
141 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
142 ParallelOutlinedRegion;
143 }
144
145private:
146 /// A variable or parameter storing global thread id for OpenMP
147 /// constructs.
148 const VarDecl *ThreadIDVar;
149 StringRef HelperName;
150};
151
152/// API for captured statement code generation in OpenMP constructs.
153class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
154public:
155 class UntiedTaskActionTy final : public PrePostActionTy {
156 bool Untied;
157 const VarDecl *PartIDVar;
158 const RegionCodeGenTy UntiedCodeGen;
159 llvm::SwitchInst *UntiedSwitch = nullptr;
160
161 public:
162 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
163 const RegionCodeGenTy &UntiedCodeGen)
164 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
165 void Enter(CodeGenFunction &CGF) override {
166 if (Untied) {
167 // Emit task switching point.
168 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
169 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
170 PtrTy: PartIDVar->getType()->castAs<PointerType>());
171 llvm::Value *Res =
172 CGF.EmitLoadOfScalar(lvalue: PartIdLVal, Loc: PartIDVar->getLocation());
173 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: ".untied.done.");
174 UntiedSwitch = CGF.Builder.CreateSwitch(V: Res, Dest: DoneBB);
175 CGF.EmitBlock(BB: DoneBB);
176 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
177 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
178 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: 0),
179 Dest: CGF.Builder.GetInsertBlock());
180 emitUntiedSwitch(CGF);
181 }
182 }
183 void emitUntiedSwitch(CodeGenFunction &CGF) const {
184 if (Untied) {
185 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
186 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
187 PtrTy: PartIDVar->getType()->castAs<PointerType>());
188 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
189 lvalue: PartIdLVal);
190 UntiedCodeGen(CGF);
191 CodeGenFunction::JumpDest CurPoint =
192 CGF.getJumpDestInCurrentScope(Name: ".untied.next.");
193 CGF.EmitBranch(Block: CGF.ReturnBlock.getBlock());
194 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
195 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
196 Dest: CGF.Builder.GetInsertBlock());
197 CGF.EmitBranchThroughCleanup(Dest: CurPoint);
198 CGF.EmitBlock(BB: CurPoint.getBlock());
199 }
200 }
201 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
202 };
203 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
204 const VarDecl *ThreadIDVar,
205 const RegionCodeGenTy &CodeGen,
206 OpenMPDirectiveKind Kind, bool HasCancel,
207 const UntiedTaskActionTy &Action)
208 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
209 ThreadIDVar(ThreadIDVar), Action(Action) {
210 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
211 }
212
213 /// Get a variable or parameter for storing global thread id
214 /// inside OpenMP construct.
215 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
216
217 /// Get an LValue for the current ThreadID variable.
218 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
219
220 /// Get the name of the capture helper.
221 StringRef getHelperName() const override { return ".omp_outlined."; }
222
223 void emitUntiedSwitch(CodeGenFunction &CGF) override {
224 Action.emitUntiedSwitch(CGF);
225 }
226
227 static bool classof(const CGCapturedStmtInfo *Info) {
228 return CGOpenMPRegionInfo::classof(Info) &&
229 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
230 TaskOutlinedRegion;
231 }
232
233private:
234 /// A variable or parameter storing global thread id for OpenMP
235 /// constructs.
236 const VarDecl *ThreadIDVar;
237 /// Action for emitting code for untied tasks.
238 const UntiedTaskActionTy &Action;
239};
240
241/// API for inlined captured statement code generation in OpenMP
242/// constructs.
243class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
244public:
245 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
246 const RegionCodeGenTy &CodeGen,
247 OpenMPDirectiveKind Kind, bool HasCancel)
248 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
249 OldCSI(OldCSI),
250 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(Val: OldCSI)) {}
251
252 // Retrieve the value of the context parameter.
253 llvm::Value *getContextValue() const override {
254 if (OuterRegionInfo)
255 return OuterRegionInfo->getContextValue();
256 llvm_unreachable("No context value for inlined OpenMP region");
257 }
258
259 void setContextValue(llvm::Value *V) override {
260 if (OuterRegionInfo) {
261 OuterRegionInfo->setContextValue(V);
262 return;
263 }
264 llvm_unreachable("No context value for inlined OpenMP region");
265 }
266
267 /// Lookup the captured field decl for a variable.
268 const FieldDecl *lookup(const VarDecl *VD) const override {
269 if (OuterRegionInfo)
270 return OuterRegionInfo->lookup(VD);
271 // If there is no outer outlined region,no need to lookup in a list of
272 // captured variables, we can use the original one.
273 return nullptr;
274 }
275
276 FieldDecl *getThisFieldDecl() const override {
277 if (OuterRegionInfo)
278 return OuterRegionInfo->getThisFieldDecl();
279 return nullptr;
280 }
281
282 /// Get a variable or parameter for storing global thread id
283 /// inside OpenMP construct.
284 const VarDecl *getThreadIDVariable() const override {
285 if (OuterRegionInfo)
286 return OuterRegionInfo->getThreadIDVariable();
287 return nullptr;
288 }
289
290 /// Get an LValue for the current ThreadID variable.
291 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
292 if (OuterRegionInfo)
293 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
294 llvm_unreachable("No LValue for inlined OpenMP construct");
295 }
296
297 /// Get the name of the capture helper.
298 StringRef getHelperName() const override {
299 if (auto *OuterRegionInfo = getOldCSI())
300 return OuterRegionInfo->getHelperName();
301 llvm_unreachable("No helper name for inlined OpenMP construct");
302 }
303
304 void emitUntiedSwitch(CodeGenFunction &CGF) override {
305 if (OuterRegionInfo)
306 OuterRegionInfo->emitUntiedSwitch(CGF);
307 }
308
309 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
310
311 static bool classof(const CGCapturedStmtInfo *Info) {
312 return CGOpenMPRegionInfo::classof(Info) &&
313 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == InlinedRegion;
314 }
315
316 ~CGOpenMPInlinedRegionInfo() override = default;
317
318private:
319 /// CodeGen info about outer OpenMP region.
320 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
321 CGOpenMPRegionInfo *OuterRegionInfo;
322};
323
324/// API for captured statement code generation in OpenMP target
325/// constructs. For this captures, implicit parameters are used instead of the
326/// captured fields. The name of the target region has to be unique in a given
327/// application so it is provided by the client, because only the client has
328/// the information to generate that.
329class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
330public:
331 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
332 const RegionCodeGenTy &CodeGen, StringRef HelperName)
333 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
334 /*HasCancel=*/false),
335 HelperName(HelperName) {}
336
337 /// This is unused for target regions because each starts executing
338 /// with a single thread.
339 const VarDecl *getThreadIDVariable() const override { return nullptr; }
340
341 /// Get the name of the capture helper.
342 StringRef getHelperName() const override { return HelperName; }
343
344 static bool classof(const CGCapturedStmtInfo *Info) {
345 return CGOpenMPRegionInfo::classof(Info) &&
346 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == TargetRegion;
347 }
348
349private:
350 StringRef HelperName;
351};
352
353static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
354 llvm_unreachable("No codegen for expressions");
355}
356/// API for generation of expressions captured in a innermost OpenMP
357/// region.
358class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
359public:
360 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
361 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
362 OMPD_unknown,
363 /*HasCancel=*/false),
364 PrivScope(CGF) {
365 // Make sure the globals captured in the provided statement are local by
366 // using the privatization logic. We assume the same variable is not
367 // captured more than once.
368 for (const auto &C : CS.captures()) {
369 if (!C.capturesVariable() && !C.capturesVariableByCopy())
370 continue;
371
372 const VarDecl *VD = C.getCapturedVar();
373 if (VD->isLocalVarDeclOrParm())
374 continue;
375
376 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
377 /*RefersToEnclosingVariableOrCapture=*/false,
378 VD->getType().getNonReferenceType(), VK_LValue,
379 C.getLocation());
380 PrivScope.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
381 }
382 (void)PrivScope.Privatize();
383 }
384
385 /// Lookup the captured field decl for a variable.
386 const FieldDecl *lookup(const VarDecl *VD) const override {
387 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
388 return FD;
389 return nullptr;
390 }
391
392 /// Emit the captured statement body.
393 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
394 llvm_unreachable("No body for expressions");
395 }
396
397 /// Get a variable or parameter for storing global thread id
398 /// inside OpenMP construct.
399 const VarDecl *getThreadIDVariable() const override {
400 llvm_unreachable("No thread id for expressions");
401 }
402
403 /// Get the name of the capture helper.
404 StringRef getHelperName() const override {
405 llvm_unreachable("No helper name for expressions");
406 }
407
408 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
409
410private:
411 /// Private scope to capture global variables.
412 CodeGenFunction::OMPPrivateScope PrivScope;
413};
414
415/// RAII for emitting code of OpenMP constructs.
416class InlinedOpenMPRegionRAII {
417 CodeGenFunction &CGF;
418 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
419 FieldDecl *LambdaThisCaptureField = nullptr;
420 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
421 bool NoInheritance = false;
422
423public:
424 /// Constructs region for combined constructs.
425 /// \param CodeGen Code generation sequence for combined directives. Includes
426 /// a list of functions used for code generation of implicitly inlined
427 /// regions.
428 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
429 OpenMPDirectiveKind Kind, bool HasCancel,
430 bool NoInheritance = true)
431 : CGF(CGF), NoInheritance(NoInheritance) {
432 // Start emission for the construct.
433 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
434 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
435 if (NoInheritance) {
436 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
437 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
438 CGF.LambdaThisCaptureField = nullptr;
439 BlockInfo = CGF.BlockInfo;
440 CGF.BlockInfo = nullptr;
441 }
442 }
443
444 ~InlinedOpenMPRegionRAII() {
445 // Restore original CapturedStmtInfo only if we're done with code emission.
446 auto *OldCSI =
447 cast<CGOpenMPInlinedRegionInfo>(Val: CGF.CapturedStmtInfo)->getOldCSI();
448 delete CGF.CapturedStmtInfo;
449 CGF.CapturedStmtInfo = OldCSI;
450 if (NoInheritance) {
451 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
452 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
453 CGF.BlockInfo = BlockInfo;
454 }
455 }
456};
457
458/// Values for bit flags used in the ident_t to describe the fields.
459/// All enumeric elements are named and described in accordance with the code
460/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
461enum OpenMPLocationFlags : unsigned {
462 /// Use trampoline for internal microtask.
463 OMP_IDENT_IMD = 0x01,
464 /// Use c-style ident structure.
465 OMP_IDENT_KMPC = 0x02,
466 /// Atomic reduction option for kmpc_reduce.
467 OMP_ATOMIC_REDUCE = 0x10,
468 /// Explicit 'barrier' directive.
469 OMP_IDENT_BARRIER_EXPL = 0x20,
470 /// Implicit barrier in code.
471 OMP_IDENT_BARRIER_IMPL = 0x40,
472 /// Implicit barrier in 'for' directive.
473 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
474 /// Implicit barrier in 'sections' directive.
475 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
476 /// Implicit barrier in 'single' directive.
477 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
478 /// Call of __kmp_for_static_init for static loop.
479 OMP_IDENT_WORK_LOOP = 0x200,
480 /// Call of __kmp_for_static_init for sections.
481 OMP_IDENT_WORK_SECTIONS = 0x400,
482 /// Call of __kmp_for_static_init for distribute.
483 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
484 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
485};
486
487/// Describes ident structure that describes a source location.
488/// All descriptions are taken from
489/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
490/// Original structure:
491/// typedef struct ident {
492/// kmp_int32 reserved_1; /**< might be used in Fortran;
493/// see above */
494/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
495/// KMP_IDENT_KMPC identifies this union
496/// member */
497/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
498/// see above */
499///#if USE_ITT_BUILD
500/// /* but currently used for storing
501/// region-specific ITT */
502/// /* contextual information. */
503///#endif /* USE_ITT_BUILD */
504/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
505/// C++ */
506/// char const *psource; /**< String describing the source location.
507/// The string is composed of semi-colon separated
508// fields which describe the source file,
509/// the function and a pair of line numbers that
510/// delimit the construct.
511/// */
512/// } ident_t;
513enum IdentFieldIndex {
514 /// might be used in Fortran
515 IdentField_Reserved_1,
516 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
517 IdentField_Flags,
518 /// Not really used in Fortran any more
519 IdentField_Reserved_2,
520 /// Source[4] in Fortran, do not use for C++
521 IdentField_Reserved_3,
522 /// String describing the source location. The string is composed of
523 /// semi-colon separated fields which describe the source file, the function
524 /// and a pair of line numbers that delimit the construct.
525 IdentField_PSource
526};
527
528/// Schedule types for 'omp for' loops (these enumerators are taken from
529/// the enum sched_type in kmp.h).
530enum OpenMPSchedType {
531 /// Lower bound for default (unordered) versions.
532 OMP_sch_lower = 32,
533 OMP_sch_static_chunked = 33,
534 OMP_sch_static = 34,
535 OMP_sch_dynamic_chunked = 35,
536 OMP_sch_guided_chunked = 36,
537 OMP_sch_runtime = 37,
538 OMP_sch_auto = 38,
539 /// static with chunk adjustment (e.g., simd)
540 OMP_sch_static_balanced_chunked = 45,
541 /// Lower bound for 'ordered' versions.
542 OMP_ord_lower = 64,
543 OMP_ord_static_chunked = 65,
544 OMP_ord_static = 66,
545 OMP_ord_dynamic_chunked = 67,
546 OMP_ord_guided_chunked = 68,
547 OMP_ord_runtime = 69,
548 OMP_ord_auto = 70,
549 OMP_sch_default = OMP_sch_static,
550 /// dist_schedule types
551 OMP_dist_sch_static_chunked = 91,
552 OMP_dist_sch_static = 92,
553 /// Fused distribute+for static schedule (entityId = team*nthreads + tid,
554 /// num_entities = nteams*nthreads). One for_static_init call, no
555 /// surrounding distribute_static_init. Matches
556 /// kmp_sched_distr_static_chunk_sched_static_chunkone in the device RTL
557 /// (openmp/device/include/DeviceTypes.h).
558 OMP_dist_sch_static_chunked_sch_static_chunkone = 93,
559 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
560 /// Set if the monotonic schedule modifier was present.
561 OMP_sch_modifier_monotonic = (1 << 29),
562 /// Set if the nonmonotonic schedule modifier was present.
563 OMP_sch_modifier_nonmonotonic = (1 << 30),
564};
565
566/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
567/// region.
568class CleanupTy final : public EHScopeStack::Cleanup {
569 PrePostActionTy *Action;
570
571public:
572 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
573 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
574 if (!CGF.HaveInsertPoint())
575 return;
576 Action->Exit(CGF);
577 }
578};
579
580} // anonymous namespace
581
582void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
583 CodeGenFunction::RunCleanupsScope Scope(CGF);
584 if (PrePostAction) {
585 CGF.EHStack.pushCleanup<CleanupTy>(Kind: NormalAndEHCleanup, A: PrePostAction);
586 Callback(CodeGen, CGF, *PrePostAction);
587 } else {
588 PrePostActionTy Action;
589 Callback(CodeGen, CGF, Action);
590 }
591}
592
593/// Check if the combiner is a call to UDR combiner and if it is so return the
594/// UDR decl used for reduction.
595static const OMPDeclareReductionDecl *
596getReductionInit(const Expr *ReductionOp) {
597 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
598 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
599 if (const auto *DRE =
600 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
601 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl()))
602 return DRD;
603 return nullptr;
604}
605
606static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
607 const OMPDeclareReductionDecl *DRD,
608 const Expr *InitOp,
609 Address Private, Address Original,
610 QualType Ty) {
611 if (DRD->getInitializer()) {
612 std::pair<llvm::Function *, llvm::Function *> Reduction =
613 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
614 const auto *CE = cast<CallExpr>(Val: InitOp);
615 const auto *OVE = cast<OpaqueValueExpr>(Val: CE->getCallee());
616 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
617 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
618 const auto *LHSDRE =
619 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: LHS)->getSubExpr());
620 const auto *RHSDRE =
621 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: RHS)->getSubExpr());
622 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
623 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: LHSDRE->getDecl()), Addr: Private);
624 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: RHSDRE->getDecl()), Addr: Original);
625 (void)PrivateScope.Privatize();
626 RValue Func = RValue::get(V: Reduction.second);
627 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
628 CGF.EmitIgnoredExpr(E: InitOp);
629 } else {
630 llvm::Constant *Init = CGF.CGM.EmitNullConstant(T: Ty);
631 std::string Name = CGF.CGM.getOpenMPRuntime().getName(Parts: {"init"});
632 auto *GV = new llvm::GlobalVariable(
633 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
634 llvm::GlobalValue::PrivateLinkage, Init, Name);
635 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V: GV, T: Ty);
636 RValue InitRVal;
637 switch (CGF.getEvaluationKind(T: Ty)) {
638 case TEK_Scalar:
639 InitRVal = CGF.EmitLoadOfLValue(V: LV, Loc: DRD->getLocation());
640 break;
641 case TEK_Complex:
642 InitRVal =
643 RValue::getComplex(C: CGF.EmitLoadOfComplex(src: LV, loc: DRD->getLocation()));
644 break;
645 case TEK_Aggregate: {
646 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);
647 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
648 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
649 /*IsInitializer=*/false);
650 return;
651 }
652 }
653 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue);
654 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
655 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
656 /*IsInitializer=*/false);
657 }
658}
659
660/// Emit initialization of arrays of complex types.
661/// \param DestAddr Address of the array.
662/// \param Type Type of array.
663/// \param Init Initial expression of array.
664/// \param SrcAddr Address of the original array.
665static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
666 QualType Type, bool EmitDeclareReductionInit,
667 const Expr *Init,
668 const OMPDeclareReductionDecl *DRD,
669 Address SrcAddr = Address::invalid()) {
670 // Perform element-by-element initialization.
671 QualType ElementTy;
672
673 // Drill down to the base element type on both arrays.
674 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
675 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
676 if (DRD)
677 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
678
679 llvm::Value *SrcBegin = nullptr;
680 if (DRD)
681 SrcBegin = SrcAddr.emitRawPointer(CGF);
682 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
683 // Cast from pointer to array type to pointer to single element.
684 llvm::Value *DestEnd =
685 CGF.Builder.CreateGEP(Ty: DestAddr.getElementType(), Ptr: DestBegin, IdxList: NumElements);
686 // The basic structure here is a while-do loop.
687 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arrayinit.body");
688 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arrayinit.done");
689 llvm::Value *IsEmpty =
690 CGF.Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arrayinit.isempty");
691 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
692
693 // Enter the loop body, making that address the current address.
694 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
695 CGF.EmitBlock(BB: BodyBB);
696
697 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
698
699 llvm::PHINode *SrcElementPHI = nullptr;
700 Address SrcElementCurrent = Address::invalid();
701 if (DRD) {
702 SrcElementPHI = CGF.Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2,
703 Name: "omp.arraycpy.srcElementPast");
704 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
705 SrcElementCurrent =
706 Address(SrcElementPHI, SrcAddr.getElementType(),
707 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
708 }
709 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
710 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
711 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
712 Address DestElementCurrent =
713 Address(DestElementPHI, DestAddr.getElementType(),
714 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
715
716 // Emit copy.
717 {
718 CodeGenFunction::RunCleanupsScope InitScope(CGF);
719 if (EmitDeclareReductionInit) {
720 emitInitWithReductionInitializer(CGF, DRD, InitOp: Init, Private: DestElementCurrent,
721 Original: SrcElementCurrent, Ty: ElementTy);
722 } else
723 CGF.EmitAnyExprToMem(E: Init, Location: DestElementCurrent, Quals: ElementTy.getQualifiers(),
724 /*IsInitializer=*/false);
725 }
726
727 if (DRD) {
728 // Shift the address forward by one element.
729 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
730 Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI, /*Idx0=*/1,
731 Name: "omp.arraycpy.dest.element");
732 SrcElementPHI->addIncoming(V: SrcElementNext, BB: CGF.Builder.GetInsertBlock());
733 }
734
735 // Shift the address forward by one element.
736 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
737 Ty: DestAddr.getElementType(), Ptr: DestElementPHI, /*Idx0=*/1,
738 Name: "omp.arraycpy.dest.element");
739 // Check whether we've reached the end.
740 llvm::Value *Done =
741 CGF.Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
742 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
743 DestElementPHI->addIncoming(V: DestElementNext, BB: CGF.Builder.GetInsertBlock());
744
745 // Done.
746 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
747}
748
749LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
750 return CGF.EmitOMPSharedLValue(E);
751}
752
753LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
754 const Expr *E) {
755 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E))
756 return CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false);
757 return LValue();
758}
759
760void ReductionCodeGen::emitAggregateInitialization(
761 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
762 const OMPDeclareReductionDecl *DRD) {
763 // Emit VarDecl with copy init for arrays.
764 // Get the address of the original variable captured in current
765 // captured region.
766 const auto *PrivateVD =
767 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
768 bool EmitDeclareReductionInit =
769 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
770 EmitOMPAggregateInit(CGF, DestAddr: PrivateAddr, Type: PrivateVD->getType(),
771 EmitDeclareReductionInit,
772 Init: EmitDeclareReductionInit ? ClausesData[N].ReductionOp
773 : PrivateVD->getInit(),
774 DRD, SrcAddr: SharedAddr);
775}
776
777ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
778 ArrayRef<const Expr *> Origs,
779 ArrayRef<const Expr *> Privates,
780 ArrayRef<const Expr *> ReductionOps) {
781 ClausesData.reserve(N: Shareds.size());
782 SharedAddresses.reserve(N: Shareds.size());
783 Sizes.reserve(N: Shareds.size());
784 BaseDecls.reserve(N: Shareds.size());
785 const auto *IOrig = Origs.begin();
786 const auto *IPriv = Privates.begin();
787 const auto *IRed = ReductionOps.begin();
788 for (const Expr *Ref : Shareds) {
789 ClausesData.emplace_back(Args&: Ref, Args: *IOrig, Args: *IPriv, Args: *IRed);
790 std::advance(i&: IOrig, n: 1);
791 std::advance(i&: IPriv, n: 1);
792 std::advance(i&: IRed, n: 1);
793 }
794}
795
796void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
797 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
798 "Number of generated lvalues must be exactly N.");
799 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Shared);
800 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Shared);
801 SharedAddresses.emplace_back(Args&: First, Args&: Second);
802 if (ClausesData[N].Shared == ClausesData[N].Ref) {
803 OrigAddresses.emplace_back(Args&: First, Args&: Second);
804 } else {
805 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Ref);
806 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Ref);
807 OrigAddresses.emplace_back(Args&: First, Args&: Second);
808 }
809}
810
811void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
812 QualType PrivateType = getPrivateType(N);
813 bool AsArraySection = isa<ArraySectionExpr>(Val: ClausesData[N].Ref);
814 if (!PrivateType->isVariablyModifiedType()) {
815 Sizes.emplace_back(
816 Args: CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType()),
817 Args: nullptr);
818 return;
819 }
820 llvm::Value *Size;
821 llvm::Value *SizeInChars;
822 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
823 auto *ElemSizeOf = llvm::ConstantInt::get(
824 Ty: CGF.SizeTy, V: CGF.CGM.getDataLayout().getTypeAllocSize(Ty: ElemType));
825 if (AsArraySection) {
826 SizeInChars =
827 CGF.Builder.CreatePtrDiff(LHS: OrigAddresses[N].second.getPointer(CGF),
828 RHS: OrigAddresses[N].first.getPointer(CGF));
829 SizeInChars = CGF.Builder.CreateNUWAdd(LHS: SizeInChars, RHS: ElemSizeOf);
830 } else {
831 SizeInChars =
832 CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType());
833 }
834 Size = ElemSizeOf->isOne()
835 ? SizeInChars
836 : CGF.Builder.CreateExactUDiv(LHS: SizeInChars, RHS: ElemSizeOf);
837 Sizes.emplace_back(Args&: SizeInChars, Args&: Size);
838 CodeGenFunction::OpaqueValueMapping OpaqueMap(
839 CGF,
840 cast<OpaqueValueExpr>(
841 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
842 RValue::get(V: Size));
843 CGF.EmitVariablyModifiedType(Ty: PrivateType);
844}
845
846void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
847 llvm::Value *Size) {
848 QualType PrivateType = getPrivateType(N);
849 if (!PrivateType->isVariablyModifiedType()) {
850 assert(!Size && !Sizes[N].second &&
851 "Size should be nullptr for non-variably modified reduction "
852 "items.");
853 return;
854 }
855 CodeGenFunction::OpaqueValueMapping OpaqueMap(
856 CGF,
857 cast<OpaqueValueExpr>(
858 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
859 RValue::get(V: Size));
860 CGF.EmitVariablyModifiedType(Ty: PrivateType);
861}
862
863void ReductionCodeGen::emitInitialization(
864 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
865 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
866 assert(SharedAddresses.size() > N && "No variable was generated");
867 const auto *PrivateVD =
868 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
869 const OMPDeclareReductionDecl *DRD =
870 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
871 if (CGF.getContext().getAsArrayType(T: PrivateVD->getType())) {
872 if (DRD && DRD->getInitializer())
873 (void)DefaultInit(CGF);
874 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
875 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
876 (void)DefaultInit(CGF);
877 QualType SharedType = SharedAddresses[N].first.getType();
878 emitInitWithReductionInitializer(CGF, DRD, InitOp: ClausesData[N].ReductionOp,
879 Private: PrivateAddr, Original: SharedAddr, Ty: SharedType);
880 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
881 !CGF.isTrivialInitializer(Init: PrivateVD->getInit())) {
882 CGF.EmitAnyExprToMem(E: PrivateVD->getInit(), Location: PrivateAddr,
883 Quals: PrivateVD->getType().getQualifiers(),
884 /*IsInitializer=*/false);
885 }
886}
887
888bool ReductionCodeGen::needCleanups(unsigned N) {
889 QualType PrivateType = getPrivateType(N);
890 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
891 return DTorKind != QualType::DK_none;
892}
893
894void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
895 Address PrivateAddr) {
896 QualType PrivateType = getPrivateType(N);
897 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
898 if (needCleanups(N)) {
899 PrivateAddr =
900 PrivateAddr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: PrivateType));
901 CGF.pushDestroy(dtorKind: DTorKind, addr: PrivateAddr, type: PrivateType);
902 }
903}
904
905static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
906 LValue BaseLV) {
907 BaseTy = BaseTy.getNonReferenceType();
908 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
909 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
910 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
911 BaseLV = CGF.EmitLoadOfPointerLValue(Ptr: BaseLV.getAddress(), PtrTy);
912 } else {
913 LValue RefLVal = CGF.MakeAddrLValue(Addr: BaseLV.getAddress(), T: BaseTy);
914 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
915 }
916 BaseTy = BaseTy->getPointeeType();
917 }
918 return CGF.MakeAddrLValue(
919 Addr: BaseLV.getAddress().withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy)),
920 T: BaseLV.getType(), BaseInfo: BaseLV.getBaseInfo(),
921 TBAAInfo: CGF.CGM.getTBAAInfoForSubobject(Base: BaseLV, AccessType: BaseLV.getType()));
922}
923
924static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
925 Address OriginalBaseAddress, llvm::Value *Addr) {
926 RawAddress Tmp = RawAddress::invalid();
927 Address TopTmp = Address::invalid();
928 Address MostTopTmp = Address::invalid();
929 BaseTy = BaseTy.getNonReferenceType();
930 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
931 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
932 Tmp = CGF.CreateMemTempWithoutCast(T: BaseTy);
933 if (TopTmp.isValid())
934 CGF.Builder.CreateStore(Val: Tmp.getPointer(), Addr: TopTmp);
935 else
936 MostTopTmp = Tmp;
937 TopTmp = Tmp;
938 BaseTy = BaseTy->getPointeeType();
939 }
940
941 if (Tmp.isValid()) {
942 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
943 V: Addr, DestTy: Tmp.getElementType());
944 CGF.Builder.CreateStore(Val: Addr, Addr: Tmp);
945 return MostTopTmp;
946 }
947
948 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
949 V: Addr, DestTy: OriginalBaseAddress.getType());
950 return OriginalBaseAddress.withPointer(NewPointer: Addr, IsKnownNonNull: NotKnownNonNull);
951}
952
953static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
954 const VarDecl *OrigVD = nullptr;
955 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Ref)) {
956 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
957 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Val: Base))
958 Base = TempOASE->getBase()->IgnoreParenImpCasts();
959 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
960 Base = TempASE->getBase()->IgnoreParenImpCasts();
961 DE = cast<DeclRefExpr>(Val: Base);
962 OrigVD = cast<VarDecl>(Val: DE->getDecl());
963 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Ref)) {
964 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
965 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
966 Base = TempASE->getBase()->IgnoreParenImpCasts();
967 DE = cast<DeclRefExpr>(Val: Base);
968 OrigVD = cast<VarDecl>(Val: DE->getDecl());
969 }
970 return OrigVD;
971}
972
973Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
974 Address PrivateAddr) {
975 const DeclRefExpr *DE;
976 if (const VarDecl *OrigVD = ::getBaseDecl(Ref: ClausesData[N].Ref, DE)) {
977 BaseDecls.emplace_back(Args&: OrigVD);
978 LValue OriginalBaseLValue = CGF.EmitLValue(E: DE);
979 LValue BaseLValue =
980 loadToBegin(CGF, BaseTy: OrigVD->getType(), ElTy: SharedAddresses[N].first.getType(),
981 BaseLV: OriginalBaseLValue);
982 Address SharedAddr = SharedAddresses[N].first.getAddress();
983 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
984 ElemTy: SharedAddr.getElementType(), LHS: BaseLValue.getPointer(CGF),
985 RHS: SharedAddr.emitRawPointer(CGF));
986 llvm::Value *PrivatePointer =
987 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
988 V: PrivateAddr.emitRawPointer(CGF), DestTy: SharedAddr.getType());
989 llvm::Value *Ptr = CGF.Builder.CreateGEP(
990 Ty: SharedAddr.getElementType(), Ptr: PrivatePointer, IdxList: Adjustment);
991 return castToBase(CGF, BaseTy: OrigVD->getType(),
992 ElTy: SharedAddresses[N].first.getType(),
993 OriginalBaseAddress: OriginalBaseLValue.getAddress(), Addr: Ptr);
994 }
995 BaseDecls.emplace_back(
996 Args: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Ref)->getDecl()));
997 return PrivateAddr;
998}
999
1000bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1001 const OMPDeclareReductionDecl *DRD =
1002 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
1003 return DRD && DRD->getInitializer();
1004}
1005
1006LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1007 return CGF.EmitLoadOfPointerLValue(
1008 Ptr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1009 PtrTy: getThreadIDVariable()->getType()->castAs<PointerType>());
1010}
1011
1012void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1013 if (!CGF.HaveInsertPoint())
1014 return;
1015 // 1.2.2 OpenMP Language Terminology
1016 // Structured block - An executable statement with a single entry at the
1017 // top and a single exit at the bottom.
1018 // The point of exit cannot be a branch out of the structured block.
1019 // longjmp() and throw() must not violate the entry/exit criteria.
1020 CGF.EHStack.pushTerminate();
1021 if (S)
1022 CGF.incrementProfileCounter(S);
1023 CodeGen(CGF);
1024 CGF.EHStack.popTerminate();
1025}
1026
1027LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1028 CodeGenFunction &CGF) {
1029 return CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1030 T: getThreadIDVariable()->getType(),
1031 Source: AlignmentSource::Decl);
1032}
1033
1034static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1035 QualType FieldTy) {
1036 auto *Field = FieldDecl::Create(
1037 C, DC, StartLoc: SourceLocation(), IdLoc: SourceLocation(), /*Id=*/nullptr, T: FieldTy,
1038 TInfo: C.getTrivialTypeSourceInfo(T: FieldTy, Loc: SourceLocation()),
1039 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1040 Field->setAccess(AS_public);
1041 DC->addDecl(D: Field);
1042 return Field;
1043}
1044
1045CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
1046 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1047 KmpCriticalNameTy = llvm::ArrayType::get(ElementType: CGM.Int32Ty, /*NumElements*/ 8);
1048 llvm::OpenMPIRBuilderConfig Config(
1049 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),
1050 CGM.getLangOpts().OpenMPOffloadMandatory,
1051 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,
1052 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);
1053 Config.setDefaultTargetAS(
1054 CGM.getContext().getTargetInfo().getTargetAddressSpace(AS: LangAS::Default));
1055 Config.setRuntimeCC(CGM.getRuntimeCC());
1056
1057 OMPBuilder.setConfig(Config);
1058 OMPBuilder.initialize();
1059 OMPBuilder.loadOffloadInfoMetadata(VFS&: *CGM.getFileSystem(),
1060 HostFilePath: CGM.getLangOpts().OpenMPIsTargetDevice
1061 ? CGM.getLangOpts().OMPHostIRFile
1062 : StringRef{});
1063
1064 // The user forces the compiler to behave as if omp requires
1065 // unified_shared_memory was given.
1066 if (CGM.getLangOpts().OpenMPForceUSM) {
1067 HasRequiresUnifiedSharedMemory = true;
1068 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
1069 }
1070}
1071
1072void CGOpenMPRuntime::clear() {
1073 InternalVars.clear();
1074 // Clean non-target variable declarations possibly used only in debug info.
1075 for (const auto &Data : EmittedNonTargetVariables) {
1076 if (!Data.getValue().pointsToAliveValue())
1077 continue;
1078 auto *GV = dyn_cast<llvm::GlobalVariable>(Val: Data.getValue());
1079 if (!GV)
1080 continue;
1081 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1082 continue;
1083 GV->eraseFromParent();
1084 }
1085}
1086
1087std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1088 return OMPBuilder.createPlatformSpecificName(Parts);
1089}
1090
1091static llvm::Function *
1092emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1093 const Expr *CombinerInitializer, const VarDecl *In,
1094 const VarDecl *Out, bool IsCombiner) {
1095 // void .omp_combiner.(Ty *in, Ty *out);
1096 ASTContext &C = CGM.getContext();
1097 QualType PtrTy = C.getPointerType(T: Ty).withRestrict();
1098 auto *OmpOutParm = ImplicitParamDecl::Create(
1099 C, /*DC=*/nullptr, IdLoc: Out->getLocation(),
1100 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1101 auto *OmpInParm = ImplicitParamDecl::Create(
1102 C, /*DC=*/nullptr, IdLoc: In->getLocation(),
1103 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1104 FunctionArgList Args{OmpOutParm, OmpInParm};
1105 const CGFunctionInfo &FnInfo =
1106 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
1107 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1108 std::string Name = CGM.getOpenMPRuntime().getName(
1109 Parts: {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1110 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
1111 N: Name, M: &CGM.getModule());
1112 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
1113 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1114 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
1115 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
1116 Fn->removeFnAttr(Kind: llvm::Attribute::NoInline);
1117 Fn->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
1118 Fn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1119 }
1120 CodeGenFunction CGF(CGM);
1121 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1122 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1123 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc: In->getLocation(),
1124 StartLoc: Out->getLocation());
1125 CodeGenFunction::OMPPrivateScope Scope(CGF);
1126 Address AddrIn = CGF.GetAddrOfLocalVar(VD: OmpInParm);
1127 Scope.addPrivate(
1128 LocalVD: In, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrIn, PtrTy: PtrTy->castAs<PointerType>())
1129 .getAddress());
1130 Address AddrOut = CGF.GetAddrOfLocalVar(VD: OmpOutParm);
1131 Scope.addPrivate(
1132 LocalVD: Out, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrOut, PtrTy: PtrTy->castAs<PointerType>())
1133 .getAddress());
1134 (void)Scope.Privatize();
1135 if (!IsCombiner && Out->hasInit() &&
1136 !CGF.isTrivialInitializer(Init: Out->getInit())) {
1137 CGF.EmitAnyExprToMem(E: Out->getInit(), Location: CGF.GetAddrOfLocalVar(VD: Out),
1138 Quals: Out->getType().getQualifiers(),
1139 /*IsInitializer=*/true);
1140 }
1141 if (CombinerInitializer)
1142 CGF.EmitIgnoredExpr(E: CombinerInitializer);
1143 Scope.ForceCleanup();
1144 CGF.FinishFunction();
1145 return Fn;
1146}
1147
1148void CGOpenMPRuntime::emitUserDefinedReduction(
1149 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1150 if (UDRMap.count(Val: D) > 0)
1151 return;
1152 llvm::Function *Combiner = emitCombinerOrInitializer(
1153 CGM, Ty: D->getType(), CombinerInitializer: D->getCombiner(),
1154 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerIn())->getDecl()),
1155 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerOut())->getDecl()),
1156 /*IsCombiner=*/true);
1157 llvm::Function *Initializer = nullptr;
1158 if (const Expr *Init = D->getInitializer()) {
1159 Initializer = emitCombinerOrInitializer(
1160 CGM, Ty: D->getType(),
1161 CombinerInitializer: D->getInitializerKind() == OMPDeclareReductionInitKind::Call ? Init
1162 : nullptr,
1163 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitOrig())->getDecl()),
1164 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitPriv())->getDecl()),
1165 /*IsCombiner=*/false);
1166 }
1167 UDRMap.try_emplace(Key: D, Args&: Combiner, Args&: Initializer);
1168 if (CGF)
1169 FunctionUDRMap[CGF->CurFn].push_back(Elt: D);
1170}
1171
1172std::pair<llvm::Function *, llvm::Function *>
1173CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1174 auto I = UDRMap.find(Val: D);
1175 if (I != UDRMap.end())
1176 return I->second;
1177 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1178 return UDRMap.lookup(Val: D);
1179}
1180
1181namespace {
1182// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1183// Builder if one is present.
1184struct PushAndPopStackRAII {
1185 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1186 bool HasCancel, llvm::omp::Directive Kind)
1187 : OMPBuilder(OMPBuilder) {
1188 if (!OMPBuilder)
1189 return;
1190
1191 // The following callback is the crucial part of clangs cleanup process.
1192 //
1193 // NOTE:
1194 // Once the OpenMPIRBuilder is used to create parallel regions (and
1195 // similar), the cancellation destination (Dest below) is determined via
1196 // IP. That means if we have variables to finalize we split the block at IP,
1197 // use the new block (=BB) as destination to build a JumpDest (via
1198 // getJumpDestInCurrentScope(BB)) which then is fed to
1199 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1200 // to push & pop an FinalizationInfo object.
1201 // The FiniCB will still be needed but at the point where the
1202 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1203 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1204 assert(IP == IP.getNodeParent()->end() &&
1205 "Clang CG should cause non-terminated block!");
1206 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1207 CGF.Builder.restoreIP(IP);
1208 CodeGenFunction::JumpDest Dest =
1209 CGF.getOMPCancelDestination(Kind: OMPD_parallel);
1210 CGF.EmitBranchThroughCleanup(Dest);
1211 return llvm::Error::success();
1212 };
1213
1214 // TODO: Remove this once we emit parallel regions through the
1215 // OpenMPIRBuilder as it can do this setup internally.
1216 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1217 OMPBuilder->pushFinalizationCB(FI: std::move(FI));
1218 }
1219 ~PushAndPopStackRAII() {
1220 if (OMPBuilder)
1221 OMPBuilder->popFinalizationCB();
1222 }
1223 llvm::OpenMPIRBuilder *OMPBuilder;
1224};
1225} // namespace
1226
1227static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1228 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1229 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1230 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1231 assert(ThreadIDVar->getType()->isPointerType() &&
1232 "thread id variable must be of type kmp_int32 *");
1233 CodeGenFunction CGF(CGM, true);
1234 bool HasCancel = false;
1235 if (const auto *OPD = dyn_cast<OMPParallelDirective>(Val: &D))
1236 HasCancel = OPD->hasCancel();
1237 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(Val: &D))
1238 HasCancel = OPD->hasCancel();
1239 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &D))
1240 HasCancel = OPSD->hasCancel();
1241 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(Val: &D))
1242 HasCancel = OPFD->hasCancel();
1243 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(Val: &D))
1244 HasCancel = OPFD->hasCancel();
1245 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(Val: &D))
1246 HasCancel = OPFD->hasCancel();
1247 else if (const auto *OPFD =
1248 dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &D))
1249 HasCancel = OPFD->hasCancel();
1250 else if (const auto *OPFD =
1251 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &D))
1252 HasCancel = OPFD->hasCancel();
1253
1254 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1255 // parallel region to make cancellation barriers work properly.
1256 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1257 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1258 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1259 HasCancel, OutlinedHelperName);
1260 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1261 return CGF.GenerateOpenMPCapturedStmtFunction(S: *CS, D);
1262}
1263
1264std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const {
1265 std::string Suffix = getName(Parts: {"omp_outlined"});
1266 return (Name + Suffix).str();
1267}
1268
1269std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {
1270 return getOutlinedHelperName(Name: CGF.CurFn->getName());
1271}
1272
1273std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const {
1274 std::string Suffix = getName(Parts: {"omp", "reduction", "reduction_func"});
1275 return (Name + Suffix).str();
1276}
1277
1278llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1279 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1280 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1281 const RegionCodeGenTy &CodeGen) {
1282 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_parallel);
1283 return emitParallelOrTeamsOutlinedFunction(
1284 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1285 CodeGen);
1286}
1287
1288llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1289 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1290 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1291 const RegionCodeGenTy &CodeGen) {
1292 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_teams);
1293 llvm::Function *OutlinedFn = emitParallelOrTeamsOutlinedFunction(
1294 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1295 CodeGen);
1296 // A teams body is called once per team and is not handed back to the runtime
1297 // as a callback, so unlike a parallel body it cannot be re-entered while a
1298 // call to it is live.
1299 OutlinedFn->setDoesNotRecurse();
1300 return OutlinedFn;
1301}
1302
1303llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1304 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1305 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1306 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1307 bool Tied, unsigned &NumberOfParts) {
1308 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1309 PrePostActionTy &) {
1310 llvm::Value *ThreadID = getThreadID(CGF, Loc: D.getBeginLoc());
1311 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
1312 llvm::Value *TaskArgs[] = {
1313 UpLoc, ThreadID,
1314 CGF.EmitLoadOfPointerLValue(Ptr: CGF.GetAddrOfLocalVar(VD: TaskTVar),
1315 PtrTy: TaskTVar->getType()->castAs<PointerType>())
1316 .getPointer(CGF)};
1317 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1318 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
1319 args: TaskArgs);
1320 };
1321 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1322 UntiedCodeGen);
1323 CodeGen.setAction(Action);
1324 assert(!ThreadIDVar->getType()->isPointerType() &&
1325 "thread id variable must be of type kmp_int32 for tasks");
1326 const OpenMPDirectiveKind Region =
1327 isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) ? OMPD_taskloop
1328 : OMPD_task;
1329 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: Region);
1330 bool HasCancel = false;
1331 if (const auto *TD = dyn_cast<OMPTaskDirective>(Val: &D))
1332 HasCancel = TD->hasCancel();
1333 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(Val: &D))
1334 HasCancel = TD->hasCancel();
1335 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(Val: &D))
1336 HasCancel = TD->hasCancel();
1337 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(Val: &D))
1338 HasCancel = TD->hasCancel();
1339
1340 CodeGenFunction CGF(CGM, true);
1341 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1342 InnermostKind, HasCancel, Action);
1343 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1344 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(S: *CS);
1345 if (!Tied)
1346 NumberOfParts = Action.getNumberOfParts();
1347 return Res;
1348}
1349
1350void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1351 bool AtCurrentPoint) {
1352 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1353 assert(!Elem.ServiceInsertPt && "Insert point is set already.");
1354
1355 llvm::Value *Undef = llvm::UndefValue::get(T: CGF.Int32Ty);
1356 if (AtCurrentPoint) {
1357 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",
1358 CGF.Builder.GetInsertBlock());
1359 } else {
1360 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1361 Elem.ServiceInsertPt->insertAfter(InsertPos: CGF.AllocaInsertPt->getIterator());
1362 }
1363}
1364
1365void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1366 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1367 if (Elem.ServiceInsertPt) {
1368 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1369 Elem.ServiceInsertPt = nullptr;
1370 Ptr->eraseFromParent();
1371 }
1372}
1373
1374static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1375 SourceLocation Loc,
1376 SmallString<128> &Buffer) {
1377 llvm::raw_svector_ostream OS(Buffer);
1378 // Build debug location
1379 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1380 OS << ";";
1381 if (auto *DbgInfo = CGF.getDebugInfo())
1382 OS << DbgInfo->remapDIPath(PLoc.getFilename());
1383 else
1384 OS << PLoc.getFilename();
1385 OS << ";";
1386 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1387 OS << FD->getQualifiedNameAsString();
1388 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1389 return OS.str();
1390}
1391
1392llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1393 SourceLocation Loc,
1394 unsigned Flags, bool EmitLoc) {
1395 uint32_t SrcLocStrSize;
1396 llvm::Constant *SrcLocStr;
1397 if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() ==
1398 llvm::codegenoptions::NoDebugInfo) ||
1399 Loc.isInvalid()) {
1400 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1401 } else {
1402 std::string FunctionName;
1403 std::string FileName;
1404 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1405 FunctionName = FD->getQualifiedNameAsString();
1406 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1407 if (auto *DbgInfo = CGF.getDebugInfo())
1408 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
1409 else
1410 FileName = PLoc.getFilename();
1411 unsigned Line = PLoc.getLine();
1412 unsigned Column = PLoc.getColumn();
1413 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1414 Column, SrcLocStrSize);
1415 }
1416 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1417 return OMPBuilder.getOrCreateIdent(
1418 SrcLocStr, SrcLocStrSize, Flags: llvm::omp::IdentFlag(Flags), Reserve2Flags: Reserved2Flags);
1419}
1420
1421llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1422 SourceLocation Loc) {
1423 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1424 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1425 // the clang invariants used below might be broken.
1426 if (CGM.getLangOpts().OpenMPIRBuilder) {
1427 SmallString<128> Buffer;
1428 OMPBuilder.updateToLocation(Loc: CGF.Builder);
1429 uint32_t SrcLocStrSize;
1430 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1431 LocStr: getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1432 return OMPBuilder.getOrCreateThreadID(
1433 Ident: OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1434 }
1435
1436 llvm::Value *ThreadID = nullptr;
1437 // Check whether we've already cached a load of the thread id in this
1438 // function.
1439 auto I = OpenMPLocThreadIDMap.find(Val: CGF.CurFn);
1440 if (I != OpenMPLocThreadIDMap.end()) {
1441 ThreadID = I->second.ThreadID;
1442 if (ThreadID != nullptr)
1443 return ThreadID;
1444 }
1445 // If exceptions are enabled, do not use parameter to avoid possible crash.
1446 if (auto *OMPRegionInfo =
1447 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
1448 if (OMPRegionInfo->getThreadIDVariable()) {
1449 // Check if this an outlined function with thread id passed as argument.
1450 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1451 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1452 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1453 !CGF.getLangOpts().CXXExceptions ||
1454 CGF.Builder.GetInsertBlock() == TopBlock ||
1455 !isa<llvm::Instruction>(Val: LVal.getPointer(CGF)) ||
1456 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1457 TopBlock ||
1458 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1459 CGF.Builder.GetInsertBlock()) {
1460 ThreadID = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
1461 // If value loaded in entry block, cache it and use it everywhere in
1462 // function.
1463 if (CGF.Builder.GetInsertBlock() == TopBlock)
1464 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;
1465 return ThreadID;
1466 }
1467 }
1468 }
1469
1470 // This is not an outlined function region - need to call __kmpc_int32
1471 // kmpc_global_thread_num(ident_t *loc).
1472 // Generate thread id value and cache this value for use across the
1473 // function.
1474 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1475 if (!Elem.ServiceInsertPt)
1476 setLocThreadIdInsertPt(CGF);
1477 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1478 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);
1479 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
1480 llvm::CallInst *Call = CGF.Builder.CreateCall(
1481 Callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
1482 FnID: OMPRTL___kmpc_global_thread_num),
1483 Args: emitUpdateLocation(CGF, Loc));
1484 Call->setCallingConv(CGF.getRuntimeCC());
1485 Elem.ThreadID = Call;
1486 return Call;
1487}
1488
1489void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1490 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1491 if (OpenMPLocThreadIDMap.count(Val: CGF.CurFn)) {
1492 clearLocThreadIdInsertPt(CGF);
1493 OpenMPLocThreadIDMap.erase(Val: CGF.CurFn);
1494 }
1495 if (auto I = FunctionUDRMap.find(Val: CGF.CurFn); I != FunctionUDRMap.end()) {
1496 for (const auto *D : I->second)
1497 UDRMap.erase(Val: D);
1498 FunctionUDRMap.erase(I);
1499 }
1500 if (auto I = FunctionUDMMap.find(Val: CGF.CurFn); I != FunctionUDMMap.end()) {
1501 for (const auto *D : I->second)
1502 UDMMap.erase(Val: D);
1503 FunctionUDMMap.erase(I);
1504 }
1505 LastprivateConditionalToTypes.erase(Val: CGF.CurFn);
1506 FunctionToUntiedTaskStackMap.erase(Val: CGF.CurFn);
1507}
1508
1509llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1510 return OMPBuilder.IdentPtr;
1511}
1512
1513static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1514convertDeviceClause(const VarDecl *VD) {
1515 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1516 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1517 if (!DevTy)
1518 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1519
1520 switch ((int)*DevTy) { // Avoid -Wcovered-switch-default
1521 case OMPDeclareTargetDeclAttr::DT_Host:
1522 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1523 break;
1524 case OMPDeclareTargetDeclAttr::DT_NoHost:
1525 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1526 break;
1527 case OMPDeclareTargetDeclAttr::DT_Any:
1528 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1529 break;
1530 default:
1531 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1532 break;
1533 }
1534}
1535
1536static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1537convertCaptureClause(const VarDecl *VD) {
1538 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1539 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1540 if (!MapType)
1541 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1542 switch ((int)*MapType) { // Avoid -Wcovered-switch-default
1543 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1544 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1545 break;
1546 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1547 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1548 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1549 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1550 break;
1551 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local:
1552 // MT_Local variables don't need offload entry (device-local).
1553 llvm_unreachable("MT_Local should not reach convertCaptureClause");
1554 break;
1555 default:
1556 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1557 break;
1558 }
1559}
1560
1561static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
1562 CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder,
1563 SourceLocation BeginLoc, llvm::StringRef ParentName = "") {
1564
1565 auto FileInfoCallBack = [&]() {
1566 SourceManager &SM = CGM.getContext().getSourceManager();
1567 PresumedLoc PLoc = SM.getPresumedLoc(Loc: BeginLoc);
1568
1569 if (!CGM.getFileSystem()->exists(Path: PLoc.getFilename()))
1570 PLoc = SM.getPresumedLoc(Loc: BeginLoc, /*UseLineDirectives=*/false);
1571
1572 return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine());
1573 };
1574
1575 return OMPBuilder.getTargetEntryUniqueInfo(CallBack: FileInfoCallBack,
1576 VFS&: *CGM.getFileSystem(), ParentName);
1577}
1578
1579ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1580 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
1581
1582 auto LinkageForVariable = [&VD, this]() {
1583 return CGM.getLLVMLinkageVarDefinition(VD);
1584 };
1585
1586 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1587
1588 llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(
1589 T: CGM.getContext().getPointerType(T: VD->getType()));
1590 llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar(
1591 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
1592 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
1593 IsExternallyVisible: VD->isExternallyVisible(),
1594 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
1595 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
1596 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
1597 TargetTriple: CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, GlobalInitializer: AddrOfGlobal,
1598 VariableLinkage: LinkageForVariable);
1599
1600 if (!addr)
1601 return ConstantAddress::invalid();
1602 return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(D: VD));
1603}
1604
1605llvm::Constant *
1606CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1607 assert(!CGM.getLangOpts().OpenMPUseTLS ||
1608 !CGM.getContext().getTargetInfo().isTLSSupported());
1609 // Lookup the entry, lazily creating it if necessary.
1610 std::string Suffix = getName(Parts: {"cache", ""});
1611 return OMPBuilder.getOrCreateInternalVariable(
1612 Ty: CGM.Int8PtrPtrTy, Name: Twine(CGM.getMangledName(GD: VD)).concat(Suffix).str());
1613}
1614
1615Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1616 const VarDecl *VD,
1617 Address VDAddr,
1618 SourceLocation Loc) {
1619 if (CGM.getLangOpts().OpenMPUseTLS &&
1620 CGM.getContext().getTargetInfo().isTLSSupported())
1621 return VDAddr;
1622
1623 llvm::Type *VarTy = VDAddr.getElementType();
1624 llvm::Value *Args[] = {
1625 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1626 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy),
1627 CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy)),
1628 getOrCreateThreadPrivateCache(VD)};
1629 return Address(
1630 CGF.EmitRuntimeCall(
1631 callee: OMPBuilder.getOrCreateRuntimeFunction(
1632 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1633 args: Args),
1634 CGF.Int8Ty, VDAddr.getAlignment());
1635}
1636
1637void CGOpenMPRuntime::emitThreadPrivateVarInit(
1638 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1639 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1640 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1641 // library.
1642 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1643 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1644 M&: CGM.getModule(), FnID: OMPRTL___kmpc_global_thread_num),
1645 args: OMPLoc);
1646 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1647 // to register constructor/destructor for variable.
1648 llvm::Value *Args[] = {
1649 OMPLoc,
1650 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy),
1651 Ctor, CopyCtor, Dtor};
1652 CGF.EmitRuntimeCall(
1653 callee: OMPBuilder.getOrCreateRuntimeFunction(
1654 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_register),
1655 args: Args);
1656}
1657
1658llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1659 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1660 bool PerformInit, CodeGenFunction *CGF) {
1661 if (CGM.getLangOpts().OpenMPUseTLS &&
1662 CGM.getContext().getTargetInfo().isTLSSupported())
1663 return nullptr;
1664
1665 VD = VD->getDefinition(C&: CGM.getContext());
1666 if (VD && ThreadPrivateWithDefinition.insert(key: CGM.getMangledName(GD: VD)).second) {
1667 QualType ASTTy = VD->getType();
1668
1669 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1670 const Expr *Init = VD->getAnyInitializer();
1671 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1672 // Generate function that re-emits the declaration's initializer into the
1673 // threadprivate copy of the variable VD
1674 CodeGenFunction CtorCGF(CGM);
1675 auto *Dst = ImplicitParamDecl::Create(
1676 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1677 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1678
1679 FunctionArgList Args{Dst};
1680 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1681 resultType: CGM.getContext().VoidPtrTy, args: Args);
1682 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1683 std::string Name = getName(Parts: {"__kmpc_global_ctor_", ""});
1684 llvm::Function *Fn =
1685 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1686 CtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidPtrTy, Fn, FnInfo: FI,
1687 Args, Loc, StartLoc: Loc);
1688 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1689 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1690 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1691 Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(T: ASTTy),
1692 VDAddr.getAlignment());
1693 CtorCGF.EmitAnyExprToMem(E: Init, Location: Arg, Quals: Init->getType().getQualifiers(),
1694 /*IsInitializer=*/true);
1695 ArgVal = CtorCGF.EmitLoadOfScalar(
1696 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1697 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1698 CtorCGF.Builder.CreateStore(Val: ArgVal, Addr: CtorCGF.ReturnValue);
1699 CtorCGF.FinishFunction();
1700 Ctor = Fn;
1701 }
1702 if (VD->getType().isDestructedType() != QualType::DK_none) {
1703 // Generate function that emits destructor call for the threadprivate copy
1704 // of the variable VD
1705 CodeGenFunction DtorCGF(CGM);
1706 auto *Dst = ImplicitParamDecl::Create(
1707 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1708 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1709
1710 FunctionArgList Args{Dst};
1711 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1712 resultType: CGM.getContext().VoidTy, args: Args);
1713 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1714 std::string Name = getName(Parts: {"__kmpc_global_dtor_", ""});
1715 llvm::Function *Fn =
1716 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1717 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: DtorCGF);
1718 DtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn, FnInfo: FI, Args,
1719 Loc, StartLoc: Loc);
1720 // Create a scope with an artificial location for the body of this function.
1721 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: DtorCGF);
1722 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1723 Addr: DtorCGF.GetAddrOfLocalVar(VD: Dst),
1724 /*Volatile=*/false, Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1725 DtorCGF.emitDestroy(
1726 addr: Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), type: ASTTy,
1727 destroyer: DtorCGF.getDestroyer(destructionKind: ASTTy.isDestructedType()),
1728 useEHCleanupForArray: DtorCGF.needsEHCleanup(kind: ASTTy.isDestructedType()));
1729 DtorCGF.FinishFunction();
1730 Dtor = Fn;
1731 }
1732 // Do not emit init function if it is not required.
1733 if (!Ctor && !Dtor)
1734 return nullptr;
1735
1736 // Copying constructor for the threadprivate variable.
1737 // Must be NULL - reserved by runtime, but currently it requires that this
1738 // parameter is always NULL. Otherwise it fires assertion.
1739 CopyCtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1740 if (Ctor == nullptr) {
1741 Ctor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1742 }
1743 if (Dtor == nullptr) {
1744 Dtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1745 }
1746 if (!CGF) {
1747 auto *InitFunctionTy =
1748 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg*/ false);
1749 std::string Name = getName(Parts: {"__omp_threadprivate_init_", ""});
1750 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1751 ty: InitFunctionTy, name: Name, FI: CGM.getTypes().arrangeNullaryFunction());
1752 CodeGenFunction InitCGF(CGM);
1753 FunctionArgList ArgList;
1754 InitCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: InitFunction,
1755 FnInfo: CGM.getTypes().arrangeNullaryFunction(), Args: ArgList,
1756 Loc, StartLoc: Loc);
1757 emitThreadPrivateVarInit(CGF&: InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1758 InitCGF.FinishFunction();
1759 return InitFunction;
1760 }
1761 emitThreadPrivateVarInit(CGF&: *CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1762 }
1763 return nullptr;
1764}
1765
1766void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD,
1767 llvm::GlobalValue *GV) {
1768 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1769 OMPDeclareTargetDeclAttr::getActiveAttr(VD: FD);
1770
1771 // We only need to handle active 'indirect' declare target functions.
1772 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1773 return;
1774
1775 // Get a mangled name to store the new device global in.
1776 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1777 CGM, OMPBuilder, BeginLoc: FD->getCanonicalDecl()->getBeginLoc(), ParentName: FD->getName());
1778 SmallString<128> Name;
1779 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1780
1781 // We need to generate a new global to hold the address of the indirectly
1782 // called device function. Doing this allows us to keep the visibility and
1783 // linkage of the associated function unchanged while allowing the runtime to
1784 // access its value.
1785 llvm::GlobalValue *Addr = GV;
1786 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1787 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1788 C&: CGM.getLLVMContext(),
1789 AddressSpace: CGM.getModule().getDataLayout().getProgramAddressSpace());
1790 Addr = new llvm::GlobalVariable(
1791 CGM.getModule(), FnPtrTy,
1792 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1793 nullptr, llvm::GlobalValue::NotThreadLocal,
1794 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1795 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1796 }
1797
1798 // Register the indirect Vtable:
1799 // This is similar to OMPTargetGlobalVarEntryIndirect, except that the
1800 // size field refers to the size of memory pointed to, not the size of
1801 // the pointer symbol itself (which is implicitly the size of a pointer).
1802 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1803 VarName: Name, Addr, VarSize: CGM.GetTargetTypeStoreSize(Ty: CGM.VoidPtrTy).getQuantity(),
1804 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1805 Linkage: llvm::GlobalValue::WeakODRLinkage);
1806}
1807
1808void CGOpenMPRuntime::registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1809 const VarDecl *VD) {
1810 // TODO: add logic to avoid duplicate vtable registrations per
1811 // translation unit; though for external linkage, this should no
1812 // longer be an issue - or at least we can avoid the issue by
1813 // checking for an existing offloading entry. But, perhaps the
1814 // better approach is to defer emission of the vtables and offload
1815 // entries until later (by tracking a list of items that need to be
1816 // emitted).
1817
1818 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1819
1820 // Generate a new externally visible global to point to the
1821 // internally visible vtable. Doing this allows us to keep the
1822 // visibility and linkage of the associated vtable unchanged while
1823 // allowing the runtime to access its value. The externally
1824 // visible global var needs to be emitted with a unique mangled
1825 // name that won't conflict with similarly named (internal)
1826 // vtables in other translation units.
1827
1828 // Register vtable with source location of dynamic object in map
1829 // clause.
1830 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1831 CGM, OMPBuilder, BeginLoc: VD->getCanonicalDecl()->getBeginLoc(),
1832 ParentName: VTable->getName());
1833
1834 llvm::GlobalVariable *Addr = VTable;
1835 SmallString<128> AddrName;
1836 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name&: AddrName, EntryInfo);
1837 AddrName.append(RHS: "addr");
1838
1839 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1840 Addr = new llvm::GlobalVariable(
1841 CGM.getModule(), VTable->getType(),
1842 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, VTable,
1843 AddrName,
1844 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1845 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1846 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1847 }
1848 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1849 VarName: AddrName, Addr: VTable,
1850 VarSize: CGM.getDataLayout().getTypeAllocSize(Ty: VTable->getInitializer()->getType()),
1851 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable,
1852 Linkage: llvm::GlobalValue::WeakODRLinkage);
1853}
1854
1855void CGOpenMPRuntime::emitAndRegisterVTable(CodeGenModule &CGM,
1856 CXXRecordDecl *CXXRecord,
1857 const VarDecl *VD) {
1858 // Register C++ VTable to OpenMP Offload Entry if it's a new
1859 // CXXRecordDecl.
1860 if (CXXRecord && CXXRecord->hasDefinition() && CXXRecord->isDynamicClass() &&
1861 !CGM.getOpenMPRuntime().VTableDeclMap.contains(Val: CXXRecord)) {
1862 auto Res = CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(Key: CXXRecord, Args&: VD);
1863 if (Res.second) {
1864 CGM.EmitVTable(Class: CXXRecord);
1865 CodeGenVTables VTables = CGM.getVTables();
1866 llvm::GlobalVariable *VTablesAddr = VTables.GetAddrOfVTable(RD: CXXRecord);
1867 assert(VTablesAddr && "Expected non-null VTable address");
1868 // Must set VTables to weak since we're emitting them in multiple TUs now
1869 if (VTablesAddr->hasExternalLinkage())
1870 VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1871 CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTable: VTablesAddr, VD);
1872 // Emit VTable for all the fields containing dynamic CXXRecord
1873 for (const FieldDecl *Field : CXXRecord->fields()) {
1874 if (CXXRecordDecl *RecordDecl = Field->getType()->getAsCXXRecordDecl())
1875 emitAndRegisterVTable(CGM, CXXRecord: RecordDecl, VD);
1876 }
1877 // Emit VTable for all dynamic parent class
1878 for (CXXBaseSpecifier &Base : CXXRecord->bases()) {
1879 if (CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl())
1880 emitAndRegisterVTable(CGM, CXXRecord: BaseDecl, VD);
1881 }
1882 }
1883 }
1884}
1885
1886void CGOpenMPRuntime::registerVTable(const OMPExecutableDirective &D) {
1887 // Register VTable by scanning through the map clause of OpenMP target region.
1888 // Get CXXRecordDecl and VarDecl from Expr.
1889 auto GetVTableDecl = [](const Expr *E) {
1890 QualType VDTy = E->getType();
1891 CXXRecordDecl *CXXRecord = nullptr;
1892 if (const auto *RefType = VDTy->getAs<LValueReferenceType>())
1893 VDTy = RefType->getPointeeType();
1894 if (VDTy->isPointerType())
1895 CXXRecord = VDTy->getPointeeType()->getAsCXXRecordDecl();
1896 else
1897 CXXRecord = VDTy->getAsCXXRecordDecl();
1898
1899 const VarDecl *VD = nullptr;
1900 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1901 // Handle BindingDecls by redirecting to their DecompositionDecl.
1902 if (auto *BD = dyn_cast<BindingDecl>(Val: DRE->getDecl()))
1903 VD = cast<VarDecl>(Val: BD->getDecomposedDecl());
1904 else
1905 VD = cast<VarDecl>(Val: DRE->getDecl());
1906 } else if (auto *MRE = dyn_cast<MemberExpr>(Val: E)) {
1907 if (auto *BaseDRE = dyn_cast<DeclRefExpr>(Val: MRE->getBase())) {
1908 if (auto *BaseVD = dyn_cast<VarDecl>(Val: BaseDRE->getDecl()))
1909 VD = BaseVD;
1910 }
1911 }
1912 return std::pair<CXXRecordDecl *, const VarDecl *>(CXXRecord, VD);
1913 };
1914 // Collect VTable from OpenMP map clause.
1915 for (const auto *C : D.getClausesOfKind<OMPMapClause>()) {
1916 for (const auto *E : C->varlist()) {
1917 auto DeclPair = GetVTableDecl(E);
1918 // Ensure VD is not null
1919 if (DeclPair.second)
1920 emitAndRegisterVTable(CGM, CXXRecord: DeclPair.first, VD: DeclPair.second);
1921 }
1922 }
1923}
1924
1925Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1926 QualType VarType,
1927 StringRef Name) {
1928 std::string Suffix = getName(Parts: {"artificial", ""});
1929 llvm::Type *VarLVType = CGF.ConvertTypeForMem(T: VarType);
1930 llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable(
1931 Ty: VarLVType, Name: Twine(Name).concat(Suffix).str());
1932 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1933 CGM.getTarget().isTLSSupported()) {
1934 GAddr->setThreadLocal(/*Val=*/true);
1935 return Address(GAddr, GAddr->getValueType(),
1936 CGM.getContext().getTypeAlignInChars(T: VarType));
1937 }
1938 std::string CacheSuffix = getName(Parts: {"cache", ""});
1939 llvm::Value *Args[] = {
1940 emitUpdateLocation(CGF, Loc: SourceLocation()),
1941 getThreadID(CGF, Loc: SourceLocation()),
1942 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: GAddr, DestTy: CGM.VoidPtrTy),
1943 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: VarType), DestTy: CGM.SizeTy,
1944 /*isSigned=*/false),
1945 OMPBuilder.getOrCreateInternalVariable(
1946 Ty: CGM.VoidPtrPtrTy,
1947 Name: Twine(Name).concat(Suffix).concat(Suffix: CacheSuffix).str())};
1948 return Address(
1949 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1950 V: CGF.EmitRuntimeCall(
1951 callee: OMPBuilder.getOrCreateRuntimeFunction(
1952 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1953 args: Args),
1954 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
1955 VarLVType, CGM.getContext().getTypeAlignInChars(T: VarType));
1956}
1957
1958void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
1959 const RegionCodeGenTy &ThenGen,
1960 const RegionCodeGenTy &ElseGen) {
1961 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1962
1963 // If the condition constant folds and can be elided, try to avoid emitting
1964 // the condition and the dead arm of the if/else.
1965 bool CondConstant;
1966 if (CGF.ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant)) {
1967 if (CondConstant)
1968 ThenGen(CGF);
1969 else
1970 ElseGen(CGF);
1971 return;
1972 }
1973
1974 // Otherwise, the condition did not fold, or we couldn't elide it. Just
1975 // emit the conditional branch.
1976 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
1977 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock(name: "omp_if.else");
1978 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "omp_if.end");
1979 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, /*TrueCount=*/0);
1980
1981 // Emit the 'then' code.
1982 CGF.EmitBlock(BB: ThenBlock);
1983 ThenGen(CGF);
1984 CGF.EmitBranch(Block: ContBlock);
1985 // Emit the 'else' code if present.
1986 // There is no need to emit line number for unconditional branch.
1987 (void)ApplyDebugLocation::CreateEmpty(CGF);
1988 CGF.EmitBlock(BB: ElseBlock);
1989 ElseGen(CGF);
1990 // There is no need to emit line number for unconditional branch.
1991 (void)ApplyDebugLocation::CreateEmpty(CGF);
1992 CGF.EmitBranch(Block: ContBlock);
1993 // Emit the continuation block for code after the if.
1994 CGF.EmitBlock(BB: ContBlock, /*IsFinished=*/true);
1995}
1996
1997void CGOpenMPRuntime::emitParallelCall(
1998 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
1999 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
2000 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
2001 OpenMPSeverityClauseKind Severity, const Expr *Message) {
2002 if (!CGF.HaveInsertPoint())
2003 return;
2004
2005 // The caller waits for the parallel region to finish, so its captured stack
2006 // variables remain alive until the region completes. Global variables also
2007 // remain alive. Record this lifetime guarantee on the outlined function's
2008 // parameters, even if other code has access to the same storage.
2009 // The storage may still be modified or accessed through other pointers.
2010 // For a captured pointer variable, this guarantees the lifetime of the
2011 // variable itself, not the object it points to.
2012 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
2013 assert(OutlinedFn->arg_size() == CapturedVars.size() + 2 &&
2014 "Expected tid and bound tid before the captured arguments");
2015 for (auto [I, Captured] : llvm::enumerate(First&: CapturedVars)) {
2016 if (Captured->getType()->isPointerTy() &&
2017 llvm::isa<llvm::AllocaInst, llvm::GlobalVariable>(
2018 Val: llvm::getUnderlyingObject(V: Captured)))
2019 OutlinedFn->addParamAttr(ArgNo: I + 2, Kind: llvm::Attribute::NoFreeObj);
2020 }
2021 }
2022
2023 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2024 auto &M = CGM.getModule();
2025 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2026 this](CodeGenFunction &CGF, PrePostActionTy &) {
2027 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2028 llvm::Value *Args[] = {
2029 RTLoc,
2030 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
2031 OutlinedFn};
2032 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2033 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
2034 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2035
2036 llvm::FunctionCallee RTLFn =
2037 OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_fork_call);
2038 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
2039 };
2040 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2041 this](CodeGenFunction &CGF, PrePostActionTy &) {
2042 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2043 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2044 // Build calls:
2045 // __kmpc_serialized_parallel(&Loc, GTid);
2046 llvm::Value *Args[] = {RTLoc, ThreadID};
2047 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2048 M, FnID: OMPRTL___kmpc_serialized_parallel),
2049 args: Args);
2050
2051 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2052 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2053 RawAddress ZeroAddrBound =
2054 CGF.CreateDefaultAlignTempAlloca(Ty: CGF.Int32Ty,
2055 /*Name=*/".bound.zero.addr");
2056 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(/*C*/ 0), Addr: ZeroAddrBound);
2057 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2058 // ThreadId for serialized parallels is 0.
2059 OutlinedFnArgs.push_back(Elt: ThreadIDAddr.emitRawPointer(CGF));
2060 OutlinedFnArgs.push_back(Elt: ZeroAddrBound.getPointer());
2061 OutlinedFnArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2062
2063 // Ensure we do not inline the function. This is trivially true for the ones
2064 // passed to __kmpc_fork_call but the ones called in serialized regions
2065 // could be inlined. This is not a perfect but it is closer to the invariant
2066 // we want, namely, every data environment starts with a new function.
2067 // TODO: We should pass the if condition to the runtime function and do the
2068 // handling there. Much cleaner code.
2069 OutlinedFn->removeFnAttr(Kind: llvm::Attribute::AlwaysInline);
2070 OutlinedFn->addFnAttr(Kind: llvm::Attribute::NoInline);
2071 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, Args: OutlinedFnArgs);
2072
2073 // __kmpc_end_serialized_parallel(&Loc, GTid);
2074 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2075 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2076 M, FnID: OMPRTL___kmpc_end_serialized_parallel),
2077 args: EndArgs);
2078 };
2079 if (IfCond) {
2080 emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
2081 } else {
2082 RegionCodeGenTy ThenRCG(ThenGen);
2083 ThenRCG(CGF);
2084 }
2085}
2086
2087// If we're inside an (outlined) parallel region, use the region info's
2088// thread-ID variable (it is passed in a first argument of the outlined function
2089// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2090// regular serial code region, get thread ID by calling kmp_int32
2091// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2092// return the address of that temp.
2093Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2094 SourceLocation Loc) {
2095 if (auto *OMPRegionInfo =
2096 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2097 if (OMPRegionInfo->getThreadIDVariable())
2098 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2099
2100 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2101 QualType Int32Ty =
2102 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2103 Address ThreadIDTemp =
2104 CGF.CreateMemTempWithoutCast(T: Int32Ty, /*Name*/ ".threadid_temp.");
2105 CGF.EmitStoreOfScalar(value: ThreadID,
2106 lvalue: CGF.MakeAddrLValue(Addr: ThreadIDTemp, T: Int32Ty));
2107
2108 return ThreadIDTemp;
2109}
2110
2111llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2112 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2113 std::string Name = getName(Parts: {Prefix, "var"});
2114 llvm::GlobalVariable *GV =
2115 OMPBuilder.getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
2116 CGM.setDSOLocal(GV);
2117 return GV;
2118}
2119
2120namespace {
2121/// Common pre(post)-action for different OpenMP constructs.
2122class CommonActionTy final : public PrePostActionTy {
2123 llvm::FunctionCallee EnterCallee;
2124 ArrayRef<llvm::Value *> EnterArgs;
2125 llvm::FunctionCallee ExitCallee;
2126 ArrayRef<llvm::Value *> ExitArgs;
2127 bool Conditional;
2128 llvm::BasicBlock *ContBlock = nullptr;
2129
2130public:
2131 CommonActionTy(llvm::FunctionCallee EnterCallee,
2132 ArrayRef<llvm::Value *> EnterArgs,
2133 llvm::FunctionCallee ExitCallee,
2134 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2135 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2136 ExitArgs(ExitArgs), Conditional(Conditional) {}
2137 void Enter(CodeGenFunction &CGF) override {
2138 llvm::Value *EnterRes = CGF.EmitRuntimeCall(callee: EnterCallee, args: EnterArgs);
2139 if (Conditional) {
2140 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(Arg: EnterRes);
2141 auto *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
2142 ContBlock = CGF.createBasicBlock(name: "omp_if.end");
2143 // Generate the branch (If-stmt)
2144 CGF.Builder.CreateCondBr(Cond: CallBool, True: ThenBlock, False: ContBlock);
2145 CGF.EmitBlock(BB: ThenBlock);
2146 }
2147 }
2148 void Done(CodeGenFunction &CGF) {
2149 // Emit the rest of blocks/branches
2150 CGF.EmitBranch(Block: ContBlock);
2151 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
2152 }
2153 void Exit(CodeGenFunction &CGF) override {
2154 CGF.EmitRuntimeCall(callee: ExitCallee, args: ExitArgs);
2155 }
2156};
2157} // anonymous namespace
2158
2159void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2160 StringRef CriticalName,
2161 const RegionCodeGenTy &CriticalOpGen,
2162 SourceLocation Loc, const Expr *Hint) {
2163 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2164 // CriticalOpGen();
2165 // __kmpc_end_critical(ident_t *, gtid, Lock);
2166 // Prepare arguments and build a call to __kmpc_critical
2167 if (!CGF.HaveInsertPoint())
2168 return;
2169 llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction(
2170 M&: CGM.getModule(),
2171 FnID: Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);
2172 llvm::Value *LockVar = getCriticalRegionLock(CriticalName);
2173 unsigned LockVarArgIdx = 2;
2174 if (cast<llvm::GlobalVariable>(Val: LockVar)->getAddressSpace() !=
2175 RuntimeFcn.getFunctionType()
2176 ->getParamType(i: LockVarArgIdx)
2177 ->getPointerAddressSpace())
2178 LockVar = CGF.Builder.CreateAddrSpaceCast(
2179 V: LockVar, DestTy: RuntimeFcn.getFunctionType()->getParamType(i: LockVarArgIdx));
2180 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2181 LockVar};
2182 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args),
2183 std::end(arr&: Args));
2184 if (Hint) {
2185 EnterArgs.push_back(Elt: CGF.Builder.CreateIntCast(
2186 V: CGF.EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, /*isSigned=*/false));
2187 }
2188 CommonActionTy Action(RuntimeFcn, EnterArgs,
2189 OMPBuilder.getOrCreateRuntimeFunction(
2190 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_critical),
2191 Args);
2192 CriticalOpGen.setAction(Action);
2193 emitInlinedDirective(CGF, InnermostKind: OMPD_critical, CodeGen: CriticalOpGen);
2194}
2195
2196void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2197 const RegionCodeGenTy &MasterOpGen,
2198 SourceLocation Loc) {
2199 if (!CGF.HaveInsertPoint())
2200 return;
2201 // if(__kmpc_master(ident_t *, gtid)) {
2202 // MasterOpGen();
2203 // __kmpc_end_master(ident_t *, gtid);
2204 // }
2205 // Prepare arguments and build a call to __kmpc_master
2206 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2207 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2208 M&: CGM.getModule(), FnID: OMPRTL___kmpc_master),
2209 Args,
2210 OMPBuilder.getOrCreateRuntimeFunction(
2211 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_master),
2212 Args,
2213 /*Conditional=*/true);
2214 MasterOpGen.setAction(Action);
2215 emitInlinedDirective(CGF, InnermostKind: OMPD_master, CodeGen: MasterOpGen);
2216 Action.Done(CGF);
2217}
2218
2219void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,
2220 const RegionCodeGenTy &MaskedOpGen,
2221 SourceLocation Loc, const Expr *Filter) {
2222 if (!CGF.HaveInsertPoint())
2223 return;
2224 // if(__kmpc_masked(ident_t *, gtid, filter)) {
2225 // MaskedOpGen();
2226 // __kmpc_end_masked(iden_t *, gtid);
2227 // }
2228 // Prepare arguments and build a call to __kmpc_masked
2229 llvm::Value *FilterVal = Filter
2230 ? CGF.EmitScalarExpr(E: Filter, IgnoreResultAssign: CGF.Int32Ty)
2231 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
2232 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2233 FilterVal};
2234 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2235 getThreadID(CGF, Loc)};
2236 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2237 M&: CGM.getModule(), FnID: OMPRTL___kmpc_masked),
2238 Args,
2239 OMPBuilder.getOrCreateRuntimeFunction(
2240 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_masked),
2241 ArgsEnd,
2242 /*Conditional=*/true);
2243 MaskedOpGen.setAction(Action);
2244 emitInlinedDirective(CGF, InnermostKind: OMPD_masked, CodeGen: MaskedOpGen);
2245 Action.Done(CGF);
2246}
2247
2248void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2249 SourceLocation Loc) {
2250 if (!CGF.HaveInsertPoint())
2251 return;
2252 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2253 OMPBuilder.createTaskyield(Loc: CGF.Builder);
2254 } else {
2255 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2256 llvm::Value *Args[] = {
2257 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2258 llvm::ConstantInt::get(Ty: CGM.IntTy, /*V=*/0, /*isSigned=*/IsSigned: true)};
2259 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2260 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_taskyield),
2261 args: Args);
2262 }
2263
2264 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2265 Region->emitUntiedSwitch(CGF);
2266}
2267
2268void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2269 const RegionCodeGenTy &TaskgroupOpGen,
2270 SourceLocation Loc) {
2271 if (!CGF.HaveInsertPoint())
2272 return;
2273 // __kmpc_taskgroup(ident_t *, gtid);
2274 // TaskgroupOpGen();
2275 // __kmpc_end_taskgroup(ident_t *, gtid);
2276 // Prepare arguments and build a call to __kmpc_taskgroup
2277 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2278 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2279 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskgroup),
2280 Args,
2281 OMPBuilder.getOrCreateRuntimeFunction(
2282 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_taskgroup),
2283 Args);
2284 TaskgroupOpGen.setAction(Action);
2285 emitInlinedDirective(CGF, InnermostKind: OMPD_taskgroup, CodeGen: TaskgroupOpGen);
2286}
2287
2288/// Given an array of pointers to variables, project the address of a
2289/// given variable.
2290static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2291 unsigned Index, const VarDecl *Var) {
2292 // Pull out the pointer to the variable.
2293 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Addr: Array, Index);
2294 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: PtrAddr);
2295
2296 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: Var->getType());
2297 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(D: Var));
2298}
2299
2300static llvm::Value *emitCopyprivateCopyFunction(
2301 CodeGenModule &CGM, llvm::Type *ArgsElemType,
2302 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2303 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2304 SourceLocation Loc) {
2305 ASTContext &C = CGM.getContext();
2306 // void copy_func(void *LHSArg, void *RHSArg);
2307
2308 auto *LHSArg =
2309 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2310 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2311 auto *RHSArg =
2312 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2313 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2314 FunctionArgList Args{LHSArg, RHSArg};
2315 const auto &CGFI =
2316 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
2317 std::string Name =
2318 CGM.getOpenMPRuntime().getName(Parts: {"omp", "copyprivate", "copy_func"});
2319 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
2320 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
2321 M: &CGM.getModule());
2322 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
2323 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
2324 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
2325 Fn->setDoesNotRecurse();
2326 CodeGenFunction CGF(CGM);
2327 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
2328 // Dest = (void*[n])(LHSArg);
2329 // Src = (void*[n])(RHSArg);
2330 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2331 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
2332 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2333 ArgsElemType, CGF.getPointerAlign());
2334 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2335 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
2336 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2337 ArgsElemType, CGF.getPointerAlign());
2338 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2339 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2340 // ...
2341 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2342 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2343 const auto *DestVar =
2344 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: DestExprs[I])->getDecl());
2345 Address DestAddr = emitAddrOfVarFromArray(CGF, Array: LHS, Index: I, Var: DestVar);
2346
2347 const auto *SrcVar =
2348 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: SrcExprs[I])->getDecl());
2349 Address SrcAddr = emitAddrOfVarFromArray(CGF, Array: RHS, Index: I, Var: SrcVar);
2350
2351 const auto *VD = cast<DeclRefExpr>(Val: CopyprivateVars[I])->getDecl();
2352 QualType Type = VD->getType();
2353 CGF.EmitOMPCopy(OriginalType: Type, DestAddr, SrcAddr, DestVD: DestVar, SrcVD: SrcVar, Copy: AssignmentOps[I]);
2354 }
2355 CGF.FinishFunction();
2356 return Fn;
2357}
2358
2359void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2360 const RegionCodeGenTy &SingleOpGen,
2361 SourceLocation Loc,
2362 ArrayRef<const Expr *> CopyprivateVars,
2363 ArrayRef<const Expr *> SrcExprs,
2364 ArrayRef<const Expr *> DstExprs,
2365 ArrayRef<const Expr *> AssignmentOps) {
2366 if (!CGF.HaveInsertPoint())
2367 return;
2368 assert(CopyprivateVars.size() == SrcExprs.size() &&
2369 CopyprivateVars.size() == DstExprs.size() &&
2370 CopyprivateVars.size() == AssignmentOps.size());
2371 ASTContext &C = CGM.getContext();
2372 // int32 did_it = 0;
2373 // if(__kmpc_single(ident_t *, gtid)) {
2374 // SingleOpGen();
2375 // __kmpc_end_single(ident_t *, gtid);
2376 // did_it = 1;
2377 // }
2378 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2379 // <copy_func>, did_it);
2380
2381 Address DidIt = Address::invalid();
2382 if (!CopyprivateVars.empty()) {
2383 // int32 did_it = 0;
2384 QualType KmpInt32Ty =
2385 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2386 DidIt = CGF.CreateMemTempWithoutCast(T: KmpInt32Ty, Name: ".omp.copyprivate.did_it");
2387 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 0), Addr: DidIt);
2388 }
2389 // Prepare arguments and build a call to __kmpc_single
2390 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2391 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2392 M&: CGM.getModule(), FnID: OMPRTL___kmpc_single),
2393 Args,
2394 OMPBuilder.getOrCreateRuntimeFunction(
2395 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_single),
2396 Args,
2397 /*Conditional=*/true);
2398 SingleOpGen.setAction(Action);
2399 emitInlinedDirective(CGF, InnermostKind: OMPD_single, CodeGen: SingleOpGen);
2400 if (DidIt.isValid()) {
2401 // did_it = 1;
2402 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 1), Addr: DidIt);
2403 }
2404 Action.Done(CGF);
2405 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2406 // <copy_func>, did_it);
2407 if (DidIt.isValid()) {
2408 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2409 QualType CopyprivateArrayTy = C.getConstantArrayType(
2410 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
2411 /*IndexTypeQuals=*/0);
2412 // Create a list of all private variables for copyprivate.
2413 Address CopyprivateList = CGF.CreateMemTempWithoutCast(
2414 T: CopyprivateArrayTy, Name: ".omp.copyprivate.cpr_list");
2415 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2416 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: CopyprivateList, Index: I);
2417 CGF.Builder.CreateStore(
2418 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2419 V: CGF.EmitLValue(E: CopyprivateVars[I]).getPointer(CGF),
2420 DestTy: CGF.VoidPtrTy),
2421 Addr: Elem);
2422 }
2423 // Build function that copies private values from single region to all other
2424 // threads in the corresponding parallel region.
2425 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2426 CGM, ArgsElemType: CGF.ConvertTypeForMem(T: CopyprivateArrayTy), CopyprivateVars,
2427 DestExprs: SrcExprs, SrcExprs: DstExprs, AssignmentOps, Loc);
2428 llvm::Value *BufSize = CGF.getTypeSize(Ty: CopyprivateArrayTy);
2429 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2430 Addr: CopyprivateList, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
2431 llvm::Value *DidItVal = CGF.Builder.CreateLoad(Addr: DidIt);
2432 llvm::Value *Args[] = {
2433 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2434 getThreadID(CGF, Loc), // i32 <gtid>
2435 BufSize, // size_t <buf_size>
2436 CL.emitRawPointer(CGF), // void *<copyprivate list>
2437 CpyFn, // void (*) (void *, void *) <copy_func>
2438 DidItVal // i32 did_it
2439 };
2440 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2441 M&: CGM.getModule(), FnID: OMPRTL___kmpc_copyprivate),
2442 args: Args);
2443 }
2444}
2445
2446void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2447 const RegionCodeGenTy &OrderedOpGen,
2448 SourceLocation Loc, bool IsThreads) {
2449 if (!CGF.HaveInsertPoint())
2450 return;
2451 // __kmpc_ordered(ident_t *, gtid);
2452 // OrderedOpGen();
2453 // __kmpc_end_ordered(ident_t *, gtid);
2454 // Prepare arguments and build a call to __kmpc_ordered
2455 if (IsThreads) {
2456 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2457 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2458 M&: CGM.getModule(), FnID: OMPRTL___kmpc_ordered),
2459 Args,
2460 OMPBuilder.getOrCreateRuntimeFunction(
2461 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_ordered),
2462 Args);
2463 OrderedOpGen.setAction(Action);
2464 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2465 return;
2466 }
2467 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2468}
2469
2470unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2471 unsigned Flags;
2472 if (Kind == OMPD_for)
2473 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2474 else if (Kind == OMPD_sections)
2475 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2476 else if (Kind == OMPD_single)
2477 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2478 else if (Kind == OMPD_barrier)
2479 Flags = OMP_IDENT_BARRIER_EXPL;
2480 else
2481 Flags = OMP_IDENT_BARRIER_IMPL;
2482 return Flags;
2483}
2484
2485void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2486 CodeGenFunction &CGF, const OMPLoopDirective &S,
2487 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2488 // Check if the loop directive is actually a doacross loop directive. In this
2489 // case choose static, 1 schedule.
2490 if (llvm::any_of(
2491 Range: S.getClausesOfKind<OMPOrderedClause>(),
2492 P: [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2493 ScheduleKind = OMPC_SCHEDULE_static;
2494 // Chunk size is 1 in this case.
2495 llvm::APInt ChunkSize(32, 1);
2496 ChunkExpr = IntegerLiteral::Create(
2497 C: CGF.getContext(), V: ChunkSize,
2498 type: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
2499 l: SourceLocation());
2500 }
2501}
2502
2503void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2504 OpenMPDirectiveKind Kind, bool EmitChecks,
2505 bool ForceSimpleCall) {
2506 // Check if we should use the OMPBuilder
2507 auto *OMPRegionInfo =
2508 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo);
2509 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2510 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2511 cantFail(ValOrErr: OMPBuilder.createBarrier(Loc: CGF.Builder, Kind, ForceSimpleCall,
2512 CheckCancelFlag: EmitChecks));
2513 CGF.Builder.restoreIP(IP: AfterIP);
2514 return;
2515 }
2516
2517 if (!CGF.HaveInsertPoint())
2518 return;
2519 // Build call __kmpc_cancel_barrier(loc, thread_id);
2520 // Build call __kmpc_barrier(loc, thread_id);
2521 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2522 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2523 // thread_id);
2524 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2525 getThreadID(CGF, Loc)};
2526 if (OMPRegionInfo) {
2527 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2528 llvm::Value *Result = CGF.EmitRuntimeCall(
2529 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
2530 FnID: OMPRTL___kmpc_cancel_barrier),
2531 args: Args);
2532 if (EmitChecks) {
2533 // if (__kmpc_cancel_barrier()) {
2534 // exit from construct;
2535 // }
2536 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
2537 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
2538 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
2539 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
2540 CGF.EmitBlock(BB: ExitBB);
2541 // exit from construct;
2542 CodeGenFunction::JumpDest CancelDestination =
2543 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
2544 CGF.EmitBranchThroughCleanup(Dest: CancelDestination);
2545 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
2546 }
2547 return;
2548 }
2549 }
2550 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2551 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
2552 args: Args);
2553}
2554
2555void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,
2556 Expr *ME, bool IsFatal) {
2557 llvm::Value *MVL = ME ? CGF.EmitScalarExpr(E: ME)
2558 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2559 // Build call void __kmpc_error(ident_t *loc, int severity, const char
2560 // *message)
2561 llvm::Value *Args[] = {
2562 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/EmitLoc: true),
2563 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: IsFatal ? 2 : 1),
2564 CGF.Builder.CreatePointerCast(V: MVL, DestTy: CGM.Int8PtrTy)};
2565 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2566 M&: CGM.getModule(), FnID: OMPRTL___kmpc_error),
2567 args: Args);
2568}
2569
2570/// Map the OpenMP loop schedule to the runtime enumeration.
2571static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2572 bool Chunked, bool Ordered) {
2573 switch (ScheduleKind) {
2574 case OMPC_SCHEDULE_static:
2575 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2576 : (Ordered ? OMP_ord_static : OMP_sch_static);
2577 case OMPC_SCHEDULE_dynamic:
2578 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2579 case OMPC_SCHEDULE_guided:
2580 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2581 case OMPC_SCHEDULE_runtime:
2582 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2583 case OMPC_SCHEDULE_auto:
2584 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2585 case OMPC_SCHEDULE_unknown:
2586 assert(!Chunked && "chunk was specified but schedule kind not known");
2587 return Ordered ? OMP_ord_static : OMP_sch_static;
2588 }
2589 llvm_unreachable("Unexpected runtime schedule");
2590}
2591
2592/// Map the OpenMP distribute schedule to the runtime enumeration.
2593static OpenMPSchedType
2594getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2595 // only static is allowed for dist_schedule
2596 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2597}
2598
2599bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2600 bool Chunked) const {
2601 OpenMPSchedType Schedule =
2602 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2603 return Schedule == OMP_sch_static;
2604}
2605
2606bool CGOpenMPRuntime::isStaticNonchunked(
2607 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2608 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2609 return Schedule == OMP_dist_sch_static;
2610}
2611
2612bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2613 bool Chunked) const {
2614 OpenMPSchedType Schedule =
2615 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2616 return Schedule == OMP_sch_static_chunked;
2617}
2618
2619bool CGOpenMPRuntime::isStaticChunked(
2620 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2621 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2622 return Schedule == OMP_dist_sch_static_chunked;
2623}
2624
2625bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2626 OpenMPSchedType Schedule =
2627 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2628 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
2629 return Schedule != OMP_sch_static;
2630}
2631
2632static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2633 OpenMPScheduleClauseModifier M1,
2634 OpenMPScheduleClauseModifier M2) {
2635 int Modifier = 0;
2636 switch (M1) {
2637 case OMPC_SCHEDULE_MODIFIER_monotonic:
2638 Modifier = OMP_sch_modifier_monotonic;
2639 break;
2640 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2641 Modifier = OMP_sch_modifier_nonmonotonic;
2642 break;
2643 case OMPC_SCHEDULE_MODIFIER_simd:
2644 if (Schedule == OMP_sch_static_chunked)
2645 Schedule = OMP_sch_static_balanced_chunked;
2646 break;
2647 case OMPC_SCHEDULE_MODIFIER_last:
2648 case OMPC_SCHEDULE_MODIFIER_unknown:
2649 break;
2650 }
2651 switch (M2) {
2652 case OMPC_SCHEDULE_MODIFIER_monotonic:
2653 Modifier = OMP_sch_modifier_monotonic;
2654 break;
2655 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2656 Modifier = OMP_sch_modifier_nonmonotonic;
2657 break;
2658 case OMPC_SCHEDULE_MODIFIER_simd:
2659 if (Schedule == OMP_sch_static_chunked)
2660 Schedule = OMP_sch_static_balanced_chunked;
2661 break;
2662 case OMPC_SCHEDULE_MODIFIER_last:
2663 case OMPC_SCHEDULE_MODIFIER_unknown:
2664 break;
2665 }
2666 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2667 // If the static schedule kind is specified or if the ordered clause is
2668 // specified, and if the nonmonotonic modifier is not specified, the effect is
2669 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2670 // modifier is specified, the effect is as if the nonmonotonic modifier is
2671 // specified.
2672 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2673 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2674 Schedule == OMP_sch_static_balanced_chunked ||
2675 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2676 Schedule == OMP_dist_sch_static_chunked ||
2677 Schedule == OMP_dist_sch_static ||
2678 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone))
2679 Modifier = OMP_sch_modifier_nonmonotonic;
2680 }
2681 return Schedule | Modifier;
2682}
2683
2684void CGOpenMPRuntime::emitForDispatchInit(
2685 CodeGenFunction &CGF, SourceLocation Loc,
2686 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2687 bool Ordered, const DispatchRTInput &DispatchValues) {
2688 if (!CGF.HaveInsertPoint())
2689 return;
2690 OpenMPSchedType Schedule = getRuntimeSchedule(
2691 ScheduleKind: ScheduleKind.Schedule, Chunked: DispatchValues.Chunk != nullptr, Ordered);
2692 assert(Ordered ||
2693 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2694 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2695 Schedule != OMP_sch_static_balanced_chunked));
2696 // Call __kmpc_dispatch_init(
2697 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2698 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2699 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2700
2701 // If the Chunk was not specified in the clause - use default value 1.
2702 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2703 : CGF.Builder.getIntN(N: IVSize, C: 1);
2704 llvm::Value *Args[] = {
2705 emitUpdateLocation(CGF, Loc),
2706 getThreadID(CGF, Loc),
2707 CGF.Builder.getInt32(C: addMonoNonMonoModifier(
2708 CGM, Schedule, M1: ScheduleKind.M1, M2: ScheduleKind.M2)), // Schedule type
2709 DispatchValues.LB, // Lower
2710 DispatchValues.UB, // Upper
2711 CGF.Builder.getIntN(N: IVSize, C: 1), // Stride
2712 Chunk // Chunk
2713 };
2714 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2715 args: Args);
2716}
2717
2718void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
2719 SourceLocation Loc) {
2720 if (!CGF.HaveInsertPoint())
2721 return;
2722 // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid);
2723 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2724 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchDeinitFunction(), args: Args);
2725}
2726
2727static void emitForStaticInitCall(
2728 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2729 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2730 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2731 const CGOpenMPRuntime::StaticRTInput &Values) {
2732 if (!CGF.HaveInsertPoint())
2733 return;
2734
2735 assert(!Values.Ordered);
2736 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2737 Schedule == OMP_sch_static_balanced_chunked ||
2738 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2739 Schedule == OMP_dist_sch_static ||
2740 Schedule == OMP_dist_sch_static_chunked ||
2741 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone);
2742
2743 // Call __kmpc_for_static_init(
2744 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2745 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2746 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2747 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2748 llvm::Value *Chunk = Values.Chunk;
2749 if (Chunk == nullptr) {
2750 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2751 Schedule == OMP_dist_sch_static) &&
2752 "expected static non-chunked schedule");
2753 // If the Chunk was not specified in the clause - use default value 1.
2754 Chunk = CGF.Builder.getIntN(N: Values.IVSize, C: 1);
2755 } else {
2756 assert((Schedule == OMP_sch_static_chunked ||
2757 Schedule == OMP_sch_static_balanced_chunked ||
2758 Schedule == OMP_ord_static_chunked ||
2759 Schedule == OMP_dist_sch_static_chunked ||
2760 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) &&
2761 "expected static chunked schedule");
2762 }
2763 llvm::Value *Args[] = {
2764 UpdateLocation,
2765 ThreadId,
2766 CGF.Builder.getInt32(C: addMonoNonMonoModifier(CGM&: CGF.CGM, Schedule, M1,
2767 M2)), // Schedule type
2768 Values.IL.emitRawPointer(CGF), // &isLastIter
2769 Values.LB.emitRawPointer(CGF), // &LB
2770 Values.UB.emitRawPointer(CGF), // &UB
2771 Values.ST.emitRawPointer(CGF), // &Stride
2772 CGF.Builder.getIntN(N: Values.IVSize, C: 1), // Incr
2773 Chunk // Chunk
2774 };
2775 CGF.EmitRuntimeCall(callee: ForStaticInitFunction, args: Args);
2776}
2777
2778void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2779 SourceLocation Loc,
2780 OpenMPDirectiveKind DKind,
2781 const OpenMPScheduleTy &ScheduleKind,
2782 const StaticRTInput &Values) {
2783 OpenMPSchedType ScheduleNum =
2784 ScheduleKind.UseFusedDistChunkSchedule
2785 ? OMP_dist_sch_static_chunked_sch_static_chunkone
2786 : getRuntimeSchedule(ScheduleKind: ScheduleKind.Schedule, Chunked: Values.Chunk != nullptr,
2787 Ordered: Values.Ordered);
2788 assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&
2789 "Expected loop-based or sections-based directive.");
2790 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2791 Flags: isOpenMPLoopDirective(DKind)
2792 ? OMP_IDENT_WORK_LOOP
2793 : OMP_IDENT_WORK_SECTIONS);
2794 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2795 llvm::FunctionCallee StaticInitFunction =
2796 OMPBuilder.createForStaticInitFunction(IVSize: Values.IVSize, IVSigned: Values.IVSigned,
2797 IsGPUDistribute: false);
2798 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2799 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2800 Schedule: ScheduleNum, M1: ScheduleKind.M1, M2: ScheduleKind.M2, Values);
2801}
2802
2803void CGOpenMPRuntime::emitDistributeStaticInit(
2804 CodeGenFunction &CGF, SourceLocation Loc,
2805 OpenMPDistScheduleClauseKind SchedKind,
2806 const CGOpenMPRuntime::StaticRTInput &Values) {
2807 OpenMPSchedType ScheduleNum =
2808 getRuntimeSchedule(ScheduleKind: SchedKind, Chunked: Values.Chunk != nullptr);
2809 llvm::Value *UpdatedLocation =
2810 emitUpdateLocation(CGF, Loc, Flags: OMP_IDENT_WORK_DISTRIBUTE);
2811 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2812 llvm::FunctionCallee StaticInitFunction;
2813 bool isGPUDistribute =
2814 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU();
2815 StaticInitFunction = OMPBuilder.createForStaticInitFunction(
2816 IVSize: Values.IVSize, IVSigned: Values.IVSigned, IsGPUDistribute: isGPUDistribute);
2817
2818 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2819 Schedule: ScheduleNum, M1: OMPC_SCHEDULE_MODIFIER_unknown,
2820 M2: OMPC_SCHEDULE_MODIFIER_unknown, Values);
2821}
2822
2823void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2824 SourceLocation Loc,
2825 OpenMPDirectiveKind DKind) {
2826 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2827 DKind == OMPD_sections) &&
2828 "Expected distribute, for, or sections directive kind");
2829 if (!CGF.HaveInsertPoint())
2830 return;
2831 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2832 llvm::Value *Args[] = {
2833 emitUpdateLocation(CGF, Loc,
2834 Flags: isOpenMPDistributeDirective(DKind) ||
2835 (DKind == OMPD_target_teams_loop)
2836 ? OMP_IDENT_WORK_DISTRIBUTE
2837 : isOpenMPLoopDirective(DKind)
2838 ? OMP_IDENT_WORK_LOOP
2839 : OMP_IDENT_WORK_SECTIONS),
2840 getThreadID(CGF, Loc)};
2841 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2842 if (isOpenMPDistributeDirective(DKind) &&
2843 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU())
2844 CGF.EmitRuntimeCall(
2845 callee: OMPBuilder.getOrCreateRuntimeFunction(
2846 M&: CGM.getModule(), FnID: OMPRTL___kmpc_distribute_static_fini),
2847 args: Args);
2848 else
2849 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2850 M&: CGM.getModule(), FnID: OMPRTL___kmpc_for_static_fini),
2851 args: Args);
2852}
2853
2854void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2855 SourceLocation Loc,
2856 unsigned IVSize,
2857 bool IVSigned) {
2858 if (!CGF.HaveInsertPoint())
2859 return;
2860 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2861 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2862 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2863 args: Args);
2864}
2865
2866llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2867 SourceLocation Loc, unsigned IVSize,
2868 bool IVSigned, Address IL,
2869 Address LB, Address UB,
2870 Address ST) {
2871 // Call __kmpc_dispatch_next(
2872 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2873 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2874 // kmp_int[32|64] *p_stride);
2875 llvm::Value *Args[] = {
2876 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2877 IL.emitRawPointer(CGF), // &isLastIter
2878 LB.emitRawPointer(CGF), // &Lower
2879 UB.emitRawPointer(CGF), // &Upper
2880 ST.emitRawPointer(CGF) // &Stride
2881 };
2882 llvm::Value *Call = CGF.EmitRuntimeCall(
2883 callee: OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), args: Args);
2884 return CGF.EmitScalarConversion(
2885 Src: Call, SrcTy: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/1),
2886 DstTy: CGF.getContext().BoolTy, Loc);
2887}
2888
2889llvm::Value *CGOpenMPRuntime::emitMessageClause(CodeGenFunction &CGF,
2890 const Expr *Message,
2891 SourceLocation Loc) {
2892 if (!Message)
2893 return llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2894 return CGF.EmitScalarExpr(E: Message);
2895}
2896
2897llvm::Value *
2898CGOpenMPRuntime::emitSeverityClause(OpenMPSeverityClauseKind Severity,
2899 SourceLocation Loc) {
2900 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is
2901 // as if sev-level is fatal."
2902 return llvm::ConstantInt::get(Ty: CGM.Int32Ty,
2903 V: Severity == OMPC_SEVERITY_warning ? 1 : 2);
2904}
2905
2906void CGOpenMPRuntime::emitNumThreadsClause(
2907 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
2908 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
2909 SourceLocation SeverityLoc, const Expr *Message,
2910 SourceLocation MessageLoc) {
2911 if (!CGF.HaveInsertPoint())
2912 return;
2913 llvm::SmallVector<llvm::Value *, 4> Args(
2914 {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2915 CGF.Builder.CreateIntCast(V: NumThreads, DestTy: CGF.Int32Ty, /*isSigned*/ true)});
2916 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2917 // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity,
2918 // messsage) if strict modifier is used.
2919 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2920 if (Modifier == OMPC_NUMTHREADS_strict) {
2921 FnID = OMPRTL___kmpc_push_num_threads_strict;
2922 Args.push_back(Elt: emitSeverityClause(Severity, Loc: SeverityLoc));
2923 Args.push_back(Elt: emitMessageClause(CGF, Message, Loc: MessageLoc));
2924 }
2925 CGF.EmitRuntimeCall(
2926 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args);
2927}
2928
2929void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2930 ProcBindKind ProcBind,
2931 SourceLocation Loc) {
2932 if (!CGF.HaveInsertPoint())
2933 return;
2934 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");
2935 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2936 llvm::Value *Args[] = {
2937 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2938 llvm::ConstantInt::get(Ty: CGM.IntTy, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
2939 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2940 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_proc_bind),
2941 args: Args);
2942}
2943
2944void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2945 SourceLocation Loc, llvm::AtomicOrdering AO) {
2946 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2947 OMPBuilder.createFlush(Loc: CGF.Builder);
2948 } else {
2949 if (!CGF.HaveInsertPoint())
2950 return;
2951 // Build call void __kmpc_flush(ident_t *loc)
2952 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2953 M&: CGM.getModule(), FnID: OMPRTL___kmpc_flush),
2954 args: emitUpdateLocation(CGF, Loc));
2955 }
2956}
2957
2958namespace {
2959/// Indexes of fields for type kmp_task_t.
2960enum KmpTaskTFields {
2961 /// List of shared variables.
2962 KmpTaskTShareds,
2963 /// Task routine.
2964 KmpTaskTRoutine,
2965 /// Partition id for the untied tasks.
2966 KmpTaskTPartId,
2967 /// Function with call of destructors for private variables.
2968 Data1,
2969 /// Task priority.
2970 Data2,
2971 /// (Taskloops only) Lower bound.
2972 KmpTaskTLowerBound,
2973 /// (Taskloops only) Upper bound.
2974 KmpTaskTUpperBound,
2975 /// (Taskloops only) Stride.
2976 KmpTaskTStride,
2977 /// (Taskloops only) Is last iteration flag.
2978 KmpTaskTLastIter,
2979 /// (Taskloops only) Reduction data.
2980 KmpTaskTReductions,
2981};
2982} // anonymous namespace
2983
2984void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
2985 // If we are in simd mode or there are no entries, we don't need to do
2986 // anything.
2987 if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty())
2988 return;
2989
2990 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2991 [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2992 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
2993 SourceLocation Loc;
2994 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2995 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
2996 E = CGM.getContext().getSourceManager().fileinfo_end();
2997 I != E; ++I) {
2998 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2999 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
3000 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
3001 SourceFile: I->getFirst(), Line: EntryInfo.Line, Col: 1);
3002 break;
3003 }
3004 }
3005 }
3006 switch (Kind) {
3007 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
3008 CGM.getDiags().Report(Loc,
3009 DiagID: diag::err_target_region_offloading_entry_incorrect)
3010 << EntryInfo.ParentName;
3011 } break;
3012 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
3013 CGM.getDiags().Report(
3014 Loc, DiagID: diag::err_target_var_offloading_entry_incorrect_with_parent)
3015 << EntryInfo.ParentName;
3016 } break;
3017 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
3018 CGM.getDiags().Report(DiagID: diag::err_target_var_offloading_entry_incorrect);
3019 } break;
3020 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: {
3021 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3022 L: DiagnosticsEngine::Error, FormatString: "Offloading entry for indirect declare "
3023 "target variable is incorrect: the "
3024 "address is invalid.");
3025 CGM.getDiags().Report(DiagID);
3026 } break;
3027 }
3028 };
3029
3030 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
3031}
3032
3033void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3034 if (!KmpRoutineEntryPtrTy) {
3035 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3036 ASTContext &C = CGM.getContext();
3037 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3038 FunctionProtoType::ExtProtoInfo EPI;
3039 KmpRoutineEntryPtrQTy = C.getPointerType(
3040 T: C.getFunctionType(ResultTy: KmpInt32Ty, Args: KmpRoutineEntryTyArgs, EPI));
3041 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(T: KmpRoutineEntryPtrQTy);
3042 }
3043}
3044
3045namespace {
3046struct PrivateHelpersTy {
3047 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3048 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3049 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3050 PrivateElemInit(PrivateElemInit) {}
3051 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3052 const Expr *OriginalRef = nullptr;
3053 const VarDecl *Original = nullptr;
3054 const VarDecl *PrivateCopy = nullptr;
3055 const VarDecl *PrivateElemInit = nullptr;
3056 bool isLocalPrivate() const {
3057 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3058 }
3059};
3060typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3061} // anonymous namespace
3062
3063/// For BindingDecls, returns the DecomposedDecl as the original VarDecl.
3064/// For regular VarDecls, returns the VarDecl itself.
3065static const VarDecl *getOriginalVarDecl(const ValueDecl *Decl) {
3066 if (const auto *BD = dyn_cast<BindingDecl>(Val: Decl))
3067 return cast<VarDecl>(Val: BD->getDecomposedDecl());
3068 return cast<VarDecl>(Val: Decl);
3069}
3070
3071static bool isAllocatableDecl(const VarDecl *VD) {
3072 const VarDecl *CVD = VD->getCanonicalDecl();
3073 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3074 return false;
3075 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3076 // Use the default allocation.
3077 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3078 !AA->getAllocator());
3079}
3080
3081static RecordDecl *
3082createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3083 if (!Privates.empty()) {
3084 ASTContext &C = CGM.getContext();
3085 // Build struct .kmp_privates_t. {
3086 // /* private vars */
3087 // };
3088 RecordDecl *RD = C.buildImplicitRecord(Name: ".kmp_privates.t");
3089 RD->startDefinition();
3090 for (const auto &Pair : Privates) {
3091 const VarDecl *VD = Pair.second.Original;
3092 const VarDecl *PrivateCopy = Pair.second.PrivateCopy;
3093 // For BindingDecls, use PrivateCopy type (binding's actual type).
3094 // For regular variables, use Original type to preserve qualifiers.
3095 // Check OriginalRef to detect BindingDecls since Original may be the
3096 // DecompositionDecl.
3097 bool IsBinding =
3098 Pair.second.OriginalRef &&
3099 isa<BindingDecl>(
3100 Val: cast<DeclRefExpr>(Val: Pair.second.OriginalRef)->getDecl());
3101 QualType Type = IsBinding ? PrivateCopy->getType().getNonReferenceType()
3102 : VD->getType().getNonReferenceType();
3103
3104 // If the private variable is a local variable with lvalue ref type,
3105 // allocate the pointer instead of the pointee type.
3106 if (Pair.second.isLocalPrivate()) {
3107 if (VD->getType()->isLValueReferenceType())
3108 Type = C.getPointerType(T: Type);
3109 if (isAllocatableDecl(VD))
3110 Type = C.getPointerType(T: Type);
3111 }
3112 FieldDecl *FD = addFieldToRecordDecl(C, DC: RD, FieldTy: Type);
3113 if (VD->hasAttrs()) {
3114 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3115 E(VD->getAttrs().end());
3116 I != E; ++I)
3117 FD->addAttr(A: *I);
3118 }
3119 }
3120 RD->completeDefinition();
3121 return RD;
3122 }
3123 return nullptr;
3124}
3125
3126static RecordDecl *
3127createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3128 QualType KmpInt32Ty,
3129 QualType KmpRoutineEntryPointerQTy) {
3130 ASTContext &C = CGM.getContext();
3131 // Build struct kmp_task_t {
3132 // void * shareds;
3133 // kmp_routine_entry_t routine;
3134 // kmp_int32 part_id;
3135 // kmp_cmplrdata_t data1;
3136 // kmp_cmplrdata_t data2;
3137 // For taskloops additional fields:
3138 // kmp_uint64 lb;
3139 // kmp_uint64 ub;
3140 // kmp_int64 st;
3141 // kmp_int32 liter;
3142 // void * reductions;
3143 // };
3144 RecordDecl *UD = C.buildImplicitRecord(Name: "kmp_cmplrdata_t", TK: TagTypeKind::Union);
3145 UD->startDefinition();
3146 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpInt32Ty);
3147 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpRoutineEntryPointerQTy);
3148 UD->completeDefinition();
3149 CanQualType KmpCmplrdataTy = C.getCanonicalTagType(TD: UD);
3150 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t");
3151 RD->startDefinition();
3152 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3153 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpRoutineEntryPointerQTy);
3154 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3155 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3156 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3157 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3158 QualType KmpUInt64Ty =
3159 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3160 QualType KmpInt64Ty =
3161 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3162 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3163 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3164 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt64Ty);
3165 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3166 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3167 }
3168 RD->completeDefinition();
3169 return RD;
3170}
3171
3172static RecordDecl *
3173createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3174 ArrayRef<PrivateDataTy> Privates) {
3175 ASTContext &C = CGM.getContext();
3176 // Build struct kmp_task_t_with_privates {
3177 // kmp_task_t task_data;
3178 // .kmp_privates_t. privates;
3179 // };
3180 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t_with_privates");
3181 RD->startDefinition();
3182 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpTaskTQTy);
3183 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3184 addFieldToRecordDecl(C, DC: RD, FieldTy: C.getCanonicalTagType(TD: PrivateRD));
3185 RD->completeDefinition();
3186 return RD;
3187}
3188
3189/// Emit a proxy function which accepts kmp_task_t as the second
3190/// argument.
3191/// \code
3192/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3193/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3194/// For taskloops:
3195/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3196/// tt->reductions, tt->shareds);
3197/// return 0;
3198/// }
3199/// \endcode
3200static llvm::Function *
3201emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3202 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3203 QualType KmpTaskTWithPrivatesPtrQTy,
3204 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3205 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3206 llvm::Value *TaskPrivatesMap) {
3207 ASTContext &C = CGM.getContext();
3208 auto *GtidArg =
3209 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3210 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3211 auto *TaskTypeArg = ImplicitParamDecl::Create(
3212 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3213 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3214 FunctionArgList Args{GtidArg, TaskTypeArg};
3215 const auto &TaskEntryFnInfo =
3216 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3217 llvm::FunctionType *TaskEntryTy =
3218 CGM.getTypes().GetFunctionType(Info: TaskEntryFnInfo);
3219 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_entry", ""});
3220 auto *TaskEntry = llvm::Function::Create(
3221 Ty: TaskEntryTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3222 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskEntry, FI: TaskEntryFnInfo);
3223 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3224 TaskEntry->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3225 TaskEntry->setDoesNotRecurse();
3226 CodeGenFunction CGF(CGM);
3227 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: TaskEntry, FnInfo: TaskEntryFnInfo, Args,
3228 Loc, StartLoc: Loc);
3229
3230 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3231 // tt,
3232 // For taskloops:
3233 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3234 // tt->task_data.shareds);
3235 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3236 Addr: CGF.GetAddrOfLocalVar(VD: GtidArg), /*Volatile=*/false, Ty: KmpInt32Ty, Loc);
3237 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3238 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3239 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3240 const auto *KmpTaskTWithPrivatesQTyRD =
3241 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3242 LValue Base =
3243 CGF.EmitLValueForField(Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3244 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3245 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
3246 LValue PartIdLVal = CGF.EmitLValueForField(Base, Field: *PartIdFI);
3247 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3248
3249 auto SharedsFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds);
3250 LValue SharedsLVal = CGF.EmitLValueForField(Base, Field: *SharedsFI);
3251 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3252 V: CGF.EmitLoadOfScalar(lvalue: SharedsLVal, Loc),
3253 DestTy: CGF.ConvertTypeForMem(T: SharedsPtrTy));
3254
3255 auto PrivatesFI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin(), n: 1);
3256 llvm::Value *PrivatesParam;
3257 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3258 LValue PrivatesLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PrivatesFI);
3259 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3260 V: PrivatesLVal.getPointer(CGF), DestTy: CGF.VoidPtrTy);
3261 } else {
3262 PrivatesParam = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
3263 }
3264
3265 llvm::Value *CommonArgs[] = {
3266 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3267 CGF.Builder
3268 .CreatePointerBitCastOrAddrSpaceCast(Addr: TDBase.getAddress(),
3269 Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty)
3270 .emitRawPointer(CGF)};
3271 SmallVector<llvm::Value *, 16> CallArgs(std::begin(arr&: CommonArgs),
3272 std::end(arr&: CommonArgs));
3273 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3274 auto LBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound);
3275 LValue LBLVal = CGF.EmitLValueForField(Base, Field: *LBFI);
3276 llvm::Value *LBParam = CGF.EmitLoadOfScalar(lvalue: LBLVal, Loc);
3277 auto UBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound);
3278 LValue UBLVal = CGF.EmitLValueForField(Base, Field: *UBFI);
3279 llvm::Value *UBParam = CGF.EmitLoadOfScalar(lvalue: UBLVal, Loc);
3280 auto StFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride);
3281 LValue StLVal = CGF.EmitLValueForField(Base, Field: *StFI);
3282 llvm::Value *StParam = CGF.EmitLoadOfScalar(lvalue: StLVal, Loc);
3283 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3284 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3285 llvm::Value *LIParam = CGF.EmitLoadOfScalar(lvalue: LILVal, Loc);
3286 auto RFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions);
3287 LValue RLVal = CGF.EmitLValueForField(Base, Field: *RFI);
3288 llvm::Value *RParam = CGF.EmitLoadOfScalar(lvalue: RLVal, Loc);
3289 CallArgs.push_back(Elt: LBParam);
3290 CallArgs.push_back(Elt: UBParam);
3291 CallArgs.push_back(Elt: StParam);
3292 CallArgs.push_back(Elt: LIParam);
3293 CallArgs.push_back(Elt: RParam);
3294 }
3295 CallArgs.push_back(Elt: SharedsParam);
3296
3297 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskFunction,
3298 Args: CallArgs);
3299 CGF.EmitStoreThroughLValue(Src: RValue::get(V: CGF.Builder.getInt32(/*C=*/0)),
3300 Dst: CGF.MakeAddrLValue(Addr: CGF.ReturnValue, T: KmpInt32Ty));
3301 CGF.FinishFunction();
3302 return TaskEntry;
3303}
3304
3305static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3306 SourceLocation Loc,
3307 QualType KmpInt32Ty,
3308 QualType KmpTaskTWithPrivatesPtrQTy,
3309 QualType KmpTaskTWithPrivatesQTy) {
3310 ASTContext &C = CGM.getContext();
3311 auto *GtidArg =
3312 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3313 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3314 auto *TaskTypeArg = ImplicitParamDecl::Create(
3315 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3316 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3317 FunctionArgList Args{GtidArg, TaskTypeArg};
3318 const auto &DestructorFnInfo =
3319 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3320 llvm::FunctionType *DestructorFnTy =
3321 CGM.getTypes().GetFunctionType(Info: DestructorFnInfo);
3322 std::string Name =
3323 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_destructor", ""});
3324 auto *DestructorFn =
3325 llvm::Function::Create(Ty: DestructorFnTy, Linkage: llvm::GlobalValue::InternalLinkage,
3326 N: Name, M: &CGM.getModule());
3327 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: DestructorFn,
3328 FI: DestructorFnInfo);
3329 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3330 DestructorFn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3331 DestructorFn->setDoesNotRecurse();
3332 CodeGenFunction CGF(CGM);
3333 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: DestructorFn, FnInfo: DestructorFnInfo,
3334 Args, Loc, StartLoc: Loc);
3335
3336 LValue Base = CGF.EmitLoadOfPointerLValue(
3337 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3338 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3339 const auto *KmpTaskTWithPrivatesQTyRD =
3340 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3341 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3342 Base = CGF.EmitLValueForField(Base, Field: *FI);
3343 for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3344 if (QualType::DestructionKind DtorKind =
3345 Field->getType().isDestructedType()) {
3346 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3347 CGF.pushDestroy(dtorKind: DtorKind, addr: FieldLValue.getAddress(), type: Field->getType());
3348 }
3349 }
3350 CGF.FinishFunction();
3351 return DestructorFn;
3352}
3353
3354/// Emit a privates mapping function for correct handling of private and
3355/// firstprivate variables.
3356/// \code
3357/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3358/// **noalias priv1,..., <tyn> **noalias privn) {
3359/// *priv1 = &.privates.priv1;
3360/// ...;
3361/// *privn = &.privates.privn;
3362/// }
3363/// \endcode
3364static llvm::Value *
3365emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3366 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3367 ArrayRef<PrivateDataTy> Privates) {
3368 ASTContext &C = CGM.getContext();
3369 FunctionArgList Args;
3370 auto *TaskPrivatesArg = ImplicitParamDecl::Create(
3371 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3372 T: C.getPointerType(T: PrivatesQTy).withConst().withRestrict(),
3373 ParamKind: ImplicitParamKind::Other);
3374 Args.push_back(Elt: TaskPrivatesArg);
3375 llvm::SmallDenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3376 // Track BindingDecl positions separately since BindingDecl is not a VarDecl.
3377 llvm::SmallDenseMap<const BindingDecl *, unsigned> BindingDeclPos;
3378 unsigned Counter = 1;
3379 for (const Expr *E : Data.PrivateVars) {
3380 Args.push_back(Elt: ImplicitParamDecl::Create(
3381 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3382 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3383 .withConst()
3384 .withRestrict(),
3385 ParamKind: ImplicitParamKind::Other));
3386 const ValueDecl *VD = cast<DeclRefExpr>(Val: E)->getDecl();
3387 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
3388 BindingDeclPos[cast<BindingDecl>(Val: BD->getCanonicalDecl())] = Counter;
3389 else
3390 PrivateVarsPos[cast<VarDecl>(Val: VD)] = Counter;
3391 ++Counter;
3392 }
3393 for (const Expr *E : Data.FirstprivateVars) {
3394 Args.push_back(Elt: ImplicitParamDecl::Create(
3395 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3396 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3397 .withConst()
3398 .withRestrict(),
3399 ParamKind: ImplicitParamKind::Other));
3400 const ValueDecl *VD = cast<DeclRefExpr>(Val: E)->getDecl();
3401 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
3402 BindingDeclPos[cast<BindingDecl>(Val: BD->getCanonicalDecl())] = Counter;
3403 else
3404 PrivateVarsPos[cast<VarDecl>(Val: VD)] = Counter;
3405 ++Counter;
3406 }
3407 for (const Expr *E : Data.LastprivateVars) {
3408 Args.push_back(Elt: ImplicitParamDecl::Create(
3409 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3410 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3411 .withConst()
3412 .withRestrict(),
3413 ParamKind: ImplicitParamKind::Other));
3414 const ValueDecl *VD = cast<DeclRefExpr>(Val: E)->getDecl();
3415 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
3416 BindingDeclPos[cast<BindingDecl>(Val: BD->getCanonicalDecl())] = Counter;
3417 else
3418 PrivateVarsPos[cast<VarDecl>(Val: VD)] = Counter;
3419 ++Counter;
3420 }
3421 for (const VarDecl *VD : Data.PrivateLocals) {
3422 QualType Ty = VD->getType().getNonReferenceType();
3423 if (VD->getType()->isLValueReferenceType())
3424 Ty = C.getPointerType(T: Ty);
3425 if (isAllocatableDecl(VD))
3426 Ty = C.getPointerType(T: Ty);
3427 Args.push_back(Elt: ImplicitParamDecl::Create(
3428 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3429 T: C.getPointerType(T: C.getPointerType(T: Ty)).withConst().withRestrict(),
3430 ParamKind: ImplicitParamKind::Other));
3431 PrivateVarsPos[VD] = Counter;
3432 ++Counter;
3433 }
3434 const auto &TaskPrivatesMapFnInfo =
3435 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3436 llvm::FunctionType *TaskPrivatesMapTy =
3437 CGM.getTypes().GetFunctionType(Info: TaskPrivatesMapFnInfo);
3438 std::string Name =
3439 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_privates_map", ""});
3440 auto *TaskPrivatesMap = llvm::Function::Create(
3441 Ty: TaskPrivatesMapTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
3442 M: &CGM.getModule());
3443 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskPrivatesMap,
3444 FI: TaskPrivatesMapFnInfo);
3445 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3446 TaskPrivatesMap->addFnAttr(Kind: "sample-profile-suffix-elision-policy",
3447 Val: "selected");
3448 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
3449 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::NoInline);
3450 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
3451 TaskPrivatesMap->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
3452 }
3453 CodeGenFunction CGF(CGM);
3454 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskPrivatesMap,
3455 FnInfo: TaskPrivatesMapFnInfo, Args, Loc, StartLoc: Loc);
3456
3457 // *privi = &.privates.privi;
3458 LValue Base = CGF.EmitLoadOfPointerLValue(
3459 Ptr: CGF.GetAddrOfLocalVar(VD: TaskPrivatesArg),
3460 PtrTy: TaskPrivatesArg->getType()->castAs<PointerType>());
3461 const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl();
3462 Counter = 0;
3463 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3464 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3465 // Lookup by the original declaration (BindingDecl or VarDecl).
3466 const ValueDecl *LookupVD;
3467 if (Privates[Counter].second.OriginalRef) {
3468 LookupVD =
3469 cast<DeclRefExpr>(Val: Privates[Counter].second.OriginalRef)->getDecl();
3470 } else {
3471 LookupVD = Privates[Counter].second.Original;
3472 }
3473
3474 // For BindingDecls, the privates record now stores each binding's type
3475 // directly (not the full DecompositionDecl), so FieldLVal is already
3476 // correct.
3477
3478 unsigned Position;
3479 if (const auto *BD = dyn_cast<BindingDecl>(Val: LookupVD)) {
3480 Position =
3481 BindingDeclPos.lookup(Val: cast<BindingDecl>(Val: BD->getCanonicalDecl()));
3482 assert(Position && "binding not in privates mapping");
3483 } else {
3484 Position = PrivateVarsPos[cast<VarDecl>(Val: LookupVD)];
3485 }
3486 const VarDecl *VD = Args[Position];
3487 LValue RefLVal =
3488 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD), T: VD->getType());
3489 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3490 Ptr: RefLVal.getAddress(), PtrTy: RefLVal.getType()->castAs<PointerType>());
3491 CGF.EmitStoreOfScalar(value: FieldLVal.getPointer(CGF), lvalue: RefLoadLVal);
3492 ++Counter;
3493 }
3494 CGF.FinishFunction();
3495 return TaskPrivatesMap;
3496}
3497
3498/// Emit initialization for private variables in task-based directives.
3499static void emitPrivatesInit(CodeGenFunction &CGF,
3500 const OMPExecutableDirective &D,
3501 Address KmpTaskSharedsPtr, LValue TDBase,
3502 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3503 QualType SharedsTy, QualType SharedsPtrTy,
3504 const OMPTaskDataTy &Data,
3505 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3506 ASTContext &C = CGF.getContext();
3507 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3508 LValue PrivatesBase = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
3509 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())
3510 ? OMPD_taskloop
3511 : OMPD_task;
3512 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: Kind);
3513 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3514 LValue SrcBase;
3515 bool IsTargetTask =
3516 isOpenMPTargetDataManagementDirective(DKind: D.getDirectiveKind()) ||
3517 isOpenMPTargetExecutionDirective(DKind: D.getDirectiveKind());
3518 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3519 // PointersArray, SizesArray, and MappersArray. The original variables for
3520 // these arrays are not captured and we get their addresses explicitly.
3521 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3522 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3523 SrcBase = CGF.MakeAddrLValue(
3524 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3525 Addr: KmpTaskSharedsPtr, Ty: CGF.ConvertTypeForMem(T: SharedsPtrTy),
3526 ElementTy: CGF.ConvertTypeForMem(T: SharedsTy)),
3527 T: SharedsTy);
3528 }
3529 FI = FI->getType()->castAsRecordDecl()->field_begin();
3530 for (const PrivateDataTy &Pair : Privates) {
3531 // Do not initialize private locals.
3532 if (Pair.second.isLocalPrivate()) {
3533 ++FI;
3534 continue;
3535 }
3536 const VarDecl *VD = Pair.second.PrivateCopy;
3537 const Expr *Init = VD->getAnyInitializer();
3538 if (Init && (!ForDup || (isa<CXXConstructExpr>(Val: Init) &&
3539 !CGF.isTrivialInitializer(Init)))) {
3540 LValue PrivateLValue = CGF.EmitLValueForField(Base: PrivatesBase, Field: *FI);
3541 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3542 const VarDecl *OriginalVD = Pair.second.Original;
3543 // Check if the variable is the target-based BasePointersArray,
3544 // PointersArray, SizesArray, or MappersArray.
3545 LValue SharedRefLValue;
3546 QualType Type = PrivateLValue.getType();
3547 const FieldDecl *SharedField = CapturesInfo.lookup(VD: OriginalVD);
3548 if (IsTargetTask && !SharedField) {
3549 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3550 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
3551 cast<CapturedDecl>(OriginalVD->getDeclContext())
3552 ->getNumParams() == 0 &&
3553 isa<TranslationUnitDecl>(
3554 cast<CapturedDecl>(OriginalVD->getDeclContext())
3555 ->getDeclContext()) &&
3556 "Expected artificial target data variable.");
3557 SharedRefLValue =
3558 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: OriginalVD), T: Type);
3559 } else if (ForDup) {
3560 SharedRefLValue = CGF.EmitLValueForField(Base: SrcBase, Field: SharedField);
3561 // For BindingDecls, access the specific binding field within the
3562 // captured DecompositionDecl.
3563 if (Pair.second.OriginalRef) {
3564 if (const auto *DRE =
3565 dyn_cast<DeclRefExpr>(Val: Pair.second.OriginalRef)) {
3566 if (const auto *BD = dyn_cast<BindingDecl>(Val: DRE->getDecl())) {
3567 // Emit the binding subobject (member or array element) with
3568 // the decomposed decl temporarily mapped to the capture.
3569 const VarDecl *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3570 auto It = CGF.findLocalDecl(D: DD);
3571 bool WasMapped = It != CGF.localDeclMapEnd();
3572 Address Saved = WasMapped ? It->second : Address::invalid();
3573 if (WasMapped)
3574 It->second = SharedRefLValue.getAddress();
3575 else
3576 CGF.insertLocalDecl(D: DD, Addr: SharedRefLValue.getAddress());
3577 SharedRefLValue = CGF.EmitLValue(E: BD->getBinding());
3578 if (WasMapped) {
3579 auto RestoreIt = CGF.findLocalDecl(D: DD);
3580 RestoreIt->second = Saved;
3581 } else {
3582 CGF.eraseLocalDecl(D: DD);
3583 }
3584 }
3585 }
3586 }
3587 bool IsBinding =
3588 Pair.second.OriginalRef &&
3589 isa<BindingDecl>(
3590 Val: cast<DeclRefExpr>(Val: Pair.second.OriginalRef)->getDecl());
3591 SharedRefLValue = CGF.MakeAddrLValue(
3592 Addr: SharedRefLValue.getAddress().withAlignment(
3593 NewAlignment: IsBinding ? C.toCharUnitsFromBits(
3594 BitSize: C.getTypeAlign(T: SharedRefLValue.getType()))
3595 : C.getDeclAlign(D: OriginalVD)),
3596 T: SharedRefLValue.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3597 TBAAInfo: SharedRefLValue.getTBAAInfo());
3598 } else if (CGF.LambdaCaptureFields.count(
3599 Val: Pair.second.Original->getCanonicalDecl()) > 0 ||
3600 isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl)) {
3601 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3602 } else {
3603 // Processing for implicitly captured variables.
3604 InlinedOpenMPRegionRAII Region(
3605 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3606 /*HasCancel=*/false, /*NoInheritance=*/true);
3607 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3608 }
3609 if (Type->isArrayType()) {
3610 // Initialize firstprivate array.
3611 if (!isa<CXXConstructExpr>(Val: Init) || CGF.isTrivialInitializer(Init)) {
3612 // Perform simple memcpy.
3613 CGF.EmitAggregateAssign(Dest: PrivateLValue, Src: SharedRefLValue, EltTy: Type);
3614 } else {
3615 // Initialize firstprivate array using element-by-element
3616 // initialization.
3617 CGF.EmitOMPAggregateAssign(
3618 DestAddr: PrivateLValue.getAddress(), SrcAddr: SharedRefLValue.getAddress(), OriginalType: Type,
3619 CopyGen: [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3620 Address SrcElement) {
3621 // Clean up any temporaries needed by the initialization.
3622 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3623 InitScope.addPrivate(LocalVD: Elem, Addr: SrcElement);
3624 (void)InitScope.Privatize();
3625 // Emit initialization for single element.
3626 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3627 CGF, &CapturesInfo);
3628 CGF.EmitAnyExprToMem(E: Init, Location: DestElement,
3629 Quals: Init->getType().getQualifiers(),
3630 /*IsInitializer=*/false);
3631 });
3632 }
3633 } else {
3634 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3635 InitScope.addPrivate(LocalVD: Elem, Addr: SharedRefLValue.getAddress());
3636 (void)InitScope.Privatize();
3637 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3638 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue,
3639 /*capturedByInit=*/false);
3640 }
3641 } else {
3642 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue, /*capturedByInit=*/false);
3643 }
3644 } else if (const VarDecl *OriginalVD = Pair.second.Original) {
3645 // Handle array bindings without initializers (firstprivate only).
3646 // For private clause (PrivateElemInit is nullptr), skip initialization.
3647 // Check if OriginalRef is a BindingDecl with array type.
3648 const BindingDecl *BD = nullptr;
3649 if (Pair.second.OriginalRef) {
3650 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Pair.second.OriginalRef)) {
3651 BD = dyn_cast<BindingDecl>(Val: DRE->getDecl());
3652 }
3653 }
3654 if (BD && BD->getType()->isArrayType() && Pair.second.PrivateElemInit) {
3655 LValue PrivateLValue = CGF.EmitLValueForField(Base: PrivatesBase, Field: *FI);
3656 QualType Type = PrivateLValue.getType();
3657 const FieldDecl *SharedField = CapturesInfo.lookup(VD: OriginalVD);
3658
3659 // Lambda to temporarily map DecompositionDecl and emit the binding
3660 // expression.
3661 auto EmitBindingWithTempMap = [&CGF](const BindingDecl *BD,
3662 Address DDAddr) -> LValue {
3663 const VarDecl *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3664 auto It = CGF.findLocalDecl(D: DD);
3665 bool WasMapped = It != CGF.localDeclMapEnd();
3666 Address Saved = WasMapped ? It->second : Address::invalid();
3667 if (WasMapped)
3668 It->second = DDAddr;
3669 else
3670 CGF.insertLocalDecl(D: DD, Addr: DDAddr);
3671
3672 LValue Result = CGF.EmitLValue(E: BD->getBinding());
3673
3674 // Restore the mapping
3675 if (WasMapped) {
3676 auto RestoreIt = CGF.findLocalDecl(D: DD);
3677 RestoreIt->second = Saved;
3678 } else {
3679 CGF.eraseLocalDecl(D: DD);
3680 }
3681
3682 return Result;
3683 };
3684
3685 LValue SharedRefLValue;
3686 if (ForDup) {
3687 SharedRefLValue = CGF.EmitLValueForField(Base: SrcBase, Field: SharedField);
3688 SharedRefLValue =
3689 EmitBindingWithTempMap(BD, SharedRefLValue.getAddress());
3690 SharedRefLValue = CGF.MakeAddrLValue(
3691 Addr: SharedRefLValue.getAddress().withAlignment(
3692 NewAlignment: C.getDeclAlign(D: OriginalVD)),
3693 T: SharedRefLValue.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3694 TBAAInfo: SharedRefLValue.getTBAAInfo());
3695 } else {
3696 // For !ForDup (first task), emit binding from parent scope using
3697 // InlinedOpenMPRegionRAII to access the correct scope
3698 InlinedOpenMPRegionRAII Region(
3699 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3700 /*HasCancel=*/false, /*NoInheritance=*/true);
3701
3702 // Get the decomposed decl address from parent scope
3703 const VarDecl *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3704 Address DDAddr = CGF.GetAddrOfLocalVar(VD: DD);
3705 SharedRefLValue = EmitBindingWithTempMap(BD, DDAddr);
3706 }
3707 // Perform simple memcpy for array binding
3708 CGF.EmitAggregateAssign(Dest: PrivateLValue, Src: SharedRefLValue, EltTy: Type);
3709 }
3710 }
3711 ++FI;
3712 }
3713}
3714
3715/// Check if duplication function is required for taskloops.
3716static bool checkInitIsRequired(CodeGenFunction &CGF,
3717 ArrayRef<PrivateDataTy> Privates) {
3718 bool InitRequired = false;
3719 for (const PrivateDataTy &Pair : Privates) {
3720 if (Pair.second.isLocalPrivate())
3721 continue;
3722 const VarDecl *VD = Pair.second.PrivateCopy;
3723 const Expr *Init = VD->getAnyInitializer();
3724 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Val: Init) &&
3725 !CGF.isTrivialInitializer(Init));
3726 if (InitRequired)
3727 break;
3728 }
3729 return InitRequired;
3730}
3731
3732
3733/// Emit task_dup function (for initialization of
3734/// private/firstprivate/lastprivate vars and last_iter flag)
3735/// \code
3736/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3737/// lastpriv) {
3738/// // setup lastprivate flag
3739/// task_dst->last = lastpriv;
3740/// // could be constructor calls here...
3741/// }
3742/// \endcode
3743static llvm::Value *
3744emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3745 const OMPExecutableDirective &D,
3746 QualType KmpTaskTWithPrivatesPtrQTy,
3747 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3748 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3749 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3750 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3751 ASTContext &C = CGM.getContext();
3752 auto *DstArg = ImplicitParamDecl::Create(
3753 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3754 ParamKind: ImplicitParamKind::Other);
3755 auto *SrcArg = ImplicitParamDecl::Create(
3756 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3757 ParamKind: ImplicitParamKind::Other);
3758 auto *LastprivArg =
3759 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: C.IntTy,
3760 ParamKind: ImplicitParamKind::Other);
3761 FunctionArgList Args{DstArg, SrcArg, LastprivArg};
3762 const auto &TaskDupFnInfo =
3763 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3764 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(Info: TaskDupFnInfo);
3765 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_dup", ""});
3766 auto *TaskDup = llvm::Function::Create(
3767 Ty: TaskDupTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3768 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskDup, FI: TaskDupFnInfo);
3769 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3770 TaskDup->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3771 TaskDup->setDoesNotRecurse();
3772 CodeGenFunction CGF(CGM);
3773 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskDup, FnInfo: TaskDupFnInfo, Args, Loc,
3774 StartLoc: Loc);
3775
3776 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3777 Ptr: CGF.GetAddrOfLocalVar(VD: DstArg),
3778 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3779 // task_dst->liter = lastpriv;
3780 if (WithLastIter) {
3781 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3782 LValue Base = CGF.EmitLValueForField(
3783 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3784 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3785 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3786 Addr: CGF.GetAddrOfLocalVar(VD: LastprivArg), /*Volatile=*/false, Ty: C.IntTy, Loc);
3787 CGF.EmitStoreOfScalar(value: Lastpriv, lvalue: LILVal);
3788 }
3789
3790 // Emit initial values for private copies (if any).
3791 assert(!Privates.empty());
3792 Address KmpTaskSharedsPtr = Address::invalid();
3793 if (!Data.FirstprivateVars.empty()) {
3794 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3795 Ptr: CGF.GetAddrOfLocalVar(VD: SrcArg),
3796 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3797 LValue Base = CGF.EmitLValueForField(
3798 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3799 KmpTaskSharedsPtr = Address(
3800 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValueForField(
3801 Base, Field: *std::next(x: KmpTaskTQTyRD->field_begin(),
3802 n: KmpTaskTShareds)),
3803 Loc),
3804 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
3805 }
3806 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3807 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3808 CGF.FinishFunction();
3809 return TaskDup;
3810}
3811
3812/// Checks if destructor function is required to be generated.
3813/// \return true if cleanups are required, false otherwise.
3814static bool
3815checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3816 ArrayRef<PrivateDataTy> Privates) {
3817 for (const PrivateDataTy &P : Privates) {
3818 if (P.second.isLocalPrivate())
3819 continue;
3820 QualType Ty = P.second.Original->getType().getNonReferenceType();
3821 if (Ty.isDestructedType())
3822 return true;
3823 }
3824 return false;
3825}
3826
3827namespace {
3828/// Loop generator for OpenMP iterator expression.
3829class OMPIteratorGeneratorScope final
3830 : public CodeGenFunction::OMPPrivateScope {
3831 CodeGenFunction &CGF;
3832 const OMPIteratorExpr *E = nullptr;
3833 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3834 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3835 OMPIteratorGeneratorScope() = delete;
3836 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
3837
3838public:
3839 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3840 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3841 if (!E)
3842 return;
3843 SmallVector<llvm::Value *, 4> Uppers;
3844 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3845 Uppers.push_back(Elt: CGF.EmitScalarExpr(E: E->getHelper(I).Upper));
3846 const auto *VD = cast<VarDecl>(Val: E->getIteratorDecl(I));
3847 addPrivate(LocalVD: VD, Addr: CGF.CreateMemTemp(T: VD->getType(), Name: VD->getName()));
3848 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3849 addPrivate(
3850 LocalVD: HelperData.CounterVD,
3851 Addr: CGF.CreateMemTemp(T: HelperData.CounterVD->getType(), Name: "counter.addr"));
3852 }
3853 Privatize();
3854
3855 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3856 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3857 LValue CLVal =
3858 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: HelperData.CounterVD),
3859 T: HelperData.CounterVD->getType());
3860 // Counter = 0;
3861 CGF.EmitStoreOfScalar(
3862 value: llvm::ConstantInt::get(Ty: CLVal.getAddress().getElementType(), V: 0),
3863 lvalue: CLVal);
3864 CodeGenFunction::JumpDest &ContDest =
3865 ContDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.cont"));
3866 CodeGenFunction::JumpDest &ExitDest =
3867 ExitDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.exit"));
3868 // N = <number-of_iterations>;
3869 llvm::Value *N = Uppers[I];
3870 // cont:
3871 // if (Counter < N) goto body; else goto exit;
3872 CGF.EmitBlock(BB: ContDest.getBlock());
3873 auto *CVal =
3874 CGF.EmitLoadOfScalar(lvalue: CLVal, Loc: HelperData.CounterVD->getLocation());
3875 llvm::Value *Cmp =
3876 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3877 ? CGF.Builder.CreateICmpSLT(LHS: CVal, RHS: N)
3878 : CGF.Builder.CreateICmpULT(LHS: CVal, RHS: N);
3879 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "iter.body");
3880 CGF.Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitDest.getBlock());
3881 // body:
3882 CGF.EmitBlock(BB: BodyBB);
3883 // Iteri = Begini + Counter * Stepi;
3884 CGF.EmitIgnoredExpr(E: HelperData.Update);
3885 }
3886 }
3887 ~OMPIteratorGeneratorScope() {
3888 if (!E)
3889 return;
3890 for (unsigned I = E->numOfIterators(); I > 0; --I) {
3891 // Counter = Counter + 1;
3892 const OMPIteratorHelperData &HelperData = E->getHelper(I: I - 1);
3893 CGF.EmitIgnoredExpr(E: HelperData.CounterUpdate);
3894 // goto cont;
3895 CGF.EmitBranchThroughCleanup(Dest: ContDests[I - 1]);
3896 // exit:
3897 CGF.EmitBlock(BB: ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3898 }
3899 }
3900};
3901} // namespace
3902
3903static std::pair<llvm::Value *, llvm::Value *>
3904getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
3905 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(Val: E);
3906 llvm::Value *Addr;
3907 if (OASE) {
3908 const Expr *Base = OASE->getBase();
3909 Addr = CGF.EmitScalarExpr(E: Base);
3910 } else {
3911 Addr = CGF.EmitLValue(E).getPointer(CGF);
3912 }
3913 llvm::Value *SizeVal;
3914 QualType Ty = E->getType();
3915 if (OASE) {
3916 SizeVal = CGF.getTypeSize(Ty: OASE->getBase()->getType()->getPointeeType());
3917 for (const Expr *SE : OASE->getDimensions()) {
3918 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
3919 Sz = CGF.EmitScalarConversion(
3920 Src: Sz, SrcTy: SE->getType(), DstTy: CGF.getContext().getSizeType(), Loc: SE->getExprLoc());
3921 SizeVal = CGF.Builder.CreateNUWMul(LHS: SizeVal, RHS: Sz);
3922 }
3923 } else if (const auto *ASE =
3924 dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenImpCasts())) {
3925 LValue UpAddrLVal = CGF.EmitArraySectionExpr(E: ASE, /*IsLowerBound=*/false);
3926 Address UpAddrAddress = UpAddrLVal.getAddress();
3927 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3928 Ty: UpAddrAddress.getElementType(), Ptr: UpAddrAddress.emitRawPointer(CGF),
3929 /*Idx0=*/1);
3930 SizeVal = CGF.Builder.CreatePtrDiff(LHS: UpAddr, RHS: Addr, Name: "", /*IsNUW=*/true);
3931 } else {
3932 SizeVal = CGF.getTypeSize(Ty);
3933 }
3934 return std::make_pair(x&: Addr, y&: SizeVal);
3935}
3936
3937/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
3938static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
3939 QualType FlagsTy = C.getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/false);
3940 if (KmpTaskAffinityInfoTy.isNull()) {
3941 RecordDecl *KmpAffinityInfoRD =
3942 C.buildImplicitRecord(Name: "kmp_task_affinity_info_t");
3943 KmpAffinityInfoRD->startDefinition();
3944 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getIntPtrType());
3945 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getSizeType());
3946 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: FlagsTy);
3947 KmpAffinityInfoRD->completeDefinition();
3948 KmpTaskAffinityInfoTy = C.getCanonicalTagType(TD: KmpAffinityInfoRD);
3949 }
3950}
3951
3952CGOpenMPRuntime::TaskResultTy
3953CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3954 const OMPExecutableDirective &D,
3955 llvm::Function *TaskFunction, QualType SharedsTy,
3956 Address Shareds, const OMPTaskDataTy &Data) {
3957 ASTContext &C = CGM.getContext();
3958 llvm::SmallVector<PrivateDataTy, 4> Privates;
3959 // Aggregate privates and sort them by the alignment.
3960 const auto *I = Data.PrivateCopies.begin();
3961 for (const Expr *E : Data.PrivateVars) {
3962 const auto *Decl = cast<DeclRefExpr>(Val: E)->getDecl();
3963 const auto *VD = getOriginalVarDecl(Decl);
3964 const auto *CopyVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
3965 Privates.emplace_back(Args: C.getDeclAlign(D: VD),
3966 Args: PrivateHelpersTy(E, VD, CopyVD,
3967 /*PrivateElemInit=*/nullptr));
3968 ++I;
3969 }
3970 I = Data.FirstprivateCopies.begin();
3971 const auto *IElemInitRef = Data.FirstprivateInits.begin();
3972 for (const Expr *E : Data.FirstprivateVars) {
3973 const auto *Decl = cast<DeclRefExpr>(Val: E)->getDecl();
3974 const auto *VD = getOriginalVarDecl(Decl);
3975 const auto *CopyVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
3976 const auto *InitVD =
3977 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IElemInitRef)->getDecl());
3978 Privates.emplace_back(Args: C.getDeclAlign(D: VD),
3979 Args: PrivateHelpersTy(E, VD, CopyVD, InitVD));
3980 ++I;
3981 ++IElemInitRef;
3982 }
3983 I = Data.LastprivateCopies.begin();
3984 for (const Expr *E : Data.LastprivateVars) {
3985 const auto *Decl = cast<DeclRefExpr>(Val: E)->getDecl();
3986 const auto *VD = getOriginalVarDecl(Decl);
3987 const auto *CopyVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
3988 Privates.emplace_back(Args: C.getDeclAlign(D: VD),
3989 Args: PrivateHelpersTy(E, VD, CopyVD,
3990 /*PrivateElemInit=*/nullptr));
3991 ++I;
3992 }
3993 for (const VarDecl *VD : Data.PrivateLocals) {
3994 if (isAllocatableDecl(VD))
3995 Privates.emplace_back(Args: CGM.getPointerAlign(), Args: PrivateHelpersTy(VD));
3996 else
3997 Privates.emplace_back(Args: C.getDeclAlign(D: VD), Args: PrivateHelpersTy(VD));
3998 }
3999 llvm::stable_sort(Range&: Privates,
4000 C: [](const PrivateDataTy &L, const PrivateDataTy &R) {
4001 return L.first > R.first;
4002 });
4003 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4004 // Build type kmp_routine_entry_t (if not built yet).
4005 emitKmpRoutineEntryT(KmpInt32Ty);
4006 // Build type kmp_task_t (if not built yet).
4007 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())) {
4008 if (SavedKmpTaskloopTQTy.isNull()) {
4009 SavedKmpTaskloopTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
4010 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
4011 }
4012 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4013 } else {
4014 assert((D.getDirectiveKind() == OMPD_task ||
4015 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
4016 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
4017 "Expected taskloop, task or target directive");
4018 if (SavedKmpTaskTQTy.isNull()) {
4019 SavedKmpTaskTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
4020 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
4021 }
4022 KmpTaskTQTy = SavedKmpTaskTQTy;
4023 }
4024 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
4025 // Build particular struct kmp_task_t for the given task.
4026 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4027 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4028 CanQualType KmpTaskTWithPrivatesQTy =
4029 C.getCanonicalTagType(TD: KmpTaskTWithPrivatesQTyRD);
4030 QualType KmpTaskTWithPrivatesPtrQTy =
4031 C.getPointerType(T: KmpTaskTWithPrivatesQTy);
4032 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(AddrSpace: 0);
4033 llvm::Value *KmpTaskTWithPrivatesTySize =
4034 CGF.getTypeSize(Ty: KmpTaskTWithPrivatesQTy);
4035 QualType SharedsPtrTy = C.getPointerType(T: SharedsTy);
4036
4037 // Emit initial values for private copies (if any).
4038 llvm::Value *TaskPrivatesMap = nullptr;
4039 llvm::Type *TaskPrivatesMapTy =
4040 std::next(x: TaskFunction->arg_begin(), n: 3)->getType();
4041 if (!Privates.empty()) {
4042 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
4043 TaskPrivatesMap =
4044 emitTaskPrivateMappingFunction(CGM, Loc, Data, PrivatesQTy: FI->getType(), Privates);
4045 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4046 V: TaskPrivatesMap, DestTy: TaskPrivatesMapTy);
4047 } else {
4048 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4049 T: cast<llvm::PointerType>(Val: TaskPrivatesMapTy));
4050 }
4051 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4052 // kmp_task_t *tt);
4053 llvm::Function *TaskEntry = emitProxyTaskFunction(
4054 CGM, Loc, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4055 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4056 TaskPrivatesMap);
4057
4058 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4059 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4060 // kmp_routine_entry_t *task_entry);
4061 // Task flags. Format is taken from
4062 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
4063 // description of kmp_tasking_flags struct.
4064 enum {
4065 TiedFlag = 0x1,
4066 FinalFlag = 0x2,
4067 DestructorsFlag = 0x8,
4068 PriorityFlag = 0x20,
4069 DetachableFlag = 0x40,
4070 FreeAgentFlag = 0x80,
4071 TransparentFlag = 0x100,
4072 };
4073 unsigned Flags = Data.Tied ? TiedFlag : 0;
4074 bool NeedsCleanup = false;
4075 if (!Privates.empty()) {
4076 NeedsCleanup =
4077 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
4078 if (NeedsCleanup)
4079 Flags = Flags | DestructorsFlag;
4080 }
4081 if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {
4082 OpenMPThreadsetKind Kind = Clause->getThreadsetKind();
4083 if (Kind == OMPC_THREADSET_omp_pool)
4084 Flags = Flags | FreeAgentFlag;
4085 }
4086 if (D.getSingleClause<OMPTransparentClause>())
4087 Flags |= TransparentFlag;
4088
4089 if (Data.Priority.getInt())
4090 Flags = Flags | PriorityFlag;
4091 if (D.hasClausesOfKind<OMPDetachClause>())
4092 Flags = Flags | DetachableFlag;
4093 llvm::Value *TaskFlags =
4094 Data.Final.getPointer()
4095 ? CGF.Builder.CreateSelect(C: Data.Final.getPointer(),
4096 True: CGF.Builder.getInt32(C: FinalFlag),
4097 False: CGF.Builder.getInt32(/*C=*/0))
4098 : CGF.Builder.getInt32(C: Data.Final.getInt() ? FinalFlag : 0);
4099 TaskFlags = CGF.Builder.CreateOr(LHS: TaskFlags, RHS: CGF.Builder.getInt32(C: Flags));
4100 llvm::Value *SharedsSize = CGM.getSize(numChars: C.getTypeSizeInChars(T: SharedsTy));
4101 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
4102 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4103 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4104 V: TaskEntry, DestTy: KmpRoutineEntryPtrTy)};
4105 llvm::Value *NewTask;
4106 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4107 // Check if we have any device clause associated with the directive.
4108 const Expr *Device = nullptr;
4109 if (auto *C = D.getSingleClause<OMPDeviceClause>())
4110 Device = C->getDevice();
4111 // Emit device ID if any otherwise use default value.
4112 llvm::Value *DeviceID;
4113 if (Device)
4114 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
4115 DestTy: CGF.Int64Ty, /*isSigned=*/true);
4116 else
4117 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
4118 AllocArgs.push_back(Elt: DeviceID);
4119 NewTask = CGF.EmitRuntimeCall(
4120 callee: OMPBuilder.getOrCreateRuntimeFunction(
4121 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_target_task_alloc),
4122 args: AllocArgs);
4123 } else {
4124 NewTask =
4125 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4126 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_alloc),
4127 args: AllocArgs);
4128 }
4129 // Emit detach clause initialization.
4130 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
4131 // task_descriptor);
4132 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
4133 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
4134 LValue EvtLVal = CGF.EmitLValue(E: Evt);
4135
4136 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4137 // int gtid, kmp_task_t *task);
4138 llvm::Value *Loc = emitUpdateLocation(CGF, Loc: DC->getBeginLoc());
4139 llvm::Value *Tid = getThreadID(CGF, Loc: DC->getBeginLoc());
4140 Tid = CGF.Builder.CreateIntCast(V: Tid, DestTy: CGF.IntTy, /*isSigned=*/false);
4141 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
4142 callee: OMPBuilder.getOrCreateRuntimeFunction(
4143 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_allow_completion_event),
4144 args: {Loc, Tid, NewTask});
4145 EvtVal = CGF.EmitScalarConversion(Src: EvtVal, SrcTy: C.VoidPtrTy, DstTy: Evt->getType(),
4146 Loc: Evt->getExprLoc());
4147 CGF.EmitStoreOfScalar(value: EvtVal, lvalue: EvtLVal);
4148 }
4149 // Process affinity clauses.
4150 if (D.hasClausesOfKind<OMPAffinityClause>()) {
4151 // Process list of affinity data.
4152 ASTContext &C = CGM.getContext();
4153 Address AffinitiesArray = Address::invalid();
4154 // Calculate number of elements to form the array of affinity data.
4155 llvm::Value *NumOfElements = nullptr;
4156 unsigned NumAffinities = 0;
4157 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4158 if (const Expr *Modifier = C->getModifier()) {
4159 const auto *IE = cast<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts());
4160 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4161 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4162 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
4163 NumOfElements =
4164 NumOfElements ? CGF.Builder.CreateNUWMul(LHS: NumOfElements, RHS: Sz) : Sz;
4165 }
4166 } else {
4167 NumAffinities += C->varlist_size();
4168 }
4169 }
4170 getKmpAffinityType(C&: CGM.getContext(), KmpTaskAffinityInfoTy);
4171 // Fields ids in kmp_task_affinity_info record.
4172 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4173
4174 QualType KmpTaskAffinityInfoArrayTy;
4175 if (NumOfElements) {
4176 NumOfElements = CGF.Builder.CreateNUWAdd(
4177 LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumAffinities), RHS: NumOfElements);
4178 auto *OVE = new (C) OpaqueValueExpr(
4179 Loc,
4180 C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.getSizeType()), /*Signed=*/0),
4181 VK_PRValue);
4182 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4183 RValue::get(V: NumOfElements));
4184 KmpTaskAffinityInfoArrayTy = C.getVariableArrayType(
4185 EltTy: KmpTaskAffinityInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4186 /*IndexTypeQuals=*/0);
4187 // Properly emit variable-sized array.
4188 auto *PD = ImplicitParamDecl::Create(C, T: KmpTaskAffinityInfoArrayTy,
4189 ParamKind: ImplicitParamKind::Other);
4190 CGF.EmitVarDecl(D: *PD);
4191 AffinitiesArray = CGF.GetAddrOfLocalVar(VD: PD);
4192 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4193 /*isSigned=*/false);
4194 } else {
4195 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4196 EltTy: KmpTaskAffinityInfoTy,
4197 ArySize: llvm::APInt(C.getTypeSize(T: C.getSizeType()), NumAffinities), SizeExpr: nullptr,
4198 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4199 AffinitiesArray = CGF.CreateMemTempWithoutCast(T: KmpTaskAffinityInfoArrayTy,
4200 Name: ".affs.arr.addr");
4201 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(Addr: AffinitiesArray, Index: 0);
4202 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumAffinities,
4203 /*isSigned=*/IsSigned: false);
4204 }
4205
4206 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4207 // Fill array by elements without iterators.
4208 unsigned Pos = 0;
4209 bool HasIterator = false;
4210 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4211 if (C->getModifier()) {
4212 HasIterator = true;
4213 continue;
4214 }
4215 for (const Expr *E : C->varlist()) {
4216 llvm::Value *Addr;
4217 llvm::Value *Size;
4218 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4219 LValue Base =
4220 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateConstGEP(Addr: AffinitiesArray, Index: Pos),
4221 T: KmpTaskAffinityInfoTy);
4222 // affs[i].base_addr = &<Affinities[i].second>;
4223 LValue BaseAddrLVal = CGF.EmitLValueForField(
4224 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4225 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4226 lvalue: BaseAddrLVal);
4227 // affs[i].len = sizeof(<Affinities[i].second>);
4228 LValue LenLVal = CGF.EmitLValueForField(
4229 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4230 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4231 ++Pos;
4232 }
4233 }
4234 LValue PosLVal;
4235 if (HasIterator) {
4236 PosLVal = CGF.MakeAddrLValue(
4237 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "affs.counter.addr"),
4238 T: C.getSizeType());
4239 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4240 }
4241 // Process elements with iterators.
4242 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4243 const Expr *Modifier = C->getModifier();
4244 if (!Modifier)
4245 continue;
4246 OMPIteratorGeneratorScope IteratorScope(
4247 CGF, cast_or_null<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts()));
4248 for (const Expr *E : C->varlist()) {
4249 llvm::Value *Addr;
4250 llvm::Value *Size;
4251 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4252 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4253 LValue Base =
4254 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateGEP(CGF, Addr: AffinitiesArray, Index: Idx),
4255 T: KmpTaskAffinityInfoTy);
4256 // affs[i].base_addr = &<Affinities[i].second>;
4257 LValue BaseAddrLVal = CGF.EmitLValueForField(
4258 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4259 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4260 lvalue: BaseAddrLVal);
4261 // affs[i].len = sizeof(<Affinities[i].second>);
4262 LValue LenLVal = CGF.EmitLValueForField(
4263 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4264 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4265 Idx = CGF.Builder.CreateNUWAdd(
4266 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4267 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4268 }
4269 }
4270 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4271 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4272 // naffins, kmp_task_affinity_info_t *affin_list);
4273 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4274 llvm::Value *GTid = getThreadID(CGF, Loc);
4275 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4276 V: AffinitiesArray.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy);
4277 // FIXME: Emit the function and ignore its result for now unless the
4278 // runtime function is properly implemented.
4279 (void)CGF.EmitRuntimeCall(
4280 callee: OMPBuilder.getOrCreateRuntimeFunction(
4281 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_reg_task_with_affinity),
4282 args: {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4283 }
4284 llvm::Value *NewTaskNewTaskTTy =
4285 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4286 V: NewTask, DestTy: KmpTaskTWithPrivatesPtrTy);
4287 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(V: NewTaskNewTaskTTy,
4288 T: KmpTaskTWithPrivatesQTy);
4289 LValue TDBase =
4290 CGF.EmitLValueForField(Base, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
4291 // Fill the data in the resulting kmp_task_t record.
4292 // Copy shareds if there are any.
4293 Address KmpTaskSharedsPtr = Address::invalid();
4294 if (!SharedsTy->castAsRecordDecl()->field_empty()) {
4295 KmpTaskSharedsPtr = Address(
4296 CGF.EmitLoadOfScalar(
4297 lvalue: CGF.EmitLValueForField(
4298 Base: TDBase,
4299 Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds)),
4300 Loc),
4301 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
4302 LValue Dest = CGF.MakeAddrLValue(Addr: KmpTaskSharedsPtr, T: SharedsTy);
4303 LValue Src = CGF.MakeAddrLValue(Addr: Shareds, T: SharedsTy);
4304 CGF.EmitAggregateCopy(Dest, Src, EltTy: SharedsTy, MayOverlap: AggValueSlot::DoesNotOverlap);
4305 }
4306 // Emit initial values for private copies (if any).
4307 TaskResultTy Result;
4308 if (!Privates.empty()) {
4309 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase: Base, KmpTaskTWithPrivatesQTyRD,
4310 SharedsTy, SharedsPtrTy, Data, Privates,
4311 /*ForDup=*/false);
4312 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) &&
4313 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4314 Result.TaskDupFn = emitTaskDupFunction(
4315 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4316 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4317 /*WithLastIter=*/!Data.LastprivateVars.empty());
4318 }
4319 }
4320 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4321 enum { Priority = 0, Destructors = 1 };
4322 // Provide pointer to function with destructors for privates.
4323 auto FI = std::next(x: KmpTaskTQTyRD->field_begin(), n: Data1);
4324 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
4325 assert(KmpCmplrdataUD->isUnion());
4326 if (NeedsCleanup) {
4327 llvm::Value *DestructorFn = emitDestructorsFunction(
4328 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4329 KmpTaskTWithPrivatesQTy);
4330 LValue Data1LV = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
4331 LValue DestructorsLV = CGF.EmitLValueForField(
4332 Base: Data1LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Destructors));
4333 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4334 V: DestructorFn, DestTy: KmpRoutineEntryPtrTy),
4335 lvalue: DestructorsLV);
4336 }
4337 // Set priority.
4338 if (Data.Priority.getInt()) {
4339 LValue Data2LV = CGF.EmitLValueForField(
4340 Base: TDBase, Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: Data2));
4341 LValue PriorityLV = CGF.EmitLValueForField(
4342 Base: Data2LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Priority));
4343 CGF.EmitStoreOfScalar(value: Data.Priority.getPointer(), lvalue: PriorityLV);
4344 }
4345 Result.NewTask = NewTask;
4346 Result.TaskEntry = TaskEntry;
4347 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4348 Result.TDBase = TDBase;
4349 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4350 return Result;
4351}
4352
4353/// Translates internal dependency kind into the runtime kind.
4354static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4355 RTLDependenceKindTy DepKind;
4356 switch (K) {
4357 case OMPC_DEPEND_in:
4358 DepKind = RTLDependenceKindTy::DepIn;
4359 break;
4360 // Out and InOut dependencies must use the same code.
4361 case OMPC_DEPEND_out:
4362 case OMPC_DEPEND_inout:
4363 DepKind = RTLDependenceKindTy::DepInOut;
4364 break;
4365 case OMPC_DEPEND_mutexinoutset:
4366 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4367 break;
4368 case OMPC_DEPEND_inoutset:
4369 DepKind = RTLDependenceKindTy::DepInOutSet;
4370 break;
4371 case OMPC_DEPEND_outallmemory:
4372 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4373 break;
4374 case OMPC_DEPEND_source:
4375 case OMPC_DEPEND_sink:
4376 case OMPC_DEPEND_depobj:
4377 case OMPC_DEPEND_inoutallmemory:
4378 case OMPC_DEPEND_unknown:
4379 llvm_unreachable("Unknown task dependence type");
4380 }
4381 return DepKind;
4382}
4383
4384/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4385static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4386 QualType &FlagsTy) {
4387 FlagsTy = C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.BoolTy), /*Signed=*/false);
4388 if (KmpDependInfoTy.isNull()) {
4389 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord(Name: "kmp_depend_info");
4390 KmpDependInfoRD->startDefinition();
4391 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getIntPtrType());
4392 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getSizeType());
4393 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: FlagsTy);
4394 KmpDependInfoRD->completeDefinition();
4395 KmpDependInfoTy = C.getCanonicalTagType(TD: KmpDependInfoRD);
4396 }
4397}
4398
4399std::pair<llvm::Value *, LValue>
4400CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4401 SourceLocation Loc) {
4402 ASTContext &C = CGM.getContext();
4403 QualType FlagsTy;
4404 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4405 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4406 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4407 LValue Base = CGF.EmitLoadOfPointerLValue(
4408 Ptr: DepobjLVal.getAddress().withElementType(
4409 ElemTy: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy)),
4410 PtrTy: KmpDependInfoPtrTy->castAs<PointerType>());
4411 Address DepObjAddr = CGF.Builder.CreateGEP(
4412 CGF, Addr: Base.getAddress(),
4413 Index: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4414 LValue NumDepsBase = CGF.MakeAddrLValue(
4415 Addr: DepObjAddr, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(), TBAAInfo: Base.getTBAAInfo());
4416 // NumDeps = deps[i].base_addr;
4417 LValue BaseAddrLVal = CGF.EmitLValueForField(
4418 Base: NumDepsBase,
4419 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4420 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4421 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(lvalue: BaseAddrLVal, Loc);
4422 return std::make_pair(x&: NumDeps, y&: Base);
4423}
4424
4425static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4426 llvm::PointerUnion<unsigned *, LValue *> Pos,
4427 const OMPTaskDataTy::DependData &Data,
4428 Address DependenciesArray) {
4429 CodeGenModule &CGM = CGF.CGM;
4430 ASTContext &C = CGM.getContext();
4431 QualType FlagsTy;
4432 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4433 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4434 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4435
4436 OMPIteratorGeneratorScope IteratorScope(
4437 CGF, cast_or_null<OMPIteratorExpr>(
4438 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4439 : nullptr));
4440 for (const Expr *E : Data.DepExprs) {
4441 llvm::Value *Addr;
4442 llvm::Value *Size;
4443
4444 // The expression will be a nullptr in the 'omp_all_memory' case.
4445 if (E) {
4446 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4447 Addr = CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy);
4448 } else {
4449 Addr = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4450 Size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
4451 }
4452 LValue Base;
4453 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4454 Base = CGF.MakeAddrLValue(
4455 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: *P), T: KmpDependInfoTy);
4456 } else {
4457 assert(E && "Expected a non-null expression");
4458 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4459 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4460 Base = CGF.MakeAddrLValue(
4461 Addr: CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Idx), T: KmpDependInfoTy);
4462 }
4463 // deps[i].base_addr = &<Dependencies[i].second>;
4464 LValue BaseAddrLVal = CGF.EmitLValueForField(
4465 Base,
4466 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4467 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4468 CGF.EmitStoreOfScalar(value: Addr, lvalue: BaseAddrLVal);
4469 // deps[i].len = sizeof(<Dependencies[i].second>);
4470 LValue LenLVal = CGF.EmitLValueForField(
4471 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4472 n: static_cast<unsigned int>(RTLDependInfoFields::Len)));
4473 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4474 // deps[i].flags = <Dependencies[i].first>;
4475 RTLDependenceKindTy DepKind = translateDependencyKind(K: Data.DepKind);
4476 LValue FlagsLVal = CGF.EmitLValueForField(
4477 Base,
4478 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4479 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4480 CGF.EmitStoreOfScalar(
4481 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4482 lvalue: FlagsLVal);
4483 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4484 ++(*P);
4485 } else {
4486 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4487 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4488 Idx = CGF.Builder.CreateNUWAdd(LHS: Idx,
4489 RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4490 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4491 }
4492 }
4493}
4494
4495SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes(
4496 CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4497 const OMPTaskDataTy::DependData &Data) {
4498 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4499 "Expected depobj dependency kind.");
4500 SmallVector<llvm::Value *, 4> Sizes;
4501 SmallVector<LValue, 4> SizeLVals;
4502 ASTContext &C = CGF.getContext();
4503 {
4504 OMPIteratorGeneratorScope IteratorScope(
4505 CGF, cast_or_null<OMPIteratorExpr>(
4506 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4507 : nullptr));
4508 for (const Expr *E : Data.DepExprs) {
4509 llvm::Value *NumDeps;
4510 LValue Base;
4511 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4512 std::tie(args&: NumDeps, args&: Base) =
4513 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4514 LValue NumLVal = CGF.MakeAddrLValue(
4515 Addr: CGF.CreateMemTempWithoutCast(T: C.getUIntPtrType(), Name: "depobj.size.addr"),
4516 T: C.getUIntPtrType());
4517 CGF.Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0),
4518 Addr: NumLVal.getAddress());
4519 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(lvalue: NumLVal, Loc: E->getExprLoc());
4520 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: PrevVal, RHS: NumDeps);
4521 CGF.EmitStoreOfScalar(value: Add, lvalue: NumLVal);
4522 SizeLVals.push_back(Elt: NumLVal);
4523 }
4524 }
4525 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4526 llvm::Value *Size =
4527 CGF.EmitLoadOfScalar(lvalue: SizeLVals[I], Loc: Data.DepExprs[I]->getExprLoc());
4528 Sizes.push_back(Elt: Size);
4529 }
4530 return Sizes;
4531}
4532
4533void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
4534 QualType &KmpDependInfoTy,
4535 LValue PosLVal,
4536 const OMPTaskDataTy::DependData &Data,
4537 Address DependenciesArray) {
4538 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4539 "Expected depobj dependency kind.");
4540 llvm::Value *ElSize = CGF.getTypeSize(Ty: KmpDependInfoTy);
4541 {
4542 OMPIteratorGeneratorScope IteratorScope(
4543 CGF, cast_or_null<OMPIteratorExpr>(
4544 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4545 : nullptr));
4546 for (const Expr *E : Data.DepExprs) {
4547 llvm::Value *NumDeps;
4548 LValue Base;
4549 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4550 std::tie(args&: NumDeps, args&: Base) =
4551 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4552
4553 // memcopy dependency data.
4554 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4555 LHS: ElSize,
4556 RHS: CGF.Builder.CreateIntCast(V: NumDeps, DestTy: CGF.SizeTy, /*isSigned=*/false));
4557 llvm::Value *Pos = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4558 Address DepAddr = CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Pos);
4559 CGF.Builder.CreateMemCpy(Dest: DepAddr, Src: Base.getAddress(), Size);
4560
4561 // Increase pos.
4562 // pos += size;
4563 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: Pos, RHS: NumDeps);
4564 CGF.EmitStoreOfScalar(value: Add, lvalue: PosLVal);
4565 }
4566 }
4567}
4568
4569std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4570 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4571 SourceLocation Loc) {
4572 if (llvm::all_of(Range&: Dependencies, P: [](const OMPTaskDataTy::DependData &D) {
4573 return D.DepExprs.empty();
4574 }))
4575 return std::make_pair(x: nullptr, y: Address::invalid());
4576 // Process list of dependencies.
4577 ASTContext &C = CGM.getContext();
4578 Address DependenciesArray = Address::invalid();
4579 llvm::Value *NumOfElements = nullptr;
4580 unsigned NumDependencies = std::accumulate(
4581 first: Dependencies.begin(), last: Dependencies.end(), init: 0,
4582 binary_op: [](unsigned V, const OMPTaskDataTy::DependData &D) {
4583 return D.DepKind == OMPC_DEPEND_depobj
4584 ? V
4585 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4586 });
4587 QualType FlagsTy;
4588 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4589 bool HasDepobjDeps = false;
4590 bool HasRegularWithIterators = false;
4591 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4592 llvm::Value *NumOfRegularWithIterators =
4593 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4594 // Calculate number of depobj dependencies and regular deps with the
4595 // iterators.
4596 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4597 if (D.DepKind == OMPC_DEPEND_depobj) {
4598 SmallVector<llvm::Value *, 4> Sizes =
4599 emitDepobjElementsSizes(CGF, KmpDependInfoTy, Data: D);
4600 for (llvm::Value *Size : Sizes) {
4601 NumOfDepobjElements =
4602 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: Size);
4603 }
4604 HasDepobjDeps = true;
4605 continue;
4606 }
4607 // Include number of iterations, if any.
4608
4609 if (const auto *IE = cast_or_null<OMPIteratorExpr>(Val: D.IteratorExpr)) {
4610 llvm::Value *ClauseIteratorSpace =
4611 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 1);
4612 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4613 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4614 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4615 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(LHS: Sz, RHS: ClauseIteratorSpace);
4616 }
4617 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4618 LHS: ClauseIteratorSpace,
4619 RHS: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: D.DepExprs.size()));
4620 NumOfRegularWithIterators =
4621 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumClauseDeps);
4622 HasRegularWithIterators = true;
4623 continue;
4624 }
4625 }
4626
4627 QualType KmpDependInfoArrayTy;
4628 if (HasDepobjDeps || HasRegularWithIterators) {
4629 NumOfElements = llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: NumDependencies,
4630 /*isSigned=*/IsSigned: false);
4631 if (HasDepobjDeps) {
4632 NumOfElements =
4633 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: NumOfElements);
4634 }
4635 if (HasRegularWithIterators) {
4636 NumOfElements =
4637 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumOfElements);
4638 }
4639 auto *OVE = new (C) OpaqueValueExpr(
4640 Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4641 VK_PRValue);
4642 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4643 RValue::get(V: NumOfElements));
4644 KmpDependInfoArrayTy =
4645 C.getVariableArrayType(EltTy: KmpDependInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4646 /*IndexTypeQuals=*/0);
4647 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4648 // Properly emit variable-sized array.
4649 auto *PD = ImplicitParamDecl::Create(C, T: KmpDependInfoArrayTy,
4650 ParamKind: ImplicitParamKind::Other);
4651 CGF.EmitVarDecl(D: *PD);
4652 DependenciesArray = CGF.GetAddrOfLocalVar(VD: PD);
4653 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4654 /*isSigned=*/false);
4655 } else {
4656 KmpDependInfoArrayTy = C.getConstantArrayType(
4657 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies), SizeExpr: nullptr,
4658 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4659 DependenciesArray =
4660 CGF.CreateMemTempWithoutCast(T: KmpDependInfoArrayTy, Name: ".dep.arr.addr");
4661 DependenciesArray = CGF.Builder.CreateConstArrayGEP(Addr: DependenciesArray, Index: 0);
4662 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumDependencies,
4663 /*isSigned=*/IsSigned: false);
4664 }
4665 unsigned Pos = 0;
4666 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4667 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4668 continue;
4669 emitDependData(CGF, KmpDependInfoTy, Pos: &Pos, Data: Dep, DependenciesArray);
4670 }
4671 // Copy regular dependencies with iterators.
4672 LValue PosLVal = CGF.MakeAddrLValue(
4673 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "dep.counter.addr"),
4674 T: C.getSizeType());
4675 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4676 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4677 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4678 continue;
4679 emitDependData(CGF, KmpDependInfoTy, Pos: &PosLVal, Data: Dep, DependenciesArray);
4680 }
4681 // Copy final depobj arrays without iterators.
4682 if (HasDepobjDeps) {
4683 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4684 if (Dep.DepKind != OMPC_DEPEND_depobj)
4685 continue;
4686 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Data: Dep, DependenciesArray);
4687 }
4688 }
4689 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4690 Addr: DependenciesArray, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
4691 return std::make_pair(x&: NumOfElements, y&: DependenciesArray);
4692}
4693
4694Address CGOpenMPRuntime::emitDepobjDependClause(
4695 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4696 SourceLocation Loc) {
4697 if (Dependencies.DepExprs.empty())
4698 return Address::invalid();
4699 // Process list of dependencies.
4700 ASTContext &C = CGM.getContext();
4701 Address DependenciesArray = Address::invalid();
4702 unsigned NumDependencies = Dependencies.DepExprs.size();
4703 QualType FlagsTy;
4704 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4705 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4706
4707 llvm::Value *Size;
4708 // Define type kmp_depend_info[<Dependencies.size()>];
4709 // For depobj reserve one extra element to store the number of elements.
4710 // It is required to handle depobj(x) update(in) construct.
4711 // kmp_depend_info[<Dependencies.size()>] deps;
4712 llvm::Value *NumDepsVal;
4713 CharUnits Align = C.getTypeAlignInChars(T: KmpDependInfoTy);
4714 if (const auto *IE =
4715 cast_or_null<OMPIteratorExpr>(Val: Dependencies.IteratorExpr)) {
4716 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
4717 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4718 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4719 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
4720 NumDepsVal = CGF.Builder.CreateNUWMul(LHS: NumDepsVal, RHS: Sz);
4721 }
4722 Size = CGF.Builder.CreateNUWAdd(LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1),
4723 RHS: NumDepsVal);
4724 CharUnits SizeInBytes =
4725 C.getTypeSizeInChars(T: KmpDependInfoTy).alignTo(Align);
4726 llvm::Value *RecSize = CGM.getSize(numChars: SizeInBytes);
4727 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: RecSize);
4728 NumDepsVal =
4729 CGF.Builder.CreateIntCast(V: NumDepsVal, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4730 } else {
4731 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4732 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4733 SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4734 CharUnits Sz = C.getTypeSizeInChars(T: KmpDependInfoArrayTy);
4735 Size = CGM.getSize(numChars: Sz.alignTo(Align));
4736 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: NumDependencies);
4737 }
4738 // Need to allocate on the dynamic memory.
4739 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4740 // Use default allocator.
4741 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4742 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4743
4744 llvm::Value *Addr =
4745 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4746 M&: CGM.getModule(), FnID: OMPRTL___kmpc_alloc),
4747 args: Args, name: ".dep.arr.addr");
4748 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(T: KmpDependInfoTy);
4749 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4750 V: Addr, DestTy: CGF.Builder.getPtrTy(AddrSpace: 0));
4751 DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
4752 // Write number of elements in the first element of array for depobj.
4753 LValue Base = CGF.MakeAddrLValue(Addr: DependenciesArray, T: KmpDependInfoTy);
4754 // deps[i].base_addr = NumDependencies;
4755 LValue BaseAddrLVal = CGF.EmitLValueForField(
4756 Base,
4757 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4758 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4759 CGF.EmitStoreOfScalar(value: NumDepsVal, lvalue: BaseAddrLVal);
4760 llvm::PointerUnion<unsigned *, LValue *> Pos;
4761 unsigned Idx = 1;
4762 LValue PosLVal;
4763 if (Dependencies.IteratorExpr) {
4764 PosLVal = CGF.MakeAddrLValue(
4765 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "iterator.counter.addr"),
4766 T: C.getSizeType());
4767 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Idx), lvalue: PosLVal,
4768 /*IsInit=*/isInit: true);
4769 Pos = &PosLVal;
4770 } else {
4771 Pos = &Idx;
4772 }
4773 emitDependData(CGF, KmpDependInfoTy, Pos, Data: Dependencies, DependenciesArray);
4774 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4775 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: 1), Ty: CGF.VoidPtrTy,
4776 ElementTy: CGF.Int8Ty);
4777 return DependenciesArray;
4778}
4779
4780void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4781 SourceLocation Loc) {
4782 ASTContext &C = CGM.getContext();
4783 QualType FlagsTy;
4784 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4785 LValue Base = CGF.EmitLoadOfPointerLValue(Ptr: DepobjLVal.getAddress(),
4786 PtrTy: C.VoidPtrTy.castAs<PointerType>());
4787 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4788 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4789 Addr: Base.getAddress(), Ty: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy),
4790 ElementTy: CGF.ConvertTypeForMem(T: KmpDependInfoTy));
4791 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4792 Ty: Addr.getElementType(), Ptr: Addr.emitRawPointer(CGF),
4793 IdxList: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4794 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: DepObjAddr,
4795 DestTy: CGF.VoidPtrTy);
4796 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4797 // Use default allocator.
4798 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4799 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4800
4801 // _kmpc_free(gtid, addr, nullptr);
4802 (void)CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4803 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free),
4804 args: Args);
4805}
4806
4807void CGOpenMPRuntime::emitUpdateDependObjectsClause(
4808 CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind,
4809 SourceLocation Loc) {
4810 ASTContext &C = CGM.getContext();
4811 QualType FlagsTy;
4812 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4813 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4814 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4815 llvm::Value *NumDeps;
4816 LValue Base;
4817 std::tie(args&: NumDeps, args&: Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4818
4819 Address Begin = Base.getAddress();
4820 // Cast from pointer to array type to pointer to single element.
4821 llvm::Value *End = CGF.Builder.CreateGEP(Ty: Begin.getElementType(),
4822 Ptr: Begin.emitRawPointer(CGF), IdxList: NumDeps);
4823 // The basic structure here is a while-do loop.
4824 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.body");
4825 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.done");
4826 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4827 CGF.EmitBlock(BB: BodyBB);
4828 llvm::PHINode *ElementPHI =
4829 CGF.Builder.CreatePHI(Ty: Begin.getType(), NumReservedValues: 2, Name: "omp.elementPast");
4830 ElementPHI->addIncoming(V: Begin.emitRawPointer(CGF), BB: EntryBB);
4831 Begin = Begin.withPointer(NewPointer: ElementPHI, IsKnownNonNull: KnownNonNull);
4832 Base = CGF.MakeAddrLValue(Addr: Begin, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(),
4833 TBAAInfo: Base.getTBAAInfo());
4834 // deps[i].flags = NewDepKind;
4835 RTLDependenceKindTy DepKind = translateDependencyKind(K: NewDepKind);
4836 LValue FlagsLVal = CGF.EmitLValueForField(
4837 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4838 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4839 CGF.EmitStoreOfScalar(
4840 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4841 lvalue: FlagsLVal);
4842
4843 // Shift the address forward by one element.
4844 llvm::Value *ElementNext =
4845 CGF.Builder.CreateConstGEP(Addr: Begin, /*Index=*/1, Name: "omp.elementNext")
4846 .emitRawPointer(CGF);
4847 ElementPHI->addIncoming(V: ElementNext, BB: CGF.Builder.GetInsertBlock());
4848 llvm::Value *IsEmpty =
4849 CGF.Builder.CreateICmpEQ(LHS: ElementNext, RHS: End, Name: "omp.isempty");
4850 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4851 // Done.
4852 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4853}
4854
4855void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4856 const OMPExecutableDirective &D,
4857 llvm::Function *TaskFunction,
4858 QualType SharedsTy, Address Shareds,
4859 const Expr *IfCond,
4860 const OMPTaskDataTy &Data) {
4861 if (!CGF.HaveInsertPoint())
4862 return;
4863
4864 TaskResultTy Result =
4865 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4866 llvm::Value *NewTask = Result.NewTask;
4867 llvm::Function *TaskEntry = Result.TaskEntry;
4868 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4869 LValue TDBase = Result.TDBase;
4870 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
4871 // Process list of dependences.
4872 Address DependenciesArray = Address::invalid();
4873 llvm::Value *NumOfElements;
4874 std::tie(args&: NumOfElements, args&: DependenciesArray) =
4875 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
4876
4877 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4878 // libcall.
4879 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
4880 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
4881 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
4882 // list is not empty
4883 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4884 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4885 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4886 llvm::Value *DepTaskArgs[7];
4887 if (!Data.Dependences.empty()) {
4888 DepTaskArgs[0] = UpLoc;
4889 DepTaskArgs[1] = ThreadID;
4890 DepTaskArgs[2] = NewTask;
4891 DepTaskArgs[3] = NumOfElements;
4892 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4893 DepTaskArgs[5] = CGF.Builder.getInt32(C: 0);
4894 DepTaskArgs[6] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4895 }
4896 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4897 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4898 if (!Data.Tied) {
4899 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
4900 LValue PartIdLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PartIdFI);
4901 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: 0), lvalue: PartIdLVal);
4902 }
4903 if (!Data.Dependences.empty()) {
4904 CGF.EmitRuntimeCall(
4905 callee: OMPBuilder.getOrCreateRuntimeFunction(
4906 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_with_deps),
4907 args: DepTaskArgs);
4908 } else {
4909 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4910 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
4911 args: TaskArgs);
4912 }
4913 // Check if parent region is untied and build return for untied task;
4914 if (auto *Region =
4915 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
4916 Region->emitUntiedSwitch(CGF);
4917 };
4918
4919 llvm::Value *DepWaitTaskArgs[7];
4920 if (!Data.Dependences.empty()) {
4921 DepWaitTaskArgs[0] = UpLoc;
4922 DepWaitTaskArgs[1] = ThreadID;
4923 DepWaitTaskArgs[2] = NumOfElements;
4924 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4925 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
4926 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4927 DepWaitTaskArgs[6] =
4928 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
4929 }
4930 auto &M = CGM.getModule();
4931 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4932 TaskEntry, &Data, &DepWaitTaskArgs,
4933 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4934 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4935 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
4936 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
4937 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
4938 // is specified.
4939 if (!Data.Dependences.empty())
4940 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4941 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
4942 args: DepWaitTaskArgs);
4943 // Call proxy_task_entry(gtid, new_task);
4944 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4945 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4946 Action.Enter(CGF);
4947 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4948 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskEntry,
4949 Args: OutlinedFnArgs);
4950 };
4951
4952 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
4953 // kmp_task_t *new_task);
4954 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
4955 // kmp_task_t *new_task);
4956 RegionCodeGenTy RCG(CodeGen);
4957 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
4958 M, FnID: OMPRTL___kmpc_omp_task_begin_if0),
4959 TaskArgs,
4960 OMPBuilder.getOrCreateRuntimeFunction(
4961 M, FnID: OMPRTL___kmpc_omp_task_complete_if0),
4962 TaskArgs);
4963 RCG.setAction(Action);
4964 RCG(CGF);
4965 };
4966
4967 if (IfCond) {
4968 emitIfClause(CGF, Cond: IfCond, ThenGen: ThenCodeGen, ElseGen: ElseCodeGen);
4969 } else {
4970 RegionCodeGenTy ThenRCG(ThenCodeGen);
4971 ThenRCG(CGF);
4972 }
4973}
4974
4975void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4976 const OMPLoopDirective &D,
4977 llvm::Function *TaskFunction,
4978 QualType SharedsTy, Address Shareds,
4979 const Expr *IfCond,
4980 const OMPTaskDataTy &Data) {
4981 if (!CGF.HaveInsertPoint())
4982 return;
4983 TaskResultTy Result =
4984 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4985 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4986 // libcall.
4987 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
4988 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
4989 // sched, kmp_uint64 grainsize, void *task_dup);
4990 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4991 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4992 llvm::Value *IfVal;
4993 if (IfCond) {
4994 IfVal = CGF.Builder.CreateIntCast(V: CGF.EvaluateExprAsBool(E: IfCond), DestTy: CGF.IntTy,
4995 /*isSigned=*/true);
4996 } else {
4997 IfVal = llvm::ConstantInt::getSigned(Ty: CGF.IntTy, /*V=*/1);
4998 }
4999
5000 LValue LBLVal = CGF.EmitLValueForField(
5001 Base: Result.TDBase,
5002 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound));
5003 const auto *LBVar =
5004 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getLowerBoundVariable())->getDecl());
5005 CGF.EmitAnyExprToMem(E: LBVar->getInit(), Location: LBLVal.getAddress(), Quals: LBLVal.getQuals(),
5006 /*IsInitializer=*/true);
5007 LValue UBLVal = CGF.EmitLValueForField(
5008 Base: Result.TDBase,
5009 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound));
5010 const auto *UBVar =
5011 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getUpperBoundVariable())->getDecl());
5012 CGF.EmitAnyExprToMem(E: UBVar->getInit(), Location: UBLVal.getAddress(), Quals: UBLVal.getQuals(),
5013 /*IsInitializer=*/true);
5014 LValue StLVal = CGF.EmitLValueForField(
5015 Base: Result.TDBase,
5016 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride));
5017 const auto *StVar =
5018 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getStrideVariable())->getDecl());
5019 CGF.EmitAnyExprToMem(E: StVar->getInit(), Location: StLVal.getAddress(), Quals: StLVal.getQuals(),
5020 /*IsInitializer=*/true);
5021 // Store reductions address.
5022 LValue RedLVal = CGF.EmitLValueForField(
5023 Base: Result.TDBase,
5024 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions));
5025 if (Data.Reductions) {
5026 CGF.EmitStoreOfScalar(value: Data.Reductions, lvalue: RedLVal);
5027 } else {
5028 CGF.EmitNullInitialization(DestPtr: RedLVal.getAddress(),
5029 Ty: CGF.getContext().VoidPtrTy);
5030 }
5031 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5032 llvm::SmallVector<llvm::Value *, 12> TaskArgs{
5033 UpLoc,
5034 ThreadID,
5035 Result.NewTask,
5036 IfVal,
5037 LBLVal.getPointer(CGF),
5038 UBLVal.getPointer(CGF),
5039 CGF.EmitLoadOfScalar(lvalue: StLVal, Loc),
5040 llvm::ConstantInt::getSigned(
5041 Ty: CGF.IntTy, V: 1), // Always 1 because taskgroup emitted by the compiler
5042 llvm::ConstantInt::getSigned(
5043 Ty: CGF.IntTy, V: Data.Schedule.getPointer()
5044 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5045 : NoSchedule),
5046 Data.Schedule.getPointer()
5047 ? CGF.Builder.CreateIntCast(V: Data.Schedule.getPointer(), DestTy: CGF.Int64Ty,
5048 /*isSigned=*/false)
5049 : llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/0)};
5050 if (Data.HasModifier)
5051 TaskArgs.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 1));
5052
5053 TaskArgs.push_back(Elt: Result.TaskDupFn
5054 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5055 V: Result.TaskDupFn, DestTy: CGF.VoidPtrTy)
5056 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy));
5057 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5058 M&: CGM.getModule(), FnID: Data.HasModifier
5059 ? OMPRTL___kmpc_taskloop_5
5060 : OMPRTL___kmpc_taskloop),
5061 args: TaskArgs);
5062}
5063
5064/// Emit reduction operation for each element of array (required for
5065/// array sections) LHS op = RHS.
5066/// \param Type Type of array.
5067/// \param LHSVar Variable on the left side of the reduction operation
5068/// (references element of array in original variable).
5069/// \param RHSVar Variable on the right side of the reduction operation
5070/// (references element of array in original variable).
5071/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5072/// RHSVar.
5073static void EmitOMPAggregateReduction(
5074 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5075 const VarDecl *RHSVar,
5076 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5077 const Expr *, const Expr *)> &RedOpGen,
5078 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5079 const Expr *UpExpr = nullptr) {
5080 // Perform element-by-element initialization.
5081 QualType ElementTy;
5082 Address LHSAddr = CGF.GetAddrOfLocalVar(VD: LHSVar);
5083 Address RHSAddr = CGF.GetAddrOfLocalVar(VD: RHSVar);
5084
5085 // Drill down to the base element type on both arrays.
5086 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5087 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: LHSAddr);
5088
5089 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
5090 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
5091 // Cast from pointer to array type to pointer to single element.
5092 llvm::Value *LHSEnd =
5093 CGF.Builder.CreateGEP(Ty: LHSAddr.getElementType(), Ptr: LHSBegin, IdxList: NumElements);
5094 // The basic structure here is a while-do loop.
5095 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arraycpy.body");
5096 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arraycpy.done");
5097 llvm::Value *IsEmpty =
5098 CGF.Builder.CreateICmpEQ(LHS: LHSBegin, RHS: LHSEnd, Name: "omp.arraycpy.isempty");
5099 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
5100
5101 // Enter the loop body, making that address the current address.
5102 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5103 CGF.EmitBlock(BB: BodyBB);
5104
5105 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
5106
5107 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5108 Ty: RHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
5109 RHSElementPHI->addIncoming(V: RHSBegin, BB: EntryBB);
5110 Address RHSElementCurrent(
5111 RHSElementPHI, RHSAddr.getElementType(),
5112 RHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
5113
5114 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5115 Ty: LHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
5116 LHSElementPHI->addIncoming(V: LHSBegin, BB: EntryBB);
5117 Address LHSElementCurrent(
5118 LHSElementPHI, LHSAddr.getElementType(),
5119 LHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
5120
5121 // Emit copy.
5122 CodeGenFunction::OMPPrivateScope Scope(CGF);
5123 Scope.addPrivate(LocalVD: LHSVar, Addr: LHSElementCurrent);
5124 Scope.addPrivate(LocalVD: RHSVar, Addr: RHSElementCurrent);
5125 Scope.Privatize();
5126 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5127 Scope.ForceCleanup();
5128
5129 // Shift the address forward by one element.
5130 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5131 Ty: LHSAddr.getElementType(), Ptr: LHSElementPHI, /*Idx0=*/1,
5132 Name: "omp.arraycpy.dest.element");
5133 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5134 Ty: RHSAddr.getElementType(), Ptr: RHSElementPHI, /*Idx0=*/1,
5135 Name: "omp.arraycpy.src.element");
5136 // Check whether we've reached the end.
5137 llvm::Value *Done =
5138 CGF.Builder.CreateICmpEQ(LHS: LHSElementNext, RHS: LHSEnd, Name: "omp.arraycpy.done");
5139 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
5140 LHSElementPHI->addIncoming(V: LHSElementNext, BB: CGF.Builder.GetInsertBlock());
5141 RHSElementPHI->addIncoming(V: RHSElementNext, BB: CGF.Builder.GetInsertBlock());
5142
5143 // Done.
5144 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
5145}
5146
5147/// Emit reduction combiner. If the combiner is a simple expression emit it as
5148/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5149/// UDR combiner function.
5150static void emitReductionCombiner(CodeGenFunction &CGF,
5151 const Expr *ReductionOp) {
5152 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
5153 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
5154 if (const auto *DRE =
5155 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
5156 if (const auto *DRD =
5157 dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl())) {
5158 std::pair<llvm::Function *, llvm::Function *> Reduction =
5159 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
5160 RValue Func = RValue::get(V: Reduction.first);
5161 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5162 CGF.EmitIgnoredExpr(E: ReductionOp);
5163 return;
5164 }
5165 CGF.EmitIgnoredExpr(E: ReductionOp);
5166}
5167
5168llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5169 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
5170 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
5171 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
5172 ASTContext &C = CGM.getContext();
5173
5174 // void reduction_func(void *LHSArg, void *RHSArg);
5175 auto *LHSArg =
5176 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5177 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5178 auto *RHSArg =
5179 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5180 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5181 FunctionArgList Args{LHSArg, RHSArg};
5182 const auto &CGFI =
5183 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5184 std::string Name = getReductionFuncName(Name: ReducerName);
5185 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
5186 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
5187 M: &CGM.getModule());
5188 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
5189 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5190 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5191 Fn->setDoesNotRecurse();
5192 CodeGenFunction CGF(CGM);
5193 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
5194
5195 // Dst = (void*[n])(LHSArg);
5196 // Src = (void*[n])(RHSArg);
5197 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5198 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
5199 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5200 ArgsElemType, CGF.getPointerAlign());
5201 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5202 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
5203 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5204 ArgsElemType, CGF.getPointerAlign());
5205
5206 // ...
5207 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5208 // ...
5209 CodeGenFunction::OMPPrivateScope Scope(CGF);
5210 const auto *IPriv = Privates.begin();
5211 unsigned Idx = 0;
5212 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5213 const auto *RHSVar =
5214 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSExprs[I])->getDecl());
5215 Scope.addPrivate(LocalVD: RHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: RHS, Index: Idx, Var: RHSVar));
5216 const auto *LHSVar =
5217 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSExprs[I])->getDecl());
5218 Scope.addPrivate(LocalVD: LHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: LHS, Index: Idx, Var: LHSVar));
5219 QualType PrivTy = (*IPriv)->getType();
5220 if (PrivTy->isVariablyModifiedType()) {
5221 // Get array size and emit VLA type.
5222 ++Idx;
5223 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: LHS, Index: Idx);
5224 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Elem);
5225 const VariableArrayType *VLA =
5226 CGF.getContext().getAsVariableArrayType(T: PrivTy);
5227 const auto *OVE = cast<OpaqueValueExpr>(Val: VLA->getSizeExpr());
5228 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5229 CGF, OVE, RValue::get(V: CGF.Builder.CreatePtrToInt(V: Ptr, DestTy: CGF.SizeTy)));
5230 CGF.EmitVariablyModifiedType(Ty: PrivTy);
5231 }
5232 }
5233 Scope.Privatize();
5234 IPriv = Privates.begin();
5235 const auto *ILHS = LHSExprs.begin();
5236 const auto *IRHS = RHSExprs.begin();
5237 for (const Expr *E : ReductionOps) {
5238 if ((*IPriv)->getType()->isArrayType()) {
5239 // Emit reduction for array section.
5240 const auto *LHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5241 const auto *RHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5242 EmitOMPAggregateReduction(
5243 CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5244 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5245 emitReductionCombiner(CGF, ReductionOp: E);
5246 });
5247 } else {
5248 // Emit reduction for array subscript or single variable.
5249 emitReductionCombiner(CGF, ReductionOp: E);
5250 }
5251 ++IPriv;
5252 ++ILHS;
5253 ++IRHS;
5254 }
5255 Scope.ForceCleanup();
5256 CGF.FinishFunction();
5257 return Fn;
5258}
5259
5260void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5261 const Expr *ReductionOp,
5262 const Expr *PrivateRef,
5263 const DeclRefExpr *LHS,
5264 const DeclRefExpr *RHS) {
5265 if (PrivateRef->getType()->isArrayType()) {
5266 // Emit reduction for array section.
5267 const auto *LHSVar = cast<VarDecl>(Val: LHS->getDecl());
5268 const auto *RHSVar = cast<VarDecl>(Val: RHS->getDecl());
5269 EmitOMPAggregateReduction(
5270 CGF, Type: PrivateRef->getType(), LHSVar, RHSVar,
5271 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5272 emitReductionCombiner(CGF, ReductionOp);
5273 });
5274 } else {
5275 // Emit reduction for array subscript or single variable.
5276 emitReductionCombiner(CGF, ReductionOp);
5277 }
5278}
5279
5280static std::string generateUniqueName(CodeGenModule &CGM,
5281 llvm::StringRef Prefix, const Expr *Ref);
5282
5283void CGOpenMPRuntime::emitPrivateReduction(
5284 CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,
5285 const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {
5286
5287 // Create a shared global variable (__shared_reduction_var) to accumulate the
5288 // final result.
5289 //
5290 // Call __kmpc_barrier to synchronize threads before initialization.
5291 //
5292 // The master thread (thread_id == 0) initializes __shared_reduction_var
5293 // with the identity value or initializer.
5294 //
5295 // Call __kmpc_barrier to synchronize before combining.
5296 // For each i:
5297 // - Thread enters critical section.
5298 // - Reads its private value from LHSExprs[i].
5299 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
5300 // Privates[i]).
5301 // - Exits critical section.
5302 //
5303 // Call __kmpc_barrier after combining.
5304 //
5305 // Each thread copies __shared_reduction_var[i] back to RHSExprs[i].
5306 //
5307 // Final __kmpc_barrier to synchronize after broadcasting
5308 QualType PrivateType = Privates->getType();
5309 llvm::Type *LLVMType = CGF.ConvertTypeForMem(T: PrivateType);
5310
5311 const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOp: ReductionOps);
5312 std::string ReductionVarNameStr;
5313 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates->IgnoreParenCasts()))
5314 ReductionVarNameStr =
5315 generateUniqueName(CGM, Prefix: DRE->getDecl()->getNameAsString(), Ref: Privates);
5316 else
5317 ReductionVarNameStr = "unnamed_priv_var";
5318
5319 // Create an internal shared variable
5320 std::string SharedName =
5321 CGM.getOpenMPRuntime().getName(Parts: {"internal_pivate_", ReductionVarNameStr});
5322 llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(
5323 Ty: LLVMType, Name: ".omp.reduction." + SharedName);
5324
5325 SharedVar->setAlignment(
5326 llvm::MaybeAlign(CGF.getContext().getTypeAlign(T: PrivateType) / 8));
5327
5328 Address SharedResult =
5329 CGF.MakeNaturalAlignRawAddrLValue(V: SharedVar, T: PrivateType).getAddress();
5330
5331 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5332 llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5333 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
5334
5335 llvm::BasicBlock *InitBB = CGF.createBasicBlock(name: "init");
5336 llvm::BasicBlock *InitEndBB = CGF.createBasicBlock(name: "init.end");
5337
5338 llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(
5339 LHS: ThreadId, RHS: llvm::ConstantInt::get(Ty: ThreadId->getType(), V: 0));
5340 CGF.Builder.CreateCondBr(Cond: IsWorker, True: InitBB, False: InitEndBB);
5341
5342 CGF.EmitBlock(BB: InitBB);
5343
5344 auto EmitSharedInit = [&]() {
5345 if (UDR) { // Check if it's a User-Defined Reduction
5346 if (const Expr *UDRInitExpr = UDR->getInitializer()) {
5347 std::pair<llvm::Function *, llvm::Function *> FnPair =
5348 getUserDefinedReduction(D: UDR);
5349 llvm::Function *InitializerFn = FnPair.second;
5350 if (InitializerFn) {
5351 if (const auto *CE =
5352 dyn_cast<CallExpr>(Val: UDRInitExpr->IgnoreParenImpCasts())) {
5353 const auto *OutDRE = cast<DeclRefExpr>(
5354 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5355 ->getSubExpr());
5356 const VarDecl *OutVD = cast<VarDecl>(Val: OutDRE->getDecl());
5357
5358 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5359 LocalScope.addPrivate(LocalVD: OutVD, Addr: SharedResult);
5360
5361 (void)LocalScope.Privatize();
5362 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(
5363 Val: CE->getCallee()->IgnoreParenImpCasts())) {
5364 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5365 CGF, OVE, RValue::get(V: InitializerFn));
5366 CGF.EmitIgnoredExpr(E: CE);
5367 } else {
5368 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5369 Quals: PrivateType.getQualifiers(),
5370 /*IsInitializer=*/true);
5371 }
5372 } else {
5373 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5374 Quals: PrivateType.getQualifiers(),
5375 /*IsInitializer=*/true);
5376 }
5377 } else {
5378 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5379 Quals: PrivateType.getQualifiers(),
5380 /*IsInitializer=*/true);
5381 }
5382 } else {
5383 // EmitNullInitialization handles default construction for C++ classes
5384 // and zeroing for scalars, which is a reasonable default.
5385 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5386 }
5387 return; // UDR initialization handled
5388 }
5389 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates)) {
5390 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
5391 if (const Expr *InitExpr = VD->getInit()) {
5392 CGF.EmitAnyExprToMem(E: InitExpr, Location: SharedResult,
5393 Quals: PrivateType.getQualifiers(), IsInitializer: true);
5394 return;
5395 }
5396 }
5397 }
5398 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5399 };
5400 EmitSharedInit();
5401 CGF.Builder.CreateBr(Dest: InitEndBB);
5402 CGF.EmitBlock(BB: InitEndBB);
5403
5404 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5405 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5406 args: BarrierArgs);
5407
5408 const Expr *ReductionOp = ReductionOps;
5409 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
5410 LValue SharedLV = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5411 LValue LHSLV = CGF.EmitLValue(E: Privates);
5412
5413 auto EmitCriticalReduction = [&](auto ReductionGen) {
5414 std::string CriticalName = getName(Parts: {"reduction_critical"});
5415 emitCriticalRegion(CGF, CriticalName, CriticalOpGen: ReductionGen, Loc);
5416 };
5417
5418 if (CurrentUDR) {
5419 // Handle user-defined reduction.
5420 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5421 Action.Enter(CGF);
5422 std::pair<llvm::Function *, llvm::Function *> FnPair =
5423 getUserDefinedReduction(D: CurrentUDR);
5424 if (FnPair.first) {
5425 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp)) {
5426 const auto *OutDRE = cast<DeclRefExpr>(
5427 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5428 ->getSubExpr());
5429 const auto *InDRE = cast<DeclRefExpr>(
5430 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 1)->IgnoreParenImpCasts())
5431 ->getSubExpr());
5432 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5433 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: OutDRE->getDecl()),
5434 Addr: SharedLV.getAddress());
5435 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: InDRE->getDecl()),
5436 Addr: LHSLV.getAddress());
5437 (void)LocalScope.Privatize();
5438 emitReductionCombiner(CGF, ReductionOp);
5439 }
5440 }
5441 };
5442 EmitCriticalReduction(ReductionGen);
5443 } else {
5444 // Handle built-in reduction operations.
5445#ifndef NDEBUG
5446 const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();
5447 if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5448 ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
5449
5450 const Expr *AssignRHS = nullptr;
5451 if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5452 if (BinOp->getOpcode() == BO_Assign)
5453 AssignRHS = BinOp->getRHS();
5454 } else if (const auto *OpCall =
5455 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5456 if (OpCall->getOperator() == OO_Equal)
5457 AssignRHS = OpCall->getArg(1);
5458 }
5459
5460 assert(AssignRHS &&
5461 "Private Variable Reduction : Invalid ReductionOp expression");
5462#endif
5463
5464 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5465 Action.Enter(CGF);
5466 const auto *OmpOutDRE =
5467 dyn_cast<DeclRefExpr>(Val: LHSExprs->IgnoreParenImpCasts());
5468 const auto *OmpInDRE =
5469 dyn_cast<DeclRefExpr>(Val: RHSExprs->IgnoreParenImpCasts());
5470 assert(
5471 OmpOutDRE && OmpInDRE &&
5472 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5473 const VarDecl *OmpOutVD = cast<VarDecl>(Val: OmpOutDRE->getDecl());
5474 const VarDecl *OmpInVD = cast<VarDecl>(Val: OmpInDRE->getDecl());
5475 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5476 LocalScope.addPrivate(LocalVD: OmpOutVD, Addr: SharedLV.getAddress());
5477 LocalScope.addPrivate(LocalVD: OmpInVD, Addr: LHSLV.getAddress());
5478 (void)LocalScope.Privatize();
5479 // Emit the actual reduction operation
5480 CGF.EmitIgnoredExpr(E: ReductionOp);
5481 };
5482 EmitCriticalReduction(ReductionGen);
5483 }
5484
5485 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5486 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5487 args: BarrierArgs);
5488
5489 // Broadcast final result
5490 bool IsAggregate = PrivateType->isAggregateType();
5491 LValue SharedLV1 = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5492 llvm::Value *FinalResultVal = nullptr;
5493 Address FinalResultAddr = Address::invalid();
5494
5495 if (IsAggregate)
5496 FinalResultAddr = SharedResult;
5497 else
5498 FinalResultVal = CGF.EmitLoadOfScalar(lvalue: SharedLV1, Loc);
5499
5500 LValue TargetLHSLV = CGF.EmitLValue(E: RHSExprs);
5501 if (IsAggregate) {
5502 CGF.EmitAggregateCopy(Dest: TargetLHSLV,
5503 Src: CGF.MakeAddrLValue(Addr: FinalResultAddr, T: PrivateType),
5504 EltTy: PrivateType, MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: false);
5505 } else {
5506 CGF.EmitStoreOfScalar(value: FinalResultVal, lvalue: TargetLHSLV);
5507 }
5508 // Final synchronization barrier
5509 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5510 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5511 args: BarrierArgs);
5512
5513 // Combiner with original list item
5514 auto OriginalListCombiner = [&](CodeGenFunction &CGF,
5515 PrePostActionTy &Action) {
5516 Action.Enter(CGF);
5517 emitSingleReductionCombiner(CGF, ReductionOp: ReductionOps, PrivateRef: Privates,
5518 LHS: cast<DeclRefExpr>(Val: LHSExprs),
5519 RHS: cast<DeclRefExpr>(Val: RHSExprs));
5520 };
5521 EmitCriticalReduction(OriginalListCombiner);
5522}
5523
5524void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5525 ArrayRef<const Expr *> OrgPrivates,
5526 ArrayRef<const Expr *> OrgLHSExprs,
5527 ArrayRef<const Expr *> OrgRHSExprs,
5528 ArrayRef<const Expr *> OrgReductionOps,
5529 ReductionOptionsTy Options) {
5530 if (!CGF.HaveInsertPoint())
5531 return;
5532
5533 bool WithNowait = Options.WithNowait;
5534 bool SimpleReduction = Options.SimpleReduction;
5535
5536 // Next code should be emitted for reduction:
5537 //
5538 // static kmp_critical_name lock = { 0 };
5539 //
5540 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5541 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5542 // ...
5543 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5544 // *(Type<n>-1*)rhs[<n>-1]);
5545 // }
5546 //
5547 // ...
5548 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5549 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5550 // RedList, reduce_func, &<lock>)) {
5551 // case 1:
5552 // ...
5553 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5554 // ...
5555 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5556 // break;
5557 // case 2:
5558 // ...
5559 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5560 // ...
5561 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5562 // break;
5563 // default:;
5564 // }
5565 //
5566 // if SimpleReduction is true, only the next code is generated:
5567 // ...
5568 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5569 // ...
5570
5571 ASTContext &C = CGM.getContext();
5572
5573 if (SimpleReduction) {
5574 CodeGenFunction::RunCleanupsScope Scope(CGF);
5575 const auto *IPriv = OrgPrivates.begin();
5576 const auto *ILHS = OrgLHSExprs.begin();
5577 const auto *IRHS = OrgRHSExprs.begin();
5578 for (const Expr *E : OrgReductionOps) {
5579 emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5580 RHS: cast<DeclRefExpr>(Val: *IRHS));
5581 ++IPriv;
5582 ++ILHS;
5583 ++IRHS;
5584 }
5585 return;
5586 }
5587
5588 // Filter out shared reduction variables based on IsPrivateVarReduction flag.
5589 // Only keep entries where the corresponding variable is not private.
5590 SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
5591 FilteredRHSExprs, FilteredReductionOps;
5592 for (unsigned I : llvm::seq<unsigned>(
5593 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5594 if (!Options.IsPrivateVarReduction[I]) {
5595 FilteredPrivates.emplace_back(Args: OrgPrivates[I]);
5596 FilteredLHSExprs.emplace_back(Args: OrgLHSExprs[I]);
5597 FilteredRHSExprs.emplace_back(Args: OrgRHSExprs[I]);
5598 FilteredReductionOps.emplace_back(Args: OrgReductionOps[I]);
5599 }
5600 }
5601 // Wrap filtered vectors in ArrayRef for downstream shared reduction
5602 // processing.
5603 ArrayRef<const Expr *> Privates = FilteredPrivates;
5604 ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
5605 ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
5606 ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
5607
5608 // 1. Build a list of reduction variables.
5609 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5610 auto Size = RHSExprs.size();
5611 for (const Expr *E : Privates) {
5612 if (E->getType()->isVariablyModifiedType())
5613 // Reserve place for array size.
5614 ++Size;
5615 }
5616 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5617 QualType ReductionArrayTy = C.getConstantArrayType(
5618 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5619 /*IndexTypeQuals=*/0);
5620 RawAddress ReductionList =
5621 CGF.CreateMemTemp(T: ReductionArrayTy, Name: ".omp.reduction.red_list");
5622 const auto *IPriv = Privates.begin();
5623 unsigned Idx = 0;
5624 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5625 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5626 CGF.Builder.CreateStore(
5627 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5628 V: CGF.EmitLValue(E: RHSExprs[I]).getPointer(CGF), DestTy: CGF.VoidPtrTy),
5629 Addr: Elem);
5630 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5631 // Store array size.
5632 ++Idx;
5633 Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5634 llvm::Value *Size = CGF.Builder.CreateIntCast(
5635 V: CGF.getVLASize(
5636 vla: CGF.getContext().getAsVariableArrayType(T: (*IPriv)->getType()))
5637 .NumElts,
5638 DestTy: CGF.SizeTy, /*isSigned=*/false);
5639 CGF.Builder.CreateStore(Val: CGF.Builder.CreateIntToPtr(V: Size, DestTy: CGF.VoidPtrTy),
5640 Addr: Elem);
5641 }
5642 }
5643
5644 // 2. Emit reduce_func().
5645 llvm::Function *ReductionFn = emitReductionFunction(
5646 ReducerName: CGF.CurFn->getName(), Loc, ArgsElemType: CGF.ConvertTypeForMem(T: ReductionArrayTy),
5647 Privates, LHSExprs, RHSExprs, ReductionOps);
5648
5649 // 3. Create static kmp_critical_name lock = { 0 };
5650 std::string Name = getName(Parts: {"reduction"});
5651 llvm::Value *Lock = getCriticalRegionLock(CriticalName: Name);
5652
5653 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5654 // RedList, reduce_func, &<lock>);
5655 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5656 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5657 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(Ty: ReductionArrayTy);
5658 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5659 V: ReductionList.getPointer(), DestTy: CGF.VoidPtrTy);
5660 llvm::Value *Args[] = {
5661 IdentTLoc, // ident_t *<loc>
5662 ThreadId, // i32 <gtid>
5663 CGF.Builder.getInt32(C: RHSExprs.size()), // i32 <n>
5664 ReductionArrayTySize, // size_type sizeof(RedList)
5665 RL, // void *RedList
5666 ReductionFn, // void (*) (void *, void *) <reduce_func>
5667 Lock // kmp_critical_name *&<lock>
5668 };
5669 llvm::Value *Res = CGF.EmitRuntimeCall(
5670 callee: OMPBuilder.getOrCreateRuntimeFunction(
5671 M&: CGM.getModule(),
5672 FnID: WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5673 args: Args);
5674
5675 // 5. Build switch(res)
5676 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(name: ".omp.reduction.default");
5677 llvm::SwitchInst *SwInst =
5678 CGF.Builder.CreateSwitch(V: Res, Dest: DefaultBB, /*NumCases=*/2);
5679
5680 // 6. Build case 1:
5681 // ...
5682 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5683 // ...
5684 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5685 // break;
5686 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(name: ".omp.reduction.case1");
5687 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 1), Dest: Case1BB);
5688 CGF.EmitBlock(BB: Case1BB);
5689
5690 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5691 llvm::Value *EndArgs[] = {
5692 IdentTLoc, // ident_t *<loc>
5693 ThreadId, // i32 <gtid>
5694 Lock // kmp_critical_name *&<lock>
5695 };
5696 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5697 CodeGenFunction &CGF, PrePostActionTy &Action) {
5698 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5699 const auto *IPriv = Privates.begin();
5700 const auto *ILHS = LHSExprs.begin();
5701 const auto *IRHS = RHSExprs.begin();
5702 for (const Expr *E : ReductionOps) {
5703 RT.emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5704 RHS: cast<DeclRefExpr>(Val: *IRHS));
5705 ++IPriv;
5706 ++ILHS;
5707 ++IRHS;
5708 }
5709 };
5710 RegionCodeGenTy RCG(CodeGen);
5711 CommonActionTy Action(
5712 nullptr, {},
5713 OMPBuilder.getOrCreateRuntimeFunction(
5714 M&: CGM.getModule(), FnID: WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5715 : OMPRTL___kmpc_end_reduce),
5716 EndArgs);
5717 RCG.setAction(Action);
5718 RCG(CGF);
5719
5720 CGF.EmitBranch(Block: DefaultBB);
5721
5722 // 7. Build case 2:
5723 // ...
5724 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5725 // ...
5726 // break;
5727 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(name: ".omp.reduction.case2");
5728 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 2), Dest: Case2BB);
5729 CGF.EmitBlock(BB: Case2BB);
5730
5731 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5732 CodeGenFunction &CGF, PrePostActionTy &Action) {
5733 const auto *ILHS = LHSExprs.begin();
5734 const auto *IRHS = RHSExprs.begin();
5735 const auto *IPriv = Privates.begin();
5736 for (const Expr *E : ReductionOps) {
5737 const Expr *XExpr = nullptr;
5738 const Expr *EExpr = nullptr;
5739 const Expr *UpExpr = nullptr;
5740 BinaryOperatorKind BO = BO_Comma;
5741 if (const auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
5742 if (BO->getOpcode() == BO_Assign) {
5743 XExpr = BO->getLHS();
5744 UpExpr = BO->getRHS();
5745 }
5746 }
5747 // Try to emit update expression as a simple atomic.
5748 const Expr *RHSExpr = UpExpr;
5749 if (RHSExpr) {
5750 // Analyze RHS part of the whole expression.
5751 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5752 Val: RHSExpr->IgnoreParenImpCasts())) {
5753 // If this is a conditional operator, analyze its condition for
5754 // min/max reduction operator.
5755 RHSExpr = ACO->getCond();
5756 }
5757 if (const auto *BORHS =
5758 dyn_cast<BinaryOperator>(Val: RHSExpr->IgnoreParenImpCasts())) {
5759 EExpr = BORHS->getRHS();
5760 BO = BORHS->getOpcode();
5761 }
5762 }
5763 if (XExpr) {
5764 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5765 auto &&AtomicRedGen = [BO, VD,
5766 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5767 const Expr *EExpr, const Expr *UpExpr) {
5768 LValue X = CGF.EmitLValue(E: XExpr);
5769 RValue E;
5770 if (EExpr)
5771 E = CGF.EmitAnyExpr(E: EExpr);
5772 CGF.EmitOMPAtomicSimpleUpdateExpr(
5773 X, E, BO, /*IsXLHSInRHSPart=*/true,
5774 AO: llvm::AtomicOrdering::Monotonic, Loc,
5775 CommonGen: [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5776 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5777 Address LHSTemp = CGF.CreateMemTemp(T: VD->getType());
5778 CGF.emitOMPSimpleStore(
5779 LVal: CGF.MakeAddrLValue(Addr: LHSTemp, T: VD->getType()), RVal: XRValue,
5780 RValTy: VD->getType().getNonReferenceType(), Loc);
5781 PrivateScope.addPrivate(LocalVD: VD, Addr: LHSTemp);
5782 (void)PrivateScope.Privatize();
5783 return CGF.EmitAnyExpr(E: UpExpr);
5784 });
5785 };
5786 if ((*IPriv)->getType()->isArrayType()) {
5787 // Emit atomic reduction for array section.
5788 const auto *RHSVar =
5789 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5790 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar: VD, RHSVar,
5791 RedOpGen: AtomicRedGen, XExpr, EExpr, UpExpr);
5792 } else {
5793 // Emit atomic reduction for array subscript or single variable.
5794 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5795 }
5796 } else {
5797 // Emit as a critical region.
5798 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5799 const Expr *, const Expr *) {
5800 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5801 std::string Name = RT.getName(Parts: {"atomic_reduction"});
5802 RT.emitCriticalRegion(
5803 CGF, CriticalName: Name,
5804 CriticalOpGen: [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5805 Action.Enter(CGF);
5806 emitReductionCombiner(CGF, ReductionOp: E);
5807 },
5808 Loc);
5809 };
5810 if ((*IPriv)->getType()->isArrayType()) {
5811 const auto *LHSVar =
5812 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5813 const auto *RHSVar =
5814 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5815 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5816 RedOpGen: CritRedGen);
5817 } else {
5818 CritRedGen(CGF, nullptr, nullptr, nullptr);
5819 }
5820 }
5821 ++ILHS;
5822 ++IRHS;
5823 ++IPriv;
5824 }
5825 };
5826 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5827 if (!WithNowait) {
5828 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5829 llvm::Value *EndArgs[] = {
5830 IdentTLoc, // ident_t *<loc>
5831 ThreadId, // i32 <gtid>
5832 Lock // kmp_critical_name *&<lock>
5833 };
5834 CommonActionTy Action(nullptr, {},
5835 OMPBuilder.getOrCreateRuntimeFunction(
5836 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_reduce),
5837 EndArgs);
5838 AtomicRCG.setAction(Action);
5839 AtomicRCG(CGF);
5840 } else {
5841 AtomicRCG(CGF);
5842 }
5843
5844 CGF.EmitBranch(Block: DefaultBB);
5845 CGF.EmitBlock(BB: DefaultBB, /*IsFinished=*/true);
5846 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5847 "PrivateVarReduction: Privates size mismatch");
5848 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5849 "PrivateVarReduction: ReductionOps size mismatch");
5850 for (unsigned I : llvm::seq<unsigned>(
5851 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5852 if (Options.IsPrivateVarReduction[I])
5853 emitPrivateReduction(CGF, Loc, Privates: OrgPrivates[I], LHSExprs: OrgLHSExprs[I],
5854 RHSExprs: OrgRHSExprs[I], ReductionOps: OrgReductionOps[I]);
5855 }
5856}
5857
5858/// Generates unique name for artificial threadprivate variables.
5859/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5860static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5861 const Expr *Ref) {
5862 SmallString<256> Buffer;
5863 llvm::raw_svector_ostream Out(Buffer);
5864 const clang::DeclRefExpr *DE;
5865 const VarDecl *D = ::getBaseDecl(Ref, DE);
5866 if (!D) {
5867 auto *DRE = cast<DeclRefExpr>(Val: Ref);
5868 if (const auto *BD = dyn_cast<BindingDecl>(Val: DRE->getDecl())) {
5869 // For BindingDecls, use the decomposed declaration as the base.
5870 D = cast<VarDecl>(Val: BD->getDecomposedDecl());
5871 } else {
5872 D = cast<VarDecl>(Val: DRE->getDecl());
5873 }
5874 }
5875 D = D->getCanonicalDecl();
5876 std::string Name = CGM.getOpenMPRuntime().getName(
5877 Parts: {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(GD: D)});
5878 Out << Prefix << Name << "_"
5879 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5880 return std::string(Out.str());
5881}
5882
5883/// Emits reduction initializer function:
5884/// \code
5885/// void @.red_init(void* %arg, void* %orig) {
5886/// %0 = bitcast void* %arg to <type>*
5887/// store <type> <init>, <type>* %0
5888/// ret void
5889/// }
5890/// \endcode
5891static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5892 SourceLocation Loc,
5893 ReductionCodeGen &RCG, unsigned N) {
5894 ASTContext &C = CGM.getContext();
5895 QualType VoidPtrTy = C.VoidPtrTy;
5896 VoidPtrTy.addRestrict();
5897 FunctionArgList Args;
5898 auto *Param =
5899 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5900 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5901 auto *ParamOrig =
5902 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5903 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5904 Args.emplace_back(Args&: Param);
5905 Args.emplace_back(Args&: ParamOrig);
5906 const auto &FnInfo =
5907 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5908 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5909 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_init", ""});
5910 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5911 N: Name, M: &CGM.getModule());
5912 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5913 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5914 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5915 Fn->setDoesNotRecurse();
5916 CodeGenFunction CGF(CGM);
5917 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5918 QualType PrivateType = RCG.getPrivateType(N);
5919 Address PrivateAddr = CGF.EmitLoadOfPointer(
5920 Ptr: CGF.GetAddrOfLocalVar(VD: Param).withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5921 PtrTy: C.getPointerType(T: PrivateType)->castAs<PointerType>());
5922 llvm::Value *Size = nullptr;
5923 // If the size of the reduction item is non-constant, load it from global
5924 // threadprivate variable.
5925 if (RCG.getSizes(N).second) {
5926 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5927 CGF, VarType: CGM.getContext().getSizeType(),
5928 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5929 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5930 Ty: CGM.getContext().getSizeType(), Loc);
5931 }
5932 RCG.emitAggregateType(CGF, N, Size);
5933 Address OrigAddr = Address::invalid();
5934 // If initializer uses initializer from declare reduction construct, emit a
5935 // pointer to the address of the original reduction item (reuired by reduction
5936 // initializer)
5937 if (RCG.usesReductionInitializer(N)) {
5938 Address SharedAddr = CGF.GetAddrOfLocalVar(VD: ParamOrig);
5939 OrigAddr = CGF.EmitLoadOfPointer(
5940 Ptr: SharedAddr,
5941 PtrTy: CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5942 }
5943 // Emit the initializer:
5944 // %0 = bitcast void* %arg to <type>*
5945 // store <type> <init>, <type>* %0
5946 RCG.emitInitialization(CGF, N, PrivateAddr, SharedAddr: OrigAddr,
5947 DefaultInit: [](CodeGenFunction &) { return false; });
5948 CGF.FinishFunction();
5949 return Fn;
5950}
5951
5952/// Emits reduction combiner function:
5953/// \code
5954/// void @.red_comb(void* %arg0, void* %arg1) {
5955/// %lhs = bitcast void* %arg0 to <type>*
5956/// %rhs = bitcast void* %arg1 to <type>*
5957/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5958/// store <type> %2, <type>* %lhs
5959/// ret void
5960/// }
5961/// \endcode
5962static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5963 SourceLocation Loc,
5964 ReductionCodeGen &RCG, unsigned N,
5965 const Expr *ReductionOp,
5966 const Expr *LHS, const Expr *RHS,
5967 const Expr *PrivateRef) {
5968 ASTContext &C = CGM.getContext();
5969 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHS)->getDecl());
5970 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHS)->getDecl());
5971 FunctionArgList Args;
5972 auto *ParamInOut =
5973 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5974 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5975 auto *ParamIn =
5976 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5977 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5978 Args.emplace_back(Args&: ParamInOut);
5979 Args.emplace_back(Args&: ParamIn);
5980 const auto &FnInfo =
5981 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5982 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5983 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_comb", ""});
5984 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5985 N: Name, M: &CGM.getModule());
5986 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5987 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5988 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5989 Fn->setDoesNotRecurse();
5990 CodeGenFunction CGF(CGM);
5991 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5992 llvm::Value *Size = nullptr;
5993 // If the size of the reduction item is non-constant, load it from global
5994 // threadprivate variable.
5995 if (RCG.getSizes(N).second) {
5996 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5997 CGF, VarType: CGM.getContext().getSizeType(),
5998 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5999 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
6000 Ty: CGM.getContext().getSizeType(), Loc);
6001 }
6002 RCG.emitAggregateType(CGF, N, Size);
6003 // Remap lhs and rhs variables to the addresses of the function arguments.
6004 // %lhs = bitcast void* %arg0 to <type>*
6005 // %rhs = bitcast void* %arg1 to <type>*
6006 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6007 PrivateScope.addPrivate(
6008 LocalVD: LHSVD,
6009 // Pull out the pointer to the variable.
6010 Addr: CGF.EmitLoadOfPointer(
6011 Ptr: CGF.GetAddrOfLocalVar(VD: ParamInOut)
6012 .withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
6013 PtrTy: C.getPointerType(T: LHSVD->getType())->castAs<PointerType>()));
6014 PrivateScope.addPrivate(
6015 LocalVD: RHSVD,
6016 // Pull out the pointer to the variable.
6017 Addr: CGF.EmitLoadOfPointer(
6018 Ptr: CGF.GetAddrOfLocalVar(VD: ParamIn).withElementType(
6019 ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
6020 PtrTy: C.getPointerType(T: RHSVD->getType())->castAs<PointerType>()));
6021 PrivateScope.Privatize();
6022 // Emit the combiner body:
6023 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
6024 // store <type> %2, <type>* %lhs
6025 CGM.getOpenMPRuntime().emitSingleReductionCombiner(
6026 CGF, ReductionOp, PrivateRef, LHS: cast<DeclRefExpr>(Val: LHS),
6027 RHS: cast<DeclRefExpr>(Val: RHS));
6028 CGF.FinishFunction();
6029 return Fn;
6030}
6031
6032/// Emits reduction finalizer function:
6033/// \code
6034/// void @.red_fini(void* %arg) {
6035/// %0 = bitcast void* %arg to <type>*
6036/// <destroy>(<type>* %0)
6037/// ret void
6038/// }
6039/// \endcode
6040static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
6041 SourceLocation Loc,
6042 ReductionCodeGen &RCG, unsigned N) {
6043 if (!RCG.needCleanups(N))
6044 return nullptr;
6045 ASTContext &C = CGM.getContext();
6046 FunctionArgList Args;
6047 auto *Param =
6048 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
6049 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
6050 Args.emplace_back(Args&: Param);
6051 const auto &FnInfo =
6052 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
6053 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
6054 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_fini", ""});
6055 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
6056 N: Name, M: &CGM.getModule());
6057 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
6058 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
6059 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
6060 Fn->setDoesNotRecurse();
6061 CodeGenFunction CGF(CGM);
6062 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
6063 Address PrivateAddr = CGF.EmitLoadOfPointer(
6064 Ptr: CGF.GetAddrOfLocalVar(VD: Param), PtrTy: C.VoidPtrTy.castAs<PointerType>());
6065 llvm::Value *Size = nullptr;
6066 // If the size of the reduction item is non-constant, load it from global
6067 // threadprivate variable.
6068 if (RCG.getSizes(N).second) {
6069 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6070 CGF, VarType: CGM.getContext().getSizeType(),
6071 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6072 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
6073 Ty: CGM.getContext().getSizeType(), Loc);
6074 }
6075 RCG.emitAggregateType(CGF, N, Size);
6076 // Emit the finalizer body:
6077 // <destroy>(<type>* %0)
6078 RCG.emitCleanups(CGF, N, PrivateAddr);
6079 CGF.FinishFunction(EndLoc: Loc);
6080 return Fn;
6081}
6082
6083llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
6084 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
6085 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
6086 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
6087 return nullptr;
6088
6089 // Build typedef struct:
6090 // kmp_taskred_input {
6091 // void *reduce_shar; // shared reduction item
6092 // void *reduce_orig; // original reduction item used for initialization
6093 // size_t reduce_size; // size of data item
6094 // void *reduce_init; // data initialization routine
6095 // void *reduce_fini; // data finalization routine
6096 // void *reduce_comb; // data combiner routine
6097 // kmp_task_red_flags_t flags; // flags for additional info from compiler
6098 // } kmp_taskred_input_t;
6099 ASTContext &C = CGM.getContext();
6100 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_taskred_input_t");
6101 RD->startDefinition();
6102 const FieldDecl *SharedFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6103 const FieldDecl *OrigFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6104 const FieldDecl *SizeFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.getSizeType());
6105 const FieldDecl *InitFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6106 const FieldDecl *FiniFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6107 const FieldDecl *CombFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6108 const FieldDecl *FlagsFD = addFieldToRecordDecl(
6109 C, DC: RD, FieldTy: C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
6110 RD->completeDefinition();
6111 CanQualType RDType = C.getCanonicalTagType(TD: RD);
6112 unsigned Size = Data.ReductionVars.size();
6113 llvm::APInt ArraySize(/*numBits=*/64, Size);
6114 QualType ArrayRDType =
6115 C.getConstantArrayType(EltTy: RDType, ArySize: ArraySize, SizeExpr: nullptr,
6116 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
6117 // kmp_task_red_input_t .rd_input.[Size];
6118 RawAddress TaskRedInput = CGF.CreateMemTemp(T: ArrayRDType, Name: ".rd_input.");
6119 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
6120 Data.ReductionCopies, Data.ReductionOps);
6121 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6122 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
6123 llvm::Value *Idxs[] = {llvm::ConstantInt::get(Ty: CGM.SizeTy, /*V=*/0),
6124 llvm::ConstantInt::get(Ty: CGM.SizeTy, V: Cnt)};
6125 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
6126 ElemTy: TaskRedInput.getElementType(), Ptr: TaskRedInput.getPointer(), IdxList: Idxs,
6127 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
6128 Name: ".rd_input.gep.");
6129 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(V: GEP, T: RDType);
6130 // ElemLVal.reduce_shar = &Shareds[Cnt];
6131 LValue SharedLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SharedFD);
6132 RCG.emitSharedOrigLValue(CGF, N: Cnt);
6133 llvm::Value *Shared = RCG.getSharedLValue(N: Cnt).getPointer(CGF);
6134 CGF.EmitStoreOfScalar(value: Shared, lvalue: SharedLVal);
6135 // ElemLVal.reduce_orig = &Origs[Cnt];
6136 LValue OrigLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: OrigFD);
6137 llvm::Value *Orig = RCG.getOrigLValue(N: Cnt).getPointer(CGF);
6138 CGF.EmitStoreOfScalar(value: Orig, lvalue: OrigLVal);
6139 RCG.emitAggregateType(CGF, N: Cnt);
6140 llvm::Value *SizeValInChars;
6141 llvm::Value *SizeVal;
6142 std::tie(args&: SizeValInChars, args&: SizeVal) = RCG.getSizes(N: Cnt);
6143 // We use delayed creation/initialization for VLAs and array sections. It is
6144 // required because runtime does not provide the way to pass the sizes of
6145 // VLAs/array sections to initializer/combiner/finalizer functions. Instead
6146 // threadprivate global variables are used to store these values and use
6147 // them in the functions.
6148 bool DelayedCreation = !!SizeVal;
6149 SizeValInChars = CGF.Builder.CreateIntCast(V: SizeValInChars, DestTy: CGM.SizeTy,
6150 /*isSigned=*/false);
6151 LValue SizeLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SizeFD);
6152 CGF.EmitStoreOfScalar(value: SizeValInChars, lvalue: SizeLVal);
6153 // ElemLVal.reduce_init = init;
6154 LValue InitLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: InitFD);
6155 llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, N: Cnt);
6156 CGF.EmitStoreOfScalar(value: InitAddr, lvalue: InitLVal);
6157 // ElemLVal.reduce_fini = fini;
6158 LValue FiniLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FiniFD);
6159 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, N: Cnt);
6160 llvm::Value *FiniAddr =
6161 Fini ? Fini : llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
6162 CGF.EmitStoreOfScalar(value: FiniAddr, lvalue: FiniLVal);
6163 // ElemLVal.reduce_comb = comb;
6164 LValue CombLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: CombFD);
6165 llvm::Value *CombAddr = emitReduceCombFunction(
6166 CGM, Loc, RCG, N: Cnt, ReductionOp: Data.ReductionOps[Cnt], LHS: LHSExprs[Cnt],
6167 RHS: RHSExprs[Cnt], PrivateRef: Data.ReductionCopies[Cnt]);
6168 CGF.EmitStoreOfScalar(value: CombAddr, lvalue: CombLVal);
6169 // ElemLVal.flags = 0;
6170 LValue FlagsLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FlagsFD);
6171 if (DelayedCreation) {
6172 CGF.EmitStoreOfScalar(
6173 value: llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/1, /*isSigned=*/IsSigned: true),
6174 lvalue: FlagsLVal);
6175 } else
6176 CGF.EmitNullInitialization(DestPtr: FlagsLVal.getAddress(), Ty: FlagsLVal.getType());
6177 }
6178 if (Data.IsReductionWithTaskMod) {
6179 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6180 // is_ws, int num, void *data);
6181 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6182 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6183 DestTy: CGM.IntTy, /*isSigned=*/true);
6184 llvm::Value *Args[] = {
6185 IdentTLoc, GTid,
6186 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Data.IsWorksharingReduction ? 1 : 0,
6187 /*isSigned=*/IsSigned: true),
6188 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6189 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6190 V: TaskRedInput.getPointer(), DestTy: CGM.VoidPtrTy)};
6191 return CGF.EmitRuntimeCall(
6192 callee: OMPBuilder.getOrCreateRuntimeFunction(
6193 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_modifier_init),
6194 args: Args);
6195 }
6196 // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data);
6197 llvm::Value *Args[] = {
6198 CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc), DestTy: CGM.IntTy,
6199 /*isSigned=*/true),
6200 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6201 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: TaskRedInput.getPointer(),
6202 DestTy: CGM.VoidPtrTy)};
6203 return CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6204 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_init),
6205 args: Args);
6206}
6207
6208void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
6209 SourceLocation Loc,
6210 bool IsWorksharingReduction) {
6211 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6212 // is_ws, int num, void *data);
6213 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6214 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6215 DestTy: CGM.IntTy, /*isSigned=*/true);
6216 llvm::Value *Args[] = {IdentTLoc, GTid,
6217 llvm::ConstantInt::get(Ty: CGM.IntTy,
6218 V: IsWorksharingReduction ? 1 : 0,
6219 /*isSigned=*/IsSigned: true)};
6220 (void)CGF.EmitRuntimeCall(
6221 callee: OMPBuilder.getOrCreateRuntimeFunction(
6222 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_modifier_fini),
6223 args: Args);
6224}
6225
6226void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6227 SourceLocation Loc,
6228 ReductionCodeGen &RCG,
6229 unsigned N) {
6230 auto Sizes = RCG.getSizes(N);
6231 // Emit threadprivate global variable if the type is non-constant
6232 // (Sizes.second = nullptr).
6233 if (Sizes.second) {
6234 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(V: Sizes.second, DestTy: CGM.SizeTy,
6235 /*isSigned=*/false);
6236 Address SizeAddr = getAddrOfArtificialThreadPrivate(
6237 CGF, VarType: CGM.getContext().getSizeType(),
6238 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6239 CGF.Builder.CreateStore(Val: SizeVal, Addr: SizeAddr, /*IsVolatile=*/false);
6240 }
6241}
6242
6243Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6244 SourceLocation Loc,
6245 llvm::Value *ReductionsPtr,
6246 LValue SharedLVal) {
6247 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6248 // *d);
6249 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6250 DestTy: CGM.IntTy,
6251 /*isSigned=*/true),
6252 ReductionsPtr,
6253 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6254 V: SharedLVal.getPointer(CGF), DestTy: CGM.VoidPtrTy)};
6255 return Address(
6256 CGF.EmitRuntimeCall(
6257 callee: OMPBuilder.getOrCreateRuntimeFunction(
6258 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_get_th_data),
6259 args: Args),
6260 CGF.Int8Ty, SharedLVal.getAlignment());
6261}
6262
6263void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
6264 const OMPTaskDataTy &Data) {
6265 if (!CGF.HaveInsertPoint())
6266 return;
6267
6268 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
6269 // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder.
6270 OMPBuilder.createTaskwait(Loc: CGF.Builder);
6271 } else {
6272 llvm::Value *ThreadID = getThreadID(CGF, Loc);
6273 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
6274 auto &M = CGM.getModule();
6275 Address DependenciesArray = Address::invalid();
6276 llvm::Value *NumOfElements;
6277 std::tie(args&: NumOfElements, args&: DependenciesArray) =
6278 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
6279 if (!Data.Dependences.empty()) {
6280 llvm::Value *DepWaitTaskArgs[7];
6281 DepWaitTaskArgs[0] = UpLoc;
6282 DepWaitTaskArgs[1] = ThreadID;
6283 DepWaitTaskArgs[2] = NumOfElements;
6284 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
6285 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
6286 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6287 DepWaitTaskArgs[6] =
6288 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
6289
6290 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
6291
6292 // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
6293 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
6294 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
6295 // kmp_int32 has_no_wait); if dependence info is specified.
6296 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6297 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
6298 args: DepWaitTaskArgs);
6299
6300 } else {
6301
6302 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6303 // global_tid);
6304 llvm::Value *Args[] = {UpLoc, ThreadID};
6305 // Ignore return result until untied tasks are supported.
6306 CGF.EmitRuntimeCall(
6307 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_omp_taskwait),
6308 args: Args);
6309 }
6310 }
6311
6312 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
6313 Region->emitUntiedSwitch(CGF);
6314}
6315
6316void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6317 OpenMPDirectiveKind InnerKind,
6318 const RegionCodeGenTy &CodeGen,
6319 bool HasCancel) {
6320 if (!CGF.HaveInsertPoint())
6321 return;
6322 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
6323 InnerKind != OMPD_critical &&
6324 InnerKind != OMPD_master &&
6325 InnerKind != OMPD_masked);
6326 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6327}
6328
6329namespace {
6330enum RTCancelKind {
6331 CancelNoreq = 0,
6332 CancelParallel = 1,
6333 CancelLoop = 2,
6334 CancelSections = 3,
6335 CancelTaskgroup = 4
6336};
6337} // anonymous namespace
6338
6339static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6340 RTCancelKind CancelKind = CancelNoreq;
6341 if (CancelRegion == OMPD_parallel)
6342 CancelKind = CancelParallel;
6343 else if (CancelRegion == OMPD_for)
6344 CancelKind = CancelLoop;
6345 else if (CancelRegion == OMPD_sections)
6346 CancelKind = CancelSections;
6347 else {
6348 assert(CancelRegion == OMPD_taskgroup);
6349 CancelKind = CancelTaskgroup;
6350 }
6351 return CancelKind;
6352}
6353
6354void CGOpenMPRuntime::emitCancellationPointCall(
6355 CodeGenFunction &CGF, SourceLocation Loc,
6356 OpenMPDirectiveKind CancelRegion) {
6357 if (!CGF.HaveInsertPoint())
6358 return;
6359 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6360 // global_tid, kmp_int32 cncl_kind);
6361 if (auto *OMPRegionInfo =
6362 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6363 // For 'cancellation point taskgroup', the task region info may not have a
6364 // cancel. This may instead happen in another adjacent task.
6365 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6366 llvm::Value *Args[] = {
6367 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6368 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6369 // Ignore return result until untied tasks are supported.
6370 llvm::Value *Result = CGF.EmitRuntimeCall(
6371 callee: OMPBuilder.getOrCreateRuntimeFunction(
6372 M&: CGM.getModule(), FnID: OMPRTL___kmpc_cancellationpoint),
6373 args: Args);
6374 // if (__kmpc_cancellationpoint()) {
6375 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6376 // exit from construct;
6377 // }
6378 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6379 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6380 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6381 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6382 CGF.EmitBlock(BB: ExitBB);
6383 if (CancelRegion == OMPD_parallel)
6384 emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6385 // exit from construct;
6386 CodeGenFunction::JumpDest CancelDest =
6387 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6388 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6389 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6390 }
6391 }
6392}
6393
6394void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6395 const Expr *IfCond,
6396 OpenMPDirectiveKind CancelRegion) {
6397 if (!CGF.HaveInsertPoint())
6398 return;
6399 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6400 // kmp_int32 cncl_kind);
6401 auto &M = CGM.getModule();
6402 if (auto *OMPRegionInfo =
6403 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6404 auto &&ThenGen = [this, &M, Loc, CancelRegion,
6405 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
6406 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6407 llvm::Value *Args[] = {
6408 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6409 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6410 // Ignore return result until untied tasks are supported.
6411 llvm::Value *Result = CGF.EmitRuntimeCall(
6412 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_cancel), args: Args);
6413 // if (__kmpc_cancel()) {
6414 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6415 // exit from construct;
6416 // }
6417 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6418 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6419 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6420 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6421 CGF.EmitBlock(BB: ExitBB);
6422 if (CancelRegion == OMPD_parallel)
6423 RT.emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6424 // exit from construct;
6425 CodeGenFunction::JumpDest CancelDest =
6426 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6427 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6428 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6429 };
6430 if (IfCond) {
6431 emitIfClause(CGF, Cond: IfCond, ThenGen,
6432 ElseGen: [](CodeGenFunction &, PrePostActionTy &) {});
6433 } else {
6434 RegionCodeGenTy ThenRCG(ThenGen);
6435 ThenRCG(CGF);
6436 }
6437 }
6438}
6439
6440namespace {
6441/// Cleanup action for uses_allocators support.
6442class OMPUsesAllocatorsActionTy final : public PrePostActionTy {
6443 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators;
6444 const OMPExecutableDirective &D;
6445 bool IsOffloadEntry;
6446
6447public:
6448 OMPUsesAllocatorsActionTy(
6449 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators,
6450 const OMPExecutableDirective &D, bool IsOffloadEntry)
6451 : Allocators(Allocators), D(D), IsOffloadEntry(IsOffloadEntry) {}
6452 void Enter(CodeGenFunction &CGF) override {
6453 if (!CGF.HaveInsertPoint())
6454 return;
6455 for (const auto &AllocatorData : Allocators) {
6456 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(
6457 CGF, Allocator: AllocatorData.first, AllocatorTraits: AllocatorData.second);
6458 }
6459 // This kernel does not go through the device-side runtime
6460 // init/deinit sequence (that is GPU-only), but the runtime still
6461 // needs a '<kernel>_kernel_environment' global to know how the
6462 // kernel was configured, so emit it directly here.
6463 if (IsOffloadEntry)
6464 CGF.CGM.getOpenMPRuntime().emitHostKernelEnvironment(D, CGF);
6465 }
6466 void Exit(CodeGenFunction &CGF) override {
6467 if (!CGF.HaveInsertPoint())
6468 return;
6469 for (const auto &AllocatorData : Allocators) {
6470 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,
6471 Allocator: AllocatorData.first);
6472 }
6473 }
6474};
6475} // namespace
6476
6477void CGOpenMPRuntime::emitHostKernelEnvironment(const OMPExecutableDirective &D,
6478 CodeGenFunction &CGF) {
6479 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs Attrs;
6480 Attrs.ExecFlags = llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC;
6481 computeMinAndMaxThreadsAndTeams(D, CGF, Attrs);
6482 OMPBuilder.emitKernelEnvironment(Loc: CGF.Builder, Attrs);
6483}
6484
6485void CGOpenMPRuntime::emitTargetOutlinedFunction(
6486 const OMPExecutableDirective &D, StringRef ParentName,
6487 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6488 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6489 assert(!ParentName.empty() && "Invalid target entry parent name!");
6490 HasEmittedTargetRegion = true;
6491 SmallVector<std::pair<const Expr *, const Expr *>, 4> Allocators;
6492 for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) {
6493 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
6494 const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
6495 if (!D.AllocatorTraits)
6496 continue;
6497 Allocators.emplace_back(Args: D.Allocator, Args: D.AllocatorTraits);
6498 }
6499 }
6500 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators, D, IsOffloadEntry);
6501 CodeGen.setAction(UsesAllocatorAction);
6502 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6503 IsOffloadEntry, CodeGen);
6504}
6505
6506void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
6507 const Expr *Allocator,
6508 const Expr *AllocatorTraits) {
6509 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6510 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6511 // Use default memspace handle.
6512 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6513 llvm::Value *NumTraits = llvm::ConstantInt::get(
6514 Ty: CGF.IntTy, V: cast<ConstantArrayType>(
6515 Val: AllocatorTraits->getType()->getAsArrayTypeUnsafe())
6516 ->getSize()
6517 .getLimitedValue());
6518 LValue AllocatorTraitsLVal = CGF.EmitLValue(E: AllocatorTraits);
6519 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6520 Addr: AllocatorTraitsLVal.getAddress(), Ty: CGF.VoidPtrPtrTy, ElementTy: CGF.VoidPtrTy);
6521 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, T: CGF.getContext().VoidPtrTy,
6522 BaseInfo: AllocatorTraitsLVal.getBaseInfo(),
6523 TBAAInfo: AllocatorTraitsLVal.getTBAAInfo());
6524 llvm::Value *Traits = Addr.emitRawPointer(CGF);
6525
6526 llvm::Value *AllocatorVal =
6527 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6528 M&: CGM.getModule(), FnID: OMPRTL___kmpc_init_allocator),
6529 args: {ThreadId, MemSpaceHandle, NumTraits, Traits});
6530 // Store to allocator.
6531 CGF.EmitAutoVarAlloca(var: *cast<VarDecl>(
6532 Val: cast<DeclRefExpr>(Val: Allocator->IgnoreParenImpCasts())->getDecl()));
6533 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6534 AllocatorVal =
6535 CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: CGF.getContext().VoidPtrTy,
6536 DstTy: Allocator->getType(), Loc: Allocator->getExprLoc());
6537 CGF.EmitStoreOfScalar(value: AllocatorVal, lvalue: AllocatorLVal);
6538}
6539
6540void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
6541 const Expr *Allocator) {
6542 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6543 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6544 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6545 llvm::Value *AllocatorVal =
6546 CGF.EmitLoadOfScalar(lvalue: AllocatorLVal, Loc: Allocator->getExprLoc());
6547 AllocatorVal = CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: Allocator->getType(),
6548 DstTy: CGF.getContext().VoidPtrTy,
6549 Loc: Allocator->getExprLoc());
6550 (void)CGF.EmitRuntimeCall(
6551 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
6552 FnID: OMPRTL___kmpc_destroy_allocator),
6553 args: {ThreadId, AllocatorVal});
6554}
6555
6556void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams(
6557 const OMPExecutableDirective &D, CodeGenFunction &CGF,
6558 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6559 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6560 "invalid default attrs structure");
6561 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6562 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6563
6564 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: Attrs.MinTeams.front(),
6565 MaxTeamsVal);
6566 getNumThreadsExprForTargetDirective(CGF, D, UpperBound&: MaxThreadsVal,
6567 /*UpperBoundOnly=*/true);
6568
6569 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6570 for (auto *A : C->getAttrs()) {
6571 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6572 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6573 if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(Val: A))
6574 CGM.handleCUDALaunchBoundsAttr(F: nullptr, A: Attr, MaxThreadsVal: &AttrMaxThreadsVal,
6575 MinBlocksVal: &AttrMinBlocksVal, MaxClusterRankVal: &AttrMaxBlocksVal);
6576 else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(Val: A))
6577 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6578 F: nullptr, A: Attr, /*ReqdWGS=*/nullptr, MinThreadsVal: &AttrMinThreadsVal,
6579 MaxThreadsVal: &AttrMaxThreadsVal);
6580 else
6581 continue;
6582
6583 Attrs.MinThreads.front() =
6584 std::max(a: Attrs.MinThreads.front(), b: AttrMinThreadsVal);
6585 if (AttrMaxThreadsVal > 0)
6586 MaxThreadsVal = MaxThreadsVal > 0
6587 ? std::min(a: MaxThreadsVal, b: AttrMaxThreadsVal)
6588 : AttrMaxThreadsVal;
6589 Attrs.MinTeams.front() =
6590 std::max(a: Attrs.MinTeams.front(), b: AttrMinBlocksVal);
6591 if (AttrMaxBlocksVal > 0)
6592 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(a: MaxTeamsVal, b: AttrMaxBlocksVal)
6593 : AttrMaxBlocksVal;
6594 }
6595 }
6596}
6597
6598void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6599 const OMPExecutableDirective &D, StringRef ParentName,
6600 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6601 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6602
6603 llvm::TargetRegionEntryInfo EntryInfo =
6604 getEntryInfoFromPresumedLoc(CGM, OMPBuilder, BeginLoc: D.getBeginLoc(), ParentName);
6605
6606 CodeGenFunction CGF(CGM, true);
6607 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6608 [&CGF, &D, &CodeGen, this](StringRef EntryFnName) {
6609 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
6610
6611 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6612 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6613 if (CGM.getLangOpts().OpenMPIsTargetDevice && !isGPU())
6614 return CGF.GenerateOpenMPCapturedStmtFunctionAggregate(S: CS, D);
6615 return CGF.GenerateOpenMPCapturedStmtFunction(S: CS, D);
6616 };
6617
6618 cantFail(Err: OMPBuilder.emitTargetRegionFunction(
6619 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6620 OutlinedFnID));
6621
6622 if (!OutlinedFn)
6623 return;
6624
6625 // A target body is entered once, from the kernel, and never re-entered by
6626 // the runtime, so it cannot occur in a cycle.
6627 OutlinedFn->setDoesNotRecurse();
6628
6629 CGM.getTargetCodeGenInfo().setTargetAttributes(D: nullptr, GV: OutlinedFn, M&: CGM);
6630
6631 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6632 for (auto *A : C->getAttrs()) {
6633 if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(Val: A))
6634 CGM.handleAMDGPUWavesPerEUAttr(F: OutlinedFn, A: Attr);
6635 }
6636 }
6637 registerVTable(D);
6638}
6639
6640/// Checks if the expression is constant or does not have non-trivial function
6641/// calls.
6642static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6643 // We can skip constant expressions.
6644 // We can skip expressions with trivial calls or simple expressions.
6645 return (E->isEvaluatable(Ctx, AllowSideEffects: Expr::SE_AllowUndefinedBehavior) ||
6646 !E->hasNonTrivialCall(Ctx)) &&
6647 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6648}
6649
6650const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,
6651 const Stmt *Body) {
6652 const Stmt *Child = Body->IgnoreContainers();
6653 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Val: Child)) {
6654 Child = nullptr;
6655 for (const Stmt *S : C->body()) {
6656 if (const auto *E = dyn_cast<Expr>(Val: S)) {
6657 if (isTrivial(Ctx, E))
6658 continue;
6659 }
6660 // Some of the statements can be ignored.
6661 if (isa<AsmStmt>(Val: S) || isa<NullStmt>(Val: S) || isa<OMPFlushDirective>(Val: S) ||
6662 isa<OMPBarrierDirective>(Val: S) || isa<OMPTaskyieldDirective>(Val: S))
6663 continue;
6664 // Analyze declarations.
6665 if (const auto *DS = dyn_cast<DeclStmt>(Val: S)) {
6666 if (llvm::all_of(Range: DS->decls(), P: [](const Decl *D) {
6667 if (isa<EmptyDecl>(Val: D) || isa<DeclContext>(Val: D) ||
6668 isa<TypeDecl>(Val: D) || isa<PragmaCommentDecl>(Val: D) ||
6669 isa<PragmaDetectMismatchDecl>(Val: D) || isa<UsingDecl>(Val: D) ||
6670 isa<UsingDirectiveDecl>(Val: D) ||
6671 isa<OMPDeclareReductionDecl>(Val: D) ||
6672 isa<OMPThreadPrivateDecl>(Val: D) || isa<OMPAllocateDecl>(Val: D))
6673 return true;
6674 const auto *VD = dyn_cast<VarDecl>(Val: D);
6675 if (!VD)
6676 return false;
6677 return VD->hasGlobalStorage() || !VD->isUsed();
6678 }))
6679 continue;
6680 }
6681 // Found multiple children - cannot get the one child only.
6682 if (Child)
6683 return nullptr;
6684 Child = S;
6685 }
6686 if (Child)
6687 Child = Child->IgnoreContainers();
6688 }
6689 return Child;
6690}
6691
6692const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(
6693 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6694 int32_t &MaxTeamsVal) {
6695
6696 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6697 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6698 "Expected target-based executable directive.");
6699 switch (DirectiveKind) {
6700 case OMPD_target: {
6701 const auto *CS = D.getInnermostCapturedStmt();
6702 const auto *Body =
6703 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6704 const Stmt *ChildStmt =
6705 CGOpenMPRuntime::getSingleCompoundChild(Ctx&: CGF.getContext(), Body);
6706 if (const auto *NestedDir =
6707 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
6708 if (isOpenMPTeamsDirective(DKind: NestedDir->getDirectiveKind())) {
6709 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6710 const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>()
6711 ->getNumTeams()
6712 .front();
6713 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6714 if (auto Constant =
6715 NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6716 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6717 return NumTeams;
6718 }
6719 MinTeamsVal = MaxTeamsVal = 0;
6720 return nullptr;
6721 }
6722 MinTeamsVal = MaxTeamsVal = 1;
6723 return nullptr;
6724 }
6725 // A value of -1 is used to check if we need to emit no teams region
6726 MinTeamsVal = MaxTeamsVal = -1;
6727 return nullptr;
6728 }
6729 case OMPD_target_teams_loop:
6730 case OMPD_target_teams:
6731 case OMPD_target_teams_distribute:
6732 case OMPD_target_teams_distribute_simd:
6733 case OMPD_target_teams_distribute_parallel_for:
6734 case OMPD_target_teams_distribute_parallel_for_simd: {
6735 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6736 const Expr *NumTeams =
6737 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front();
6738 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6739 if (auto Constant = NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6740 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6741 return NumTeams;
6742 }
6743 MinTeamsVal = MaxTeamsVal = 0;
6744 return nullptr;
6745 }
6746 case OMPD_target_parallel:
6747 case OMPD_target_parallel_for:
6748 case OMPD_target_parallel_for_simd:
6749 case OMPD_target_parallel_loop:
6750 case OMPD_target_simd:
6751 MinTeamsVal = MaxTeamsVal = 1;
6752 return nullptr;
6753 case OMPD_parallel:
6754 case OMPD_for:
6755 case OMPD_parallel_for:
6756 case OMPD_parallel_loop:
6757 case OMPD_parallel_master:
6758 case OMPD_parallel_sections:
6759 case OMPD_for_simd:
6760 case OMPD_parallel_for_simd:
6761 case OMPD_cancel:
6762 case OMPD_cancellation_point:
6763 case OMPD_ordered_standalone:
6764 case OMPD_ordered_blockassoc:
6765 case OMPD_threadprivate:
6766 case OMPD_allocate:
6767 case OMPD_task:
6768 case OMPD_simd:
6769 case OMPD_tile:
6770 case OMPD_unroll:
6771 case OMPD_sections:
6772 case OMPD_section:
6773 case OMPD_single:
6774 case OMPD_master:
6775 case OMPD_critical:
6776 case OMPD_taskyield:
6777 case OMPD_barrier:
6778 case OMPD_taskwait:
6779 case OMPD_taskgroup:
6780 case OMPD_atomic:
6781 case OMPD_flush:
6782 case OMPD_depobj:
6783 case OMPD_scan:
6784 case OMPD_teams:
6785 case OMPD_target_data:
6786 case OMPD_target_exit_data:
6787 case OMPD_target_enter_data:
6788 case OMPD_distribute:
6789 case OMPD_distribute_simd:
6790 case OMPD_distribute_parallel_for:
6791 case OMPD_distribute_parallel_for_simd:
6792 case OMPD_teams_distribute:
6793 case OMPD_teams_distribute_simd:
6794 case OMPD_teams_distribute_parallel_for:
6795 case OMPD_teams_distribute_parallel_for_simd:
6796 case OMPD_target_update:
6797 case OMPD_declare_simd:
6798 case OMPD_declare_variant:
6799 case OMPD_begin_declare_variant:
6800 case OMPD_end_declare_variant:
6801 case OMPD_declare_target:
6802 case OMPD_end_declare_target:
6803 case OMPD_declare_reduction:
6804 case OMPD_declare_mapper:
6805 case OMPD_taskloop:
6806 case OMPD_taskloop_simd:
6807 case OMPD_master_taskloop:
6808 case OMPD_master_taskloop_simd:
6809 case OMPD_parallel_master_taskloop:
6810 case OMPD_parallel_master_taskloop_simd:
6811 case OMPD_requires:
6812 case OMPD_metadirective:
6813 case OMPD_unknown:
6814 break;
6815 default:
6816 break;
6817 }
6818 llvm_unreachable("Unexpected directive kind.");
6819}
6820
6821llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective(
6822 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6823 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6824 "Clauses associated with the teams directive expected to be emitted "
6825 "only for the host!");
6826 CGBuilderTy &Bld = CGF.Builder;
6827 int32_t MinNT = -1, MaxNT = -1;
6828 const Expr *NumTeams =
6829 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: MinNT, MaxTeamsVal&: MaxNT);
6830 if (NumTeams != nullptr) {
6831 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6832
6833 switch (DirectiveKind) {
6834 case OMPD_target: {
6835 const auto *CS = D.getInnermostCapturedStmt();
6836 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6837 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6838 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6839 /*IgnoreResultAssign*/ true);
6840 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6841 /*isSigned=*/true);
6842 }
6843 case OMPD_target_teams:
6844 case OMPD_target_teams_distribute:
6845 case OMPD_target_teams_distribute_simd:
6846 case OMPD_target_teams_distribute_parallel_for:
6847 case OMPD_target_teams_distribute_parallel_for_simd: {
6848 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6849 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6850 /*IgnoreResultAssign*/ true);
6851 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6852 /*isSigned=*/true);
6853 }
6854 default:
6855 break;
6856 }
6857 }
6858
6859 assert(MinNT == MaxNT && "Num threads ranges require handling here.");
6860 return llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: MinNT);
6861}
6862
6863/// Merge the thread count upper bound \p Val into \p UpperBound.
6864///
6865/// \p UpperBound is -1 while no thread limiting clause has been seen, 0 once
6866/// one has been seen whose value is not known at compile time, and otherwise
6867/// the smallest constant bound found so far.
6868///
6869/// Thread limiting clauses compose by taking the minimum, so a constant bound
6870/// stays valid whatever the clauses that are not compile time constants
6871/// evaluate to. That makes it correct to replace the 0 marker with \p Val, and
6872/// necessary to keep a clause from raising a smaller bound found earlier.
6873static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val) {
6874 UpperBound = UpperBound > 0 ? std::min(a: UpperBound, b: Val) : Val;
6875}
6876
6877/// Check for a num threads constant value (stored in \p DefaultVal), or
6878/// expression (stored in \p E). If the value is conditional (via an if-clause),
6879/// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are
6880/// nullptr, no expression evaluation is perfomed.
6881static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6882 const Expr **E, int32_t &UpperBound,
6883 bool UpperBoundOnly, llvm::Value **CondVal) {
6884 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6885 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6886 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6887 if (!Dir)
6888 return;
6889
6890 if (isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6891 // Handle if clause. If if clause present, the number of threads is
6892 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6893 if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) {
6894 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6895 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6896 const OMPIfClause *IfClause = nullptr;
6897 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6898 if (C->getNameModifier() == OMPD_unknown ||
6899 C->getNameModifier() == OMPD_parallel) {
6900 IfClause = C;
6901 break;
6902 }
6903 }
6904 if (IfClause) {
6905 const Expr *CondExpr = IfClause->getCondition();
6906 bool Result;
6907 if (CondExpr->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6908 if (!Result) {
6909 UpperBound = 1;
6910 return;
6911 }
6912 } else {
6913 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());
6914 if (const auto *PreInit =
6915 cast_or_null<DeclStmt>(Val: IfClause->getPreInitStmt())) {
6916 for (const auto *I : PreInit->decls()) {
6917 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6918 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6919 } else {
6920 CodeGenFunction::AutoVarEmission Emission =
6921 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6922 CGF.EmitAutoVarCleanups(emission: Emission);
6923 }
6924 }
6925 *CondVal = CGF.EvaluateExprAsBool(E: CondExpr);
6926 }
6927 }
6928 }
6929 }
6930 // Check the value of num_threads clause iff if clause was not specified
6931 // or is not evaluated to false.
6932 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6933 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6934 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6935 const auto *NumThreadsClause =
6936 Dir->getSingleClause<OMPNumThreadsClause>();
6937 const Expr *NTExpr = NumThreadsClause->getNumThreads().front();
6938 if (NTExpr->isIntegerConstantExpr(Ctx: CGF.getContext()))
6939 if (auto Constant = NTExpr->getIntegerConstantExpr(Ctx: CGF.getContext()))
6940 mergeThreadCountUpperBound(
6941 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
6942 // If we haven't found a upper bound, remember we saw a thread limiting
6943 // clause.
6944 if (UpperBound == -1)
6945 UpperBound = 0;
6946 if (!E)
6947 return;
6948 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6949 if (const auto *PreInit =
6950 cast_or_null<DeclStmt>(Val: NumThreadsClause->getPreInitStmt())) {
6951 for (const auto *I : PreInit->decls()) {
6952 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6953 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6954 } else {
6955 CodeGenFunction::AutoVarEmission Emission =
6956 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6957 CGF.EmitAutoVarCleanups(emission: Emission);
6958 }
6959 }
6960 }
6961 *E = NTExpr;
6962 }
6963 return;
6964 }
6965 if (isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6966 UpperBound = 1;
6967}
6968
6969const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective(
6970 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6971 bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) {
6972 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6973 "Clauses associated with the teams directive expected to be emitted "
6974 "only for the host!");
6975 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6976 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6977 "Expected target-based executable directive.");
6978
6979 const Expr *NT = nullptr;
6980 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6981
6982 auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) {
6983 if (E->isIntegerConstantExpr(Ctx: CGF.getContext())) {
6984 if (auto Constant = E->getIntegerConstantExpr(Ctx: CGF.getContext()))
6985 mergeThreadCountUpperBound(
6986 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
6987 }
6988 // If we haven't found a upper bound, remember we saw a thread limiting
6989 // clause.
6990 if (UpperBound == -1)
6991 UpperBound = 0;
6992 if (EPtr)
6993 *EPtr = E;
6994 };
6995
6996 auto ReturnSequential = [&]() {
6997 UpperBound = 1;
6998 return NT;
6999 };
7000
7001 switch (DirectiveKind) {
7002 case OMPD_target: {
7003 const CapturedStmt *CS = D.getInnermostCapturedStmt();
7004 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7005 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
7006 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
7007 // TODO: The standard is not clear how to resolve two thread limit clauses,
7008 // let's pick the teams one if it's present, otherwise the target one.
7009 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7010 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
7011 if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) {
7012 ThreadLimitClause = TLC;
7013 if (ThreadLimitExpr) {
7014 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
7015 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
7016 CodeGenFunction::LexicalScope Scope(
7017 CGF,
7018 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
7019 if (const auto *PreInit =
7020 cast_or_null<DeclStmt>(Val: ThreadLimitClause->getPreInitStmt())) {
7021 for (const auto *I : PreInit->decls()) {
7022 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
7023 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
7024 } else {
7025 CodeGenFunction::AutoVarEmission Emission =
7026 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
7027 CGF.EmitAutoVarCleanups(emission: Emission);
7028 }
7029 }
7030 }
7031 }
7032 }
7033 }
7034 if (ThreadLimitClause)
7035 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7036 ThreadLimitExpr);
7037 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
7038 if (isOpenMPTeamsDirective(DKind: Dir->getDirectiveKind()) &&
7039 !isOpenMPDistributeDirective(DKind: Dir->getDirectiveKind())) {
7040 CS = Dir->getInnermostCapturedStmt();
7041 // Now that the 'teams' level has been peeled off, the remainder is
7042 // shaped like a 'target teams' region, so pick up the num_threads of
7043 // the directive nested in it the same way the OMPD_target_teams case
7044 // below does. Without this the upper bound of a construct written as
7045 // 'target' / 'teams' / 'distribute parallel for' would stay at the
7046 // default, while every combined spelling of the same construct honors
7047 // the clause. Only the bound is taken here: passing null for the
7048 // expression and the condition keeps this from emitting anything, so
7049 // the value the host passes to the kernel launch is left as it was.
7050 getNumThreads(CGF, CS, /*E=*/nullptr, UpperBound, UpperBoundOnly,
7051 /*CondVal=*/nullptr);
7052 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
7053 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
7054 Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
7055 }
7056 if (Dir && isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
7057 CS = Dir->getInnermostCapturedStmt();
7058 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7059 } else if (Dir && isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
7060 return ReturnSequential();
7061 }
7062 return NT;
7063 }
7064 case OMPD_target_teams: {
7065 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7066 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7067 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7068 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7069 ThreadLimitExpr);
7070 }
7071 const CapturedStmt *CS = D.getInnermostCapturedStmt();
7072 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7073 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
7074 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
7075 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
7076 if (Dir->getDirectiveKind() == OMPD_distribute) {
7077 CS = Dir->getInnermostCapturedStmt();
7078 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7079 }
7080 }
7081 return NT;
7082 }
7083 case OMPD_target_teams_distribute:
7084 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7085 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7086 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7087 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7088 ThreadLimitExpr);
7089 }
7090 getNumThreads(CGF, CS: D.getInnermostCapturedStmt(), E: NTPtr, UpperBound,
7091 UpperBoundOnly, CondVal);
7092 return NT;
7093 case OMPD_target_teams_loop:
7094 case OMPD_target_parallel_loop:
7095 case OMPD_target_parallel:
7096 case OMPD_target_parallel_for:
7097 case OMPD_target_parallel_for_simd:
7098 case OMPD_target_teams_distribute_parallel_for:
7099 case OMPD_target_teams_distribute_parallel_for_simd: {
7100 if (CondVal && D.hasClausesOfKind<OMPIfClause>()) {
7101 const OMPIfClause *IfClause = nullptr;
7102 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
7103 if (C->getNameModifier() == OMPD_unknown ||
7104 C->getNameModifier() == OMPD_parallel) {
7105 IfClause = C;
7106 break;
7107 }
7108 }
7109 if (IfClause) {
7110 const Expr *Cond = IfClause->getCondition();
7111 bool Result;
7112 if (Cond->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
7113 if (!Result)
7114 return ReturnSequential();
7115 } else {
7116 CodeGenFunction::RunCleanupsScope Scope(CGF);
7117 *CondVal = CGF.EvaluateExprAsBool(E: Cond);
7118 }
7119 }
7120 }
7121 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7122 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7123 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7124 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7125 ThreadLimitExpr);
7126 }
7127 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
7128 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
7129 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
7130 CheckForConstExpr(NumThreadsClause->getNumThreads().front(), nullptr);
7131 return NumThreadsClause->getNumThreads().front();
7132 }
7133 return NT;
7134 }
7135 case OMPD_target_teams_distribute_simd:
7136 case OMPD_target_simd:
7137 return ReturnSequential();
7138 default:
7139 break;
7140 }
7141 llvm_unreachable("Unsupported directive kind.");
7142}
7143
7144llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective(
7145 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
7146 llvm::Value *NumThreadsVal = nullptr;
7147 llvm::Value *CondVal = nullptr;
7148 llvm::Value *ThreadLimitVal = nullptr;
7149 const Expr *ThreadLimitExpr = nullptr;
7150 int32_t UpperBound = -1;
7151
7152 const Expr *NT = getNumThreadsExprForTargetDirective(
7153 CGF, D, UpperBound, /* UpperBoundOnly */ false, CondVal: &CondVal,
7154 ThreadLimitExpr: &ThreadLimitExpr);
7155
7156 // Thread limit expressions are used below, emit them.
7157 if (ThreadLimitExpr) {
7158 ThreadLimitVal =
7159 CGF.EmitScalarExpr(E: ThreadLimitExpr, /*IgnoreResultAssign=*/true);
7160 ThreadLimitVal = CGF.Builder.CreateIntCast(V: ThreadLimitVal, DestTy: CGF.Int32Ty,
7161 /*isSigned=*/false);
7162 }
7163
7164 // Generate the num teams expression.
7165 if (UpperBound == 1) {
7166 NumThreadsVal = CGF.Builder.getInt32(C: UpperBound);
7167 } else if (NT) {
7168 NumThreadsVal = CGF.EmitScalarExpr(E: NT, /*IgnoreResultAssign=*/true);
7169 NumThreadsVal = CGF.Builder.CreateIntCast(V: NumThreadsVal, DestTy: CGF.Int32Ty,
7170 /*isSigned=*/false);
7171 } else if (ThreadLimitVal) {
7172 // If we do not have a num threads value but a thread limit, replace the
7173 // former with the latter. We know handled the thread limit expression.
7174 NumThreadsVal = ThreadLimitVal;
7175 ThreadLimitVal = nullptr;
7176 } else {
7177 // Default to "0" which means runtime choice.
7178 assert(!ThreadLimitVal && "Default not applicable with thread limit value");
7179 NumThreadsVal = CGF.Builder.getInt32(C: 0);
7180 }
7181
7182 // Handle if clause. If if clause present, the number of threads is
7183 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
7184 if (CondVal) {
7185 CodeGenFunction::RunCleanupsScope Scope(CGF);
7186 NumThreadsVal = CGF.Builder.CreateSelect(C: CondVal, True: NumThreadsVal,
7187 False: CGF.Builder.getInt32(C: 1));
7188 }
7189
7190 // If the thread limit and num teams expression were present, take the
7191 // minimum.
7192 if (ThreadLimitVal) {
7193 NumThreadsVal = CGF.Builder.CreateSelect(
7194 C: CGF.Builder.CreateICmpULT(LHS: ThreadLimitVal, RHS: NumThreadsVal),
7195 True: ThreadLimitVal, False: NumThreadsVal);
7196 }
7197
7198 return NumThreadsVal;
7199}
7200
7201namespace {
7202LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
7203
7204// Utility to handle information from clauses associated with a given
7205// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
7206// It provides a convenient interface to obtain the information and generate
7207// code for that information.
7208class MappableExprsHandler {
7209public:
7210 /// Custom comparator for attach-pointer expressions that compares them by
7211 /// complexity (i.e. their component-depth) first, then by the order in which
7212 /// they were computed by collectAttachPtrExprInfo(), if they are semantically
7213 /// different.
7214 struct AttachPtrExprComparator {
7215 const MappableExprsHandler &Handler;
7216 // Cache of previous equality comparison results.
7217 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool>
7218 CachedEqualityComparisons;
7219
7220 AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {}
7221 AttachPtrExprComparator() = delete;
7222
7223 // Return true iff LHS is "less than" RHS.
7224 bool operator()(const Expr *LHS, const Expr *RHS) const {
7225 if (LHS == RHS)
7226 return false;
7227
7228 // First, compare by complexity (depth)
7229 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(Val: LHS);
7230 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(Val: RHS);
7231
7232 std::optional<size_t> DepthLHS =
7233 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
7234 : std::nullopt;
7235 std::optional<size_t> DepthRHS =
7236 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
7237 : std::nullopt;
7238
7239 // std::nullopt (no attach pointer) has lowest complexity
7240 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
7241 // Both have same complexity, now check semantic equality
7242 if (areEqual(LHS, RHS))
7243 return false;
7244 // Different semantically, compare by computation order
7245 return wasComputedBefore(LHS, RHS);
7246 }
7247 if (!DepthLHS.has_value())
7248 return true; // LHS has lower complexity
7249 if (!DepthRHS.has_value())
7250 return false; // RHS has lower complexity
7251
7252 // Both have values, compare by depth (lower depth = lower complexity)
7253 if (DepthLHS.value() != DepthRHS.value())
7254 return DepthLHS.value() < DepthRHS.value();
7255
7256 // Same complexity, now check semantic equality
7257 if (areEqual(LHS, RHS))
7258 return false;
7259 // Different semantically, compare by computation order
7260 return wasComputedBefore(LHS, RHS);
7261 }
7262
7263 public:
7264 /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached
7265 /// results, if available, otherwise does a recursive semantic comparison.
7266 bool areEqual(const Expr *LHS, const Expr *RHS) const {
7267 // Check cache first for faster lookup
7268 const auto CachedResultIt = CachedEqualityComparisons.find(Val: {LHS, RHS});
7269 if (CachedResultIt != CachedEqualityComparisons.end())
7270 return CachedResultIt->second;
7271
7272 bool ComparisonResult = areSemanticallyEqual(LHS, RHS);
7273
7274 // Cache the result for future lookups (both orders since semantic
7275 // equality is commutative)
7276 CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult;
7277 CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult;
7278 return ComparisonResult;
7279 }
7280
7281 /// Compare the two attach-ptr expressions by their computation order.
7282 /// Returns true iff LHS was computed before RHS by
7283 /// collectAttachPtrExprInfo().
7284 bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const {
7285 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(Val: LHS);
7286 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(Val: RHS);
7287
7288 return OrderLHS < OrderRHS;
7289 }
7290
7291 private:
7292 /// Helper function to compare attach-pointer expressions semantically.
7293 /// This function handles various expression types that can be part of an
7294 /// attach-pointer.
7295 /// TODO: Not urgent, but we should ideally return true when comparing
7296 /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc.
7297 bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const {
7298 if (LHS == RHS)
7299 return true;
7300
7301 // If only one is null, they aren't equal
7302 if (!LHS || !RHS)
7303 return false;
7304
7305 ASTContext &Ctx = Handler.CGF.getContext();
7306 // Strip away parentheses and no-op casts to get to the core expression
7307 LHS = LHS->IgnoreParenNoopCasts(Ctx);
7308 RHS = RHS->IgnoreParenNoopCasts(Ctx);
7309
7310 // Direct pointer comparison of the underlying expressions
7311 if (LHS == RHS)
7312 return true;
7313
7314 // Check if the expression classes match
7315 if (LHS->getStmtClass() != RHS->getStmtClass())
7316 return false;
7317
7318 // Handle DeclRefExpr (variable references)
7319 if (const auto *LD = dyn_cast<DeclRefExpr>(Val: LHS)) {
7320 const auto *RD = dyn_cast<DeclRefExpr>(Val: RHS);
7321 if (!RD)
7322 return false;
7323 return LD->getDecl()->getCanonicalDecl() ==
7324 RD->getDecl()->getCanonicalDecl();
7325 }
7326
7327 // Handle ArraySubscriptExpr (array indexing like a[i])
7328 if (const auto *LA = dyn_cast<ArraySubscriptExpr>(Val: LHS)) {
7329 const auto *RA = dyn_cast<ArraySubscriptExpr>(Val: RHS);
7330 if (!RA)
7331 return false;
7332 return areSemanticallyEqual(LHS: LA->getBase(), RHS: RA->getBase()) &&
7333 areSemanticallyEqual(LHS: LA->getIdx(), RHS: RA->getIdx());
7334 }
7335
7336 // Handle MemberExpr (member access like s.m or p->m)
7337 if (const auto *LM = dyn_cast<MemberExpr>(Val: LHS)) {
7338 const auto *RM = dyn_cast<MemberExpr>(Val: RHS);
7339 if (!RM)
7340 return false;
7341 if (LM->getMemberDecl()->getCanonicalDecl() !=
7342 RM->getMemberDecl()->getCanonicalDecl())
7343 return false;
7344 return areSemanticallyEqual(LHS: LM->getBase(), RHS: RM->getBase());
7345 }
7346
7347 // Handle UnaryOperator (unary operations like *p, &x, etc.)
7348 if (const auto *LU = dyn_cast<UnaryOperator>(Val: LHS)) {
7349 const auto *RU = dyn_cast<UnaryOperator>(Val: RHS);
7350 if (!RU)
7351 return false;
7352 if (LU->getOpcode() != RU->getOpcode())
7353 return false;
7354 return areSemanticallyEqual(LHS: LU->getSubExpr(), RHS: RU->getSubExpr());
7355 }
7356
7357 // Handle BinaryOperator (binary operations like p + offset)
7358 if (const auto *LB = dyn_cast<BinaryOperator>(Val: LHS)) {
7359 const auto *RB = dyn_cast<BinaryOperator>(Val: RHS);
7360 if (!RB)
7361 return false;
7362 if (LB->getOpcode() != RB->getOpcode())
7363 return false;
7364 return areSemanticallyEqual(LHS: LB->getLHS(), RHS: RB->getLHS()) &&
7365 areSemanticallyEqual(LHS: LB->getRHS(), RHS: RB->getRHS());
7366 }
7367
7368 // Handle ArraySectionExpr (array sections like a[0:1])
7369 // Attach pointers should not contain array-sections, but currently we
7370 // don't emit an error.
7371 if (const auto *LAS = dyn_cast<ArraySectionExpr>(Val: LHS)) {
7372 const auto *RAS = dyn_cast<ArraySectionExpr>(Val: RHS);
7373 if (!RAS)
7374 return false;
7375 return areSemanticallyEqual(LHS: LAS->getBase(), RHS: RAS->getBase()) &&
7376 areSemanticallyEqual(LHS: LAS->getLowerBound(),
7377 RHS: RAS->getLowerBound()) &&
7378 areSemanticallyEqual(LHS: LAS->getLength(), RHS: RAS->getLength());
7379 }
7380
7381 // Handle CastExpr (explicit casts)
7382 if (const auto *LC = dyn_cast<CastExpr>(Val: LHS)) {
7383 const auto *RC = dyn_cast<CastExpr>(Val: RHS);
7384 if (!RC)
7385 return false;
7386 if (LC->getCastKind() != RC->getCastKind())
7387 return false;
7388 return areSemanticallyEqual(LHS: LC->getSubExpr(), RHS: RC->getSubExpr());
7389 }
7390
7391 // Handle CXXThisExpr (this pointer)
7392 if (isa<CXXThisExpr>(Val: LHS) && isa<CXXThisExpr>(Val: RHS))
7393 return true;
7394
7395 // Handle IntegerLiteral (integer constants)
7396 if (const auto *LI = dyn_cast<IntegerLiteral>(Val: LHS)) {
7397 const auto *RI = dyn_cast<IntegerLiteral>(Val: RHS);
7398 if (!RI)
7399 return false;
7400 return LI->getValue() == RI->getValue();
7401 }
7402
7403 // Handle CharacterLiteral (character constants)
7404 if (const auto *LC = dyn_cast<CharacterLiteral>(Val: LHS)) {
7405 const auto *RC = dyn_cast<CharacterLiteral>(Val: RHS);
7406 if (!RC)
7407 return false;
7408 return LC->getValue() == RC->getValue();
7409 }
7410
7411 // Handle FloatingLiteral (floating point constants)
7412 if (const auto *LF = dyn_cast<FloatingLiteral>(Val: LHS)) {
7413 const auto *RF = dyn_cast<FloatingLiteral>(Val: RHS);
7414 if (!RF)
7415 return false;
7416 // Use bitwise comparison for floating point literals
7417 return LF->getValue().bitwiseIsEqual(RHS: RF->getValue());
7418 }
7419
7420 // Handle StringLiteral (string constants)
7421 if (const auto *LS = dyn_cast<StringLiteral>(Val: LHS)) {
7422 const auto *RS = dyn_cast<StringLiteral>(Val: RHS);
7423 if (!RS)
7424 return false;
7425 return LS->getString() == RS->getString();
7426 }
7427
7428 // Handle CXXNullPtrLiteralExpr (nullptr)
7429 if (isa<CXXNullPtrLiteralExpr>(Val: LHS) && isa<CXXNullPtrLiteralExpr>(Val: RHS))
7430 return true;
7431
7432 // Handle CXXBoolLiteralExpr (true/false)
7433 if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(Val: LHS)) {
7434 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(Val: RHS);
7435 if (!RB)
7436 return false;
7437 return LB->getValue() == RB->getValue();
7438 }
7439
7440 // Fallback for other forms - use the existing comparison method
7441 return Expr::isSameComparisonOperand(E1: LHS, E2: RHS);
7442 }
7443 };
7444
7445 /// Get the offset of the OMP_MAP_MEMBER_OF field.
7446 static unsigned getFlagMemberOffset() {
7447 unsigned Offset = 0;
7448 for (uint64_t Remain =
7449 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
7450 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7451 !(Remain & 1); Remain = Remain >> 1)
7452 Offset++;
7453 return Offset;
7454 }
7455
7456 /// Class that holds debugging information for a data mapping to be passed to
7457 /// the runtime library.
7458 class MappingExprInfo {
7459 /// The variable declaration used for the data mapping.
7460 const ValueDecl *MapDecl = nullptr;
7461 /// The original expression used in the map clause, or null if there is
7462 /// none.
7463 const Expr *MapExpr = nullptr;
7464
7465 public:
7466 MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr)
7467 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7468
7469 const ValueDecl *getMapDecl() const { return MapDecl; }
7470 const Expr *getMapExpr() const { return MapExpr; }
7471 };
7472
7473 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7474 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7475 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7476 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7477 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7478 using MapNonContiguousArrayTy =
7479 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7480 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7481 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7482 using MapData =
7483 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7484 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,
7485 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;
7486 using MapDataArrayTy = SmallVector<MapData, 4>;
7487
7488 /// This structure contains combined information generated for mappable
7489 /// clauses, including base pointers, pointers, sizes, map types, user-defined
7490 /// mappers, and non-contiguous information.
7491 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7492 MapExprsArrayTy Exprs;
7493 MapValueDeclsArrayTy Mappers;
7494 MapValueDeclsArrayTy DevicePtrDecls;
7495
7496 /// Append arrays in \a CurInfo.
7497 void append(MapCombinedInfoTy &CurInfo) {
7498 Exprs.append(in_start: CurInfo.Exprs.begin(), in_end: CurInfo.Exprs.end());
7499 DevicePtrDecls.append(in_start: CurInfo.DevicePtrDecls.begin(),
7500 in_end: CurInfo.DevicePtrDecls.end());
7501 Mappers.append(in_start: CurInfo.Mappers.begin(), in_end: CurInfo.Mappers.end());
7502 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7503 }
7504 };
7505
7506 /// Map between a struct and the its lowest & highest elements which have been
7507 /// mapped.
7508 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7509 /// HE(FieldIndex, Pointer)}
7510 struct StructRangeInfoTy {
7511 MapCombinedInfoTy PreliminaryMapData;
7512 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7513 0, Address::invalid()};
7514 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7515 0, Address::invalid()};
7516 Address Base = Address::invalid();
7517 Address LB = Address::invalid();
7518 bool IsArraySection = false;
7519 bool HasCompleteRecord = false;
7520 };
7521
7522 /// A struct to store the attach pointer and pointee information, to be used
7523 /// when emitting an attach entry.
7524 struct AttachInfoTy {
7525 Address AttachPtrAddr = Address::invalid();
7526 Address AttachPteeAddr = Address::invalid();
7527 const ValueDecl *AttachPtrDecl = nullptr;
7528 const Expr *AttachMapExpr = nullptr;
7529
7530 bool isValid() const {
7531 return AttachPtrAddr.isValid() && AttachPteeAddr.isValid();
7532 }
7533 };
7534
7535 /// Check if there's any component list where the attach pointer expression
7536 /// matches the given captured variable.
7537 bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const {
7538 for (const auto &AttachEntry : AttachPtrExprMap) {
7539 if (AttachEntry.second) {
7540 // Check if the attach pointer expression is a DeclRefExpr that
7541 // references the captured variable
7542 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: AttachEntry.second))
7543 if (DRE->getDecl() == VD)
7544 return true;
7545 }
7546 }
7547 return false;
7548 }
7549
7550 /// Get the previously-cached attach pointer for a component list, if-any.
7551 const Expr *getAttachPtrExpr(
7552 OMPClauseMappableExprCommon::MappableExprComponentListRef Components)
7553 const {
7554 const auto It = AttachPtrExprMap.find(Val: Components);
7555 if (It != AttachPtrExprMap.end())
7556 return It->second;
7557
7558 return nullptr;
7559 }
7560
7561private:
7562 /// Kind that defines how a device pointer has to be returned.
7563 struct MapInfo {
7564 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7565 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
7566 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7567 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7568 bool ReturnDevicePointer = false;
7569 bool IsImplicit = false;
7570 const ValueDecl *Mapper = nullptr;
7571 const Expr *VarRef = nullptr;
7572 bool ForDeviceAddr = false;
7573 bool HasUdpFbNullify = false;
7574
7575 MapInfo() = default;
7576 MapInfo(
7577 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7578 OpenMPMapClauseKind MapType,
7579 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7580 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7581 bool ReturnDevicePointer, bool IsImplicit,
7582 const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr,
7583 bool ForDeviceAddr = false, bool HasUdpFbNullify = false)
7584 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7585 MotionModifiers(MotionModifiers),
7586 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7587 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr),
7588 HasUdpFbNullify(HasUdpFbNullify) {}
7589 };
7590
7591 /// The target directive from where the mappable clauses were extracted. It
7592 /// is either a executable directive or a user-defined mapper directive.
7593 llvm::PointerUnion<const OMPExecutableDirective *,
7594 const OMPDeclareMapperDecl *>
7595 CurDir;
7596
7597 /// Function the directive is being generated for.
7598 CodeGenFunction &CGF;
7599
7600 /// Set of all first private variables in the current directive.
7601 /// bool data is set to true if the variable is implicitly marked as
7602 /// firstprivate, false otherwise.
7603 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7604
7605 /// Set of defaultmap clause kinds that use firstprivate behavior.
7606 llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
7607
7608 /// Map between device pointer declarations and their expression components.
7609 /// The key value for declarations in 'this' is null.
7610 llvm::DenseMap<
7611 const ValueDecl *,
7612 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7613 DevPointersMap;
7614
7615 /// Map between device addr declarations and their expression components.
7616 /// The key value for declarations in 'this' is null.
7617 llvm::DenseMap<
7618 const ValueDecl *,
7619 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7620 HasDevAddrsMap;
7621
7622 /// Map between lambda declarations and their map type.
7623 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7624
7625 /// Map from component lists to their attach pointer expressions.
7626 llvm::DenseMap<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7627 const Expr *>
7628 AttachPtrExprMap;
7629
7630 /// Map from attach pointer expressions to their component depth.
7631 /// nullptr key has std::nullopt depth. This can be used to order attach-ptr
7632 /// expressions with increasing/decreasing depth.
7633 /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`.
7634 /// TODO: Not urgent, but we should ideally use the number of pointer
7635 /// dereferences in an expr as an indicator of its complexity, instead of the
7636 /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`,
7637 /// `*(p + 5 + 5)` together.
7638 llvm::DenseMap<const Expr *, std::optional<size_t>>
7639 AttachPtrComponentDepthMap = {{nullptr, std::nullopt}};
7640
7641 /// Map from attach pointer expressions to the order they were computed in, in
7642 /// collectAttachPtrExprInfo().
7643 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7644 {nullptr, 0}};
7645
7646 /// An instance of attach-ptr-expr comparator that can be used throughout the
7647 /// lifetime of this handler.
7648 AttachPtrExprComparator AttachPtrComparator;
7649
7650 llvm::Value *getExprTypeSize(const Expr *E) const {
7651 QualType ExprTy = E->getType().getCanonicalType();
7652
7653 // Calculate the size for array shaping expression.
7654 if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(Val: E)) {
7655 llvm::Value *Size =
7656 CGF.getTypeSize(Ty: OAE->getBase()->getType()->getPointeeType());
7657 for (const Expr *SE : OAE->getDimensions()) {
7658 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
7659 Sz = CGF.EmitScalarConversion(Src: Sz, SrcTy: SE->getType(),
7660 DstTy: CGF.getContext().getSizeType(),
7661 Loc: SE->getExprLoc());
7662 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: Sz);
7663 }
7664 return Size;
7665 }
7666
7667 // Reference types are ignored for mapping purposes.
7668 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7669 ExprTy = RefTy->getPointeeType().getCanonicalType();
7670
7671 // Given that an array section is considered a built-in type, we need to
7672 // do the calculation based on the length of the section instead of relying
7673 // on CGF.getTypeSize(E->getType()).
7674 if (const auto *OAE = dyn_cast<ArraySectionExpr>(Val: E)) {
7675 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(
7676 Base: OAE->getBase()->IgnoreParenImpCasts())
7677 .getCanonicalType();
7678
7679 // If there is no length associated with the expression and lower bound is
7680 // not specified too, that means we are using the whole length of the
7681 // base.
7682 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7683 !OAE->getLowerBound())
7684 return CGF.getTypeSize(Ty: BaseTy);
7685
7686 llvm::Value *ElemSize;
7687 if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7688 ElemSize = CGF.getTypeSize(Ty: PTy->getPointeeType().getCanonicalType());
7689 } else {
7690 const auto *ATy = cast<ArrayType>(Val: BaseTy.getTypePtr());
7691 assert(ATy && "Expecting array type if not a pointer type.");
7692 ElemSize = CGF.getTypeSize(Ty: ATy->getElementType().getCanonicalType());
7693 }
7694
7695 // If we don't have a length at this point, that is because we have an
7696 // array section with a single element.
7697 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7698 return ElemSize;
7699
7700 if (const Expr *LenExpr = OAE->getLength()) {
7701 llvm::Value *LengthVal = CGF.EmitScalarExpr(E: LenExpr);
7702 LengthVal = CGF.EmitScalarConversion(Src: LengthVal, SrcTy: LenExpr->getType(),
7703 DstTy: CGF.getContext().getSizeType(),
7704 Loc: LenExpr->getExprLoc());
7705 return CGF.Builder.CreateNUWMul(LHS: LengthVal, RHS: ElemSize);
7706 }
7707 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7708 OAE->getLowerBound() && "expected array_section[lb:].");
7709 // Size = sizetype - lb * elemtype;
7710 llvm::Value *LengthVal = CGF.getTypeSize(Ty: BaseTy);
7711 llvm::Value *LBVal = CGF.EmitScalarExpr(E: OAE->getLowerBound());
7712 LBVal = CGF.EmitScalarConversion(Src: LBVal, SrcTy: OAE->getLowerBound()->getType(),
7713 DstTy: CGF.getContext().getSizeType(),
7714 Loc: OAE->getLowerBound()->getExprLoc());
7715 LBVal = CGF.Builder.CreateNUWMul(LHS: LBVal, RHS: ElemSize);
7716 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LHS: LengthVal, RHS: LBVal);
7717 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LHS: LengthVal, RHS: LBVal);
7718 LengthVal = CGF.Builder.CreateSelect(
7719 C: Cmp, True: TrueVal, False: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0));
7720 return LengthVal;
7721 }
7722 return CGF.getTypeSize(Ty: ExprTy);
7723 }
7724
7725 /// Return the corresponding bits for a given map clause modifier. Add
7726 /// a flag marking the map as a pointer if requested. Add a flag marking the
7727 /// map as the first one of a series of maps that relate to the same map
7728 /// expression.
7729 OpenMPOffloadMappingFlags getMapTypeBits(
7730 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7731 ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,
7732 bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const {
7733 OpenMPOffloadMappingFlags Bits =
7734 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7735 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7736 switch (MapType) {
7737 case OMPC_MAP_alloc:
7738 case OMPC_MAP_release:
7739 // alloc and release is the default behavior in the runtime library, i.e.
7740 // if we don't pass any bits alloc/release that is what the runtime is
7741 // going to do. Therefore, we don't need to signal anything for these two
7742 // type modifiers.
7743 break;
7744 case OMPC_MAP_to:
7745 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7746 break;
7747 case OMPC_MAP_from:
7748 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7749 break;
7750 case OMPC_MAP_tofrom:
7751 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7752 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7753 break;
7754 case OMPC_MAP_delete:
7755 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7756 break;
7757 case OMPC_MAP_unknown:
7758 llvm_unreachable("Unexpected map type!");
7759 }
7760 if (AddPtrFlag)
7761 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7762 if (AddIsTargetParamFlag)
7763 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7764 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_always))
7765 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7766 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_close))
7767 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7768 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_present) ||
7769 llvm::is_contained(Range&: MotionModifiers, Element: OMPC_MOTION_MODIFIER_present))
7770 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7771 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_ompx_hold))
7772 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7773 if (IsNonContiguous)
7774 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7775 return Bits;
7776 }
7777
7778 /// Return true if the provided expression is a final array section. A
7779 /// final array section, is one whose length can't be proved to be one.
7780 bool isFinalArraySectionExpression(const Expr *E) const {
7781 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E);
7782
7783 // It is not an array section and therefore not a unity-size one.
7784 if (!OASE)
7785 return false;
7786
7787 // An array section with no colon always refer to a single element.
7788 if (OASE->getColonLocFirst().isInvalid())
7789 return false;
7790
7791 const Expr *Length = OASE->getLength();
7792
7793 // If we don't have a length we have to check if the array has size 1
7794 // for this dimension. Also, we should always expect a length if the
7795 // base type is pointer.
7796 if (!Length) {
7797 QualType BaseQTy = ArraySectionExpr::getBaseOriginalType(
7798 Base: OASE->getBase()->IgnoreParenImpCasts())
7799 .getCanonicalType();
7800 if (const auto *ATy = dyn_cast<ConstantArrayType>(Val: BaseQTy.getTypePtr()))
7801 return ATy->getSExtSize() != 1;
7802 // If we don't have a constant dimension length, we have to consider
7803 // the current section as having any size, so it is not necessarily
7804 // unitary. If it happen to be unity size, that's user fault.
7805 return true;
7806 }
7807
7808 // Check if the length evaluates to 1.
7809 Expr::EvalResult Result;
7810 if (!Length->EvaluateAsInt(Result, Ctx: CGF.getContext()))
7811 return true; // Can have more that size 1.
7812
7813 llvm::APSInt ConstLength = Result.Val.getInt();
7814 return ConstLength.getSExtValue() != 1;
7815 }
7816
7817 /// Emit an attach entry into \p CombinedInfo, using the information from \p
7818 /// AttachInfo. For example, for a map of form `int *p; ... map(p[1:10])`,
7819 /// an attach entry has the following form:
7820 /// &p, &p[1], sizeof(void*), ATTACH
7821 void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo,
7822 const AttachInfoTy &AttachInfo) const {
7823 assert(AttachInfo.isValid() &&
7824 "Expected valid attach pointer/pointee information!");
7825
7826 // Size is the size of the pointer itself - use pointer size, not BaseDecl
7827 // size
7828 llvm::Value *PointerSize = CGF.Builder.CreateIntCast(
7829 V: llvm::ConstantInt::get(
7830 Ty: CGF.CGM.SizeTy, V: CGF.getContext()
7831 .getTypeSizeInChars(T: CGF.getContext().VoidPtrTy)
7832 .getQuantity()),
7833 DestTy: CGF.Int64Ty, /*isSigned=*/true);
7834
7835 CombinedInfo.Exprs.emplace_back(Args: AttachInfo.AttachPtrDecl,
7836 Args: AttachInfo.AttachMapExpr);
7837 CombinedInfo.BasePointers.push_back(
7838 Elt: AttachInfo.AttachPtrAddr.emitRawPointer(CGF));
7839 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7840 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7841 CombinedInfo.Pointers.push_back(
7842 Elt: AttachInfo.AttachPteeAddr.emitRawPointer(CGF));
7843 CombinedInfo.Sizes.push_back(Elt: PointerSize);
7844 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
7845 // ATTACH entries themselves don't "have" a base attach-ptr.
7846 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7847 CombinedInfo.Mappers.push_back(Elt: nullptr);
7848 CombinedInfo.NonContigInfo.Dims.push_back(Elt: 1);
7849 }
7850
7851 /// A helper class to copy structures with overlapped elements, i.e. those
7852 /// which have mappings of both "s" and "s.mem". Consecutive elements that
7853 /// are not explicitly copied have mapping nodes synthesized for them,
7854 /// taking care to avoid generating zero-sized copies.
7855 class CopyOverlappedEntryGaps {
7856 CodeGenFunction &CGF;
7857 MapCombinedInfoTy &CombinedInfo;
7858 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7859 const ValueDecl *MapDecl = nullptr;
7860 const Expr *MapExpr = nullptr;
7861 Address BP = Address::invalid();
7862 bool IsNonContiguous = false;
7863 uint64_t DimSize = 0;
7864 // These elements track the position as the struct is iterated over
7865 // (in order of increasing element address).
7866 const RecordDecl *LastParent = nullptr;
7867 uint64_t Cursor = 0;
7868 unsigned LastIndex = -1u;
7869 Address LB = Address::invalid();
7870
7871 public:
7872 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7873 MapCombinedInfoTy &CombinedInfo,
7874 OpenMPOffloadMappingFlags Flags,
7875 const ValueDecl *MapDecl, const Expr *MapExpr,
7876 Address BP, Address LB, bool IsNonContiguous,
7877 uint64_t DimSize)
7878 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7879 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7880 DimSize(DimSize), LB(LB) {}
7881
7882 void processField(
7883 const OMPClauseMappableExprCommon::MappableComponent &MC,
7884 const FieldDecl *FD,
7885 llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)>
7886 EmitMemberExprBase) {
7887 const RecordDecl *RD = FD->getParent();
7888 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(D: RD);
7889 uint64_t FieldOffset = RL.getFieldOffset(FieldNo: FD->getFieldIndex());
7890 uint64_t FieldSize =
7891 CGF.getContext().getTypeSize(T: FD->getType().getCanonicalType());
7892 Address ComponentLB = Address::invalid();
7893
7894 if (FD->getType()->isLValueReferenceType()) {
7895 const auto *ME = cast<MemberExpr>(Val: MC.getAssociatedExpression());
7896 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7897 ComponentLB =
7898 CGF.EmitLValueForFieldInitialization(Base: BaseLVal, Field: FD).getAddress();
7899 } else {
7900 ComponentLB =
7901 CGF.EmitOMPSharedLValue(E: MC.getAssociatedExpression()).getAddress();
7902 }
7903
7904 if (!LastParent)
7905 LastParent = RD;
7906 if (FD->getParent() == LastParent) {
7907 if (FD->getFieldIndex() != LastIndex + 1)
7908 copyUntilField(FD, ComponentLB);
7909 } else {
7910 LastParent = FD->getParent();
7911 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7912 copyUntilField(FD, ComponentLB);
7913 }
7914 Cursor = FieldOffset + FieldSize;
7915 LastIndex = FD->getFieldIndex();
7916 LB = CGF.Builder.CreateConstGEP(Addr: ComponentLB, Index: 1);
7917 }
7918
7919 void copyUntilField(const FieldDecl *FD, Address ComponentLB) {
7920 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7921 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7922 llvm::Value *Size = CGF.Builder.CreatePtrDiff(LHS: ComponentLBPtr, RHS: LBPtr);
7923 copySizedChunk(Base: LBPtr, Size);
7924 }
7925
7926 void copyUntilEnd(Address HB) {
7927 if (LastParent) {
7928 const ASTRecordLayout &RL =
7929 CGF.getContext().getASTRecordLayout(D: LastParent);
7930 if ((uint64_t)CGF.getContext().toBits(CharSize: RL.getSize()) <= Cursor)
7931 return;
7932 }
7933 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7934 llvm::Value *Size = CGF.Builder.CreatePtrDiff(
7935 LHS: CGF.Builder.CreateConstGEP(Addr: HB, Index: 1).emitRawPointer(CGF), RHS: LBPtr);
7936 copySizedChunk(Base: LBPtr, Size);
7937 }
7938
7939 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7940 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
7941 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
7942 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7943 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7944 CombinedInfo.Pointers.push_back(Elt: Base);
7945 CombinedInfo.Sizes.push_back(
7946 Elt: CGF.Builder.CreateIntCast(V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/false));
7947 CombinedInfo.Types.push_back(Elt: Flags);
7948 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7949 CombinedInfo.Mappers.push_back(Elt: nullptr);
7950 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize : 1);
7951 }
7952 };
7953
7954 /// Generate the base pointers, section pointers, sizes, map type bits, and
7955 /// user-defined mappers (all included in \a CombinedInfo) for the provided
7956 /// map type, map or motion modifiers, and expression components.
7957 /// \a IsFirstComponent should be set to true if the provided set of
7958 /// components is the first associated with a capture.
7959 void generateInfoForComponentList(
7960 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7961 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7962 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7963 MapCombinedInfoTy &CombinedInfo,
7964 MapCombinedInfoTy &StructBaseCombinedInfo,
7965 StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo,
7966 bool IsFirstComponentList, bool IsImplicit,
7967 bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr,
7968 bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr,
7969 const Expr *MapExpr = nullptr,
7970 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7971 OverlappedElements = {}) const {
7972
7973 // The following summarizes what has to be generated for each map and the
7974 // types below. The generated information is expressed in this order:
7975 // base pointer, section pointer, size, flags
7976 // (to add to the ones that come from the map type and modifier).
7977 // Entries annotated with (+) are only generated for "target" constructs,
7978 // and only if the variable at the beginning of the expression is used in
7979 // the region.
7980 //
7981 // double d;
7982 // int i[100];
7983 // float *p;
7984 // int **a = &i;
7985 //
7986 // struct S1 {
7987 // int i;
7988 // float f[50];
7989 // }
7990 // struct S2 {
7991 // int i;
7992 // float f[50];
7993 // S1 s;
7994 // double *p;
7995 // double *&pref;
7996 // struct S2 *ps;
7997 // int &ref;
7998 // }
7999 // S2 s;
8000 // S2 *ps;
8001 //
8002 // map(d)
8003 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
8004 //
8005 // map(i)
8006 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
8007 //
8008 // map(i[1:23])
8009 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
8010 //
8011 // map(p)
8012 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
8013 //
8014 // map(p[1:24])
8015 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM // map pointee
8016 // &p, &p[1], sizeof(void*), ATTACH // attach pointer/pointee, if both
8017 // // are present, and either is new
8018 //
8019 // map(([22])p)
8020 // p, p, 22*sizeof(float), TARGET_PARAM | TO | FROM
8021 // &p, p, sizeof(void*), ATTACH
8022 //
8023 // map((*a)[0:3])
8024 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
8025 // (*a)[0], &(*a)[0], 3 * sizeof(int), TO | FROM
8026 // &(*a), &(*a)[0], sizeof(void*), ATTACH
8027 // (+) Only on target, if a is used in the region
8028 // Note: Since the attach base-pointer is `*a`, which is not a scalar
8029 // variable, it doesn't determine the clause on `a`. `a` is mapped using
8030 // a zero-length-array-section map by generateDefaultMapInfo, if it is
8031 // referenced in the target region, because it is a pointer.
8032 //
8033 // map(**a)
8034 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
8035 // &(*a)[0], &(*a)[0], sizeof(int), TO | FROM
8036 // &(*a), &(*a)[0], sizeof(void*), ATTACH
8037 // (+) Only on target, if a is used in the region
8038 //
8039 // map(s)
8040 // FIXME: This needs to also imply map(ref_ptr_ptee: s.ref), since the
8041 // effect is supposed to be same as if the user had a map for every element
8042 // of the struct. We currently do a shallow-map of s.
8043 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
8044 //
8045 // map(s.i)
8046 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
8047 //
8048 // map(s.s.f)
8049 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
8050 //
8051 // map(s.p)
8052 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
8053 //
8054 // map(to: s.p[:22])
8055 // &s, &(s.p), sizeof(double*), TARGET_PARAM | IMPLICIT // (+)
8056 // &(s.p[0]), &(s.p[0]), 22 * sizeof(double*), TO | FROM
8057 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8058 //
8059 // map(to: s.ref)
8060 // &s, &(ptr(s.ref)), sizeof(int*), TARGET_PARAM (*)
8061 // &s, &(ptee(s.ref)), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
8062 // (*) alloc space for struct members, only this is a target parameter.
8063 // (**) map the pointer (nothing to be mapped in this example) (the compiler
8064 // optimizes this entry out, same in the examples below)
8065 // (***) map the pointee (map: to)
8066 // Note: ptr(s.ref) represents the referring pointer of s.ref
8067 // ptee(s.ref) represents the referenced pointee of s.ref
8068 //
8069 // map(to: s.pref)
8070 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM
8071 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO
8072 //
8073 // map(to: s.pref[:22])
8074 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | IMPLICIT // (+)
8075 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO |
8076 // FROM | IMPLICIT // (+)
8077 // &(ptee(s.pref)[0]), &(ptee(s.pref)[0]), 22 * sizeof(double), TO
8078 // &(ptee(s.pref)), &(ptee(s.pref)[0]), sizeof(void*), ATTACH
8079 //
8080 // map(s.ps)
8081 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
8082 //
8083 // map(from: s.ps->s.i)
8084 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8085 // &(s.ps[0]), &(s.ps->s.i), sizeof(int), FROM
8086 // &(s.ps), &(s.ps->s.i), sizeof(void*), ATTACH
8087 //
8088 // map(to: s.ps->ps)
8089 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8090 // &(s.ps[0]), &(s.ps->ps), sizeof(S2*), TO
8091 // &(s.ps), &(s.ps->ps), sizeof(void*), ATTACH
8092 //
8093 // map(s.ps->ps->ps)
8094 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8095 // &(s.ps->ps[0]), &(s.ps->ps->ps), sizeof(S2*), TO
8096 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(void*), ATTACH
8097 //
8098 // map(to: s.ps->ps->s.f[:22])
8099 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8100 // &(s.ps->ps[0]), &(s.ps->ps->s.f[0]), 22*sizeof(float), TO
8101 // &(s.ps->ps), &(s.ps->ps->s.f[0]), sizeof(void*), ATTACH
8102 //
8103 // map(ps)
8104 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
8105 //
8106 // map(ps->i)
8107 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
8108 // &ps, &(ps->i), sizeof(void*), ATTACH
8109 //
8110 // map(ps->s.f)
8111 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
8112 // &ps, &(ps->s.f[0]), sizeof(ps), ATTACH
8113 //
8114 // map(from: ps->p)
8115 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
8116 // &ps, &(ps->p), sizeof(ps), ATTACH
8117 //
8118 // map(to: ps->p[:22])
8119 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
8120 // &(ps->p[0]), &(ps->p[0]), 22*sizeof(double), TO
8121 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8122 //
8123 // map(ps->ps)
8124 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
8125 // &ps, &(ps->ps), sizeof(ps), ATTACH
8126 //
8127 // map(from: ps->ps->s.i)
8128 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
8129 // &(ps->ps[0]), &(ps->ps->s.i), sizeof(int), FROM
8130 // &(ps->ps), &(ps->ps->s.i), sizeof(void*), ATTACH
8131 //
8132 // map(from: ps->ps->ps)
8133 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8134 // &(ps->ps[0]), &(ps->ps->ps), sizeof(S2*), FROM
8135 // &(ps->ps), &(ps->ps->ps), sizeof(void*), ATTACH
8136 //
8137 // map(ps->ps->ps->ps)
8138 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8139 // &(ps->ps->ps[0]), &(ps->ps->ps->ps), sizeof(S2*), FROM
8140 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(void*), ATTACH
8141 //
8142 // map(to: ps->ps->ps->s.f[:22])
8143 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8144 // &(ps->ps->ps[0]), &(ps->ps->ps->s.f[0]), 22*sizeof(float), TO
8145 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), sizeof(void*), ATTACH
8146 //
8147 // map(to: s.f[:22]) map(from: s.p[:33])
8148 // On target, and if s is used in the region:
8149 //
8150 // &s, &(s.f[0]), 50*sizeof(float) +
8151 // sizeof(struct S1) +
8152 // sizeof(double*) (**), TARGET_PARAM
8153 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
8154 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) | TO |
8155 // FROM | IMPLICIT
8156 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
8157 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8158 // (**) allocate contiguous space needed to fit all mapped members even if
8159 // we allocate space for members not mapped (in this example,
8160 // s.f[22..49] and s.s are not mapped, yet we must allocate space for
8161 // them as well because they fall between &s.f[0] and &s.p)
8162 //
8163 // On other constructs, and, if s is not used in the region, on target:
8164 // &s, &(s.f[0]), 22*sizeof(float), TO
8165 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
8166 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8167 //
8168 // map(from: s.f[:22]) map(to: ps->p[:33])
8169 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
8170 // &ps[0], &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8171 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
8172 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8173 //
8174 // map(from: s.f[:22], s.s) map(to: ps->p[:33])
8175 // &s, &(s.f[0]), 50*sizeof(float) +
8176 // sizeof(struct S1), TARGET_PARAM
8177 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
8178 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
8179 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8180 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
8181 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8182 //
8183 // map(p[:100], p)
8184 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
8185 // p, &p[0], 100*sizeof(float), TO | FROM
8186 // &p, &p[0], sizeof(float*), ATTACH
8187
8188 // Track if the map information being generated is the first for a capture.
8189 bool IsCaptureFirstInfo = IsFirstComponentList;
8190 // When the variable is on a declare target link or in a to clause with
8191 // unified memory, a reference is needed to hold the host/device address
8192 // of the variable.
8193 bool RequiresReference = false;
8194
8195 // Scan the components from the base to the complete expression.
8196 auto CI = Components.rbegin();
8197 auto CE = Components.rend();
8198 auto I = CI;
8199
8200 // Track if the map information being generated is the first for a list of
8201 // components.
8202 bool IsExpressionFirstInfo = true;
8203 bool FirstPointerInComplexData = false;
8204 Address BP = Address::invalid();
8205 Address FinalLowestElem = Address::invalid();
8206 const Expr *AssocExpr = I->getAssociatedExpression();
8207 const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr);
8208 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8209 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(Val: AssocExpr);
8210
8211 // Get the pointer-attachment base-pointer for the given list, if any.
8212 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
8213 auto [AttachPtrAddr, AttachPteeBaseAddr] =
8214 getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF);
8215
8216 bool HasAttachPtr = AttachPtrExpr != nullptr;
8217 bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr;
8218 bool SeenAttachPtr = FirstComponentIsForAttachPtr;
8219
8220 if (FirstComponentIsForAttachPtr) {
8221 // No need to process AttachPtr here. It will be processed at the end
8222 // after we have computed the pointee's address.
8223 ++I;
8224 } else if (isa<MemberExpr>(Val: AssocExpr)) {
8225 // The base is the 'this' pointer. The content of the pointer is going
8226 // to be the base of the field being mapped.
8227 BP = CGF.LoadCXXThisAddress();
8228 } else if ((AE && isa<CXXThisExpr>(Val: AE->getBase()->IgnoreParenImpCasts())) ||
8229 (OASE &&
8230 isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))) {
8231 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8232 } else if (OAShE &&
8233 isa<CXXThisExpr>(Val: OAShE->getBase()->IgnoreParenCasts())) {
8234 BP = Address(
8235 CGF.EmitScalarExpr(E: OAShE->getBase()),
8236 CGF.ConvertTypeForMem(T: OAShE->getBase()->getType()->getPointeeType()),
8237 CGF.getContext().getTypeAlignInChars(T: OAShE->getBase()->getType()));
8238 } else {
8239 // The base is the reference to the variable.
8240 // BP = &Var.
8241 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8242 if (const auto *VD =
8243 dyn_cast_or_null<VarDecl>(Val: I->getAssociatedDeclaration())) {
8244 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8245 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
8246 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
8247 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
8248 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
8249 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
8250 RequiresReference = true;
8251 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
8252 }
8253 }
8254 }
8255
8256 // If the variable is a pointer and is being dereferenced (i.e. is not
8257 // the last component), the base has to be the pointer itself, not its
8258 // reference. References are ignored for mapping purposes.
8259 QualType Ty =
8260 I->getAssociatedDeclaration()->getType().getNonReferenceType();
8261 if (Ty->isAnyPointerType() && std::next(x: I) != CE) {
8262 // No need to generate individual map information for the pointer, it
8263 // can be associated with the combined storage if shared memory mode is
8264 // active or the base declaration is not global variable.
8265 const auto *VD = dyn_cast<VarDecl>(Val: I->getAssociatedDeclaration());
8266 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8267 !VD || VD->hasLocalStorage() || HasAttachPtr)
8268 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8269 else
8270 FirstPointerInComplexData = true;
8271 ++I;
8272 }
8273 }
8274
8275 // Track whether a component of the list should be marked as MEMBER_OF some
8276 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
8277 // in a component list should be marked as MEMBER_OF, all subsequent entries
8278 // do not belong to the base struct. E.g.
8279 // struct S2 s;
8280 // s.ps->ps->ps->f[:]
8281 // (1) (2) (3) (4)
8282 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
8283 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
8284 // is the pointee of ps(2) which is not member of struct s, so it should not
8285 // be marked as such (it is still PTR_AND_OBJ).
8286 // The variable is initialized to false so that PTR_AND_OBJ entries which
8287 // are not struct members are not considered (e.g. array of pointers to
8288 // data).
8289 bool ShouldBeMemberOf = false;
8290
8291 // Variable keeping track of whether or not we have encountered a component
8292 // in the component list which is a member expression. Useful when we have a
8293 // pointer or a final array section, in which case it is the previous
8294 // component in the list which tells us whether we have a member expression.
8295 // E.g. X.f[:]
8296 // While processing the final array section "[:]" it is "f" which tells us
8297 // whether we are dealing with a member of a declared struct.
8298 const MemberExpr *EncounteredME = nullptr;
8299
8300 // Track for the total number of dimension. Start from one for the dummy
8301 // dimension.
8302 uint64_t DimSize = 1;
8303
8304 // Detects non-contiguous updates due to strided accesses.
8305 // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set
8306 // correctly when generating information to be passed to the runtime. The
8307 // flag is set to true if any array section has a stride not equal to 1, or
8308 // if the stride is not a constant expression (conservatively assumed
8309 // non-contiguous).
8310 bool IsNonContiguous =
8311 CombinedInfo.NonContigInfo.IsNonContiguous ||
8312 any_of(Range&: Components, P: [&](const auto &Component) {
8313 const auto *OASE =
8314 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
8315 if (!OASE)
8316 return false;
8317
8318 const Expr *StrideExpr = OASE->getStride();
8319 if (!StrideExpr)
8320 return false;
8321
8322 assert(StrideExpr->getType()->isIntegerType() &&
8323 "Stride expression must be of integer type");
8324
8325 // If stride is not evaluatable as a constant, treat as
8326 // non-contiguous.
8327 const auto Constant =
8328 StrideExpr->getIntegerConstantExpr(Ctx: CGF.getContext());
8329 if (!Constant)
8330 return true;
8331
8332 // Treat non-unitary strides as non-contiguous.
8333 return !Constant->isOne();
8334 });
8335
8336 bool IsPrevMemberReference = false;
8337
8338 bool IsPartialMapped =
8339 !PartialStruct.PreliminaryMapData.BasePointers.empty();
8340
8341 // We need to check if we will be encountering any MEs. If we do not
8342 // encounter any ME expression it means we will be mapping the whole struct.
8343 // In that case we need to skip adding an entry for the struct to the
8344 // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo
8345 // list only when generating all info for clauses.
8346 bool IsMappingWholeStruct = true;
8347 if (!GenerateAllInfoForClauses) {
8348 IsMappingWholeStruct = false;
8349 } else {
8350 for (auto TempI = I; TempI != CE; ++TempI) {
8351 const MemberExpr *PossibleME =
8352 dyn_cast<MemberExpr>(Val: TempI->getAssociatedExpression());
8353 if (PossibleME) {
8354 IsMappingWholeStruct = false;
8355 break;
8356 }
8357 }
8358 }
8359
8360 bool SeenFirstNonBinOpExprAfterAttachPtr = false;
8361 for (; I != CE; ++I) {
8362 // If we have a valid attach-ptr, we skip processing all components until
8363 // after the attach-ptr.
8364 if (HasAttachPtr && !SeenAttachPtr) {
8365 SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr;
8366 continue;
8367 }
8368
8369 // After finding the attach pointer, skip binary-ops, to skip past
8370 // expressions like (p + 10), for a map like map(*(p + 10)), where p is
8371 // the attach-ptr.
8372 if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) {
8373 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8374 if (BO)
8375 continue;
8376
8377 // Found the first non-binary-operator component after attach
8378 SeenFirstNonBinOpExprAfterAttachPtr = true;
8379 BP = AttachPteeBaseAddr;
8380 }
8381
8382 // If the current component is member of a struct (parent struct) mark it.
8383 if (!EncounteredME) {
8384 EncounteredME = dyn_cast<MemberExpr>(Val: I->getAssociatedExpression());
8385 // If we encounter a PTR_AND_OBJ entry from now on it should be marked
8386 // as MEMBER_OF the parent struct.
8387 if (EncounteredME) {
8388 ShouldBeMemberOf = true;
8389 // Do not emit as complex pointer if this is actually not array-like
8390 // expression.
8391 if (FirstPointerInComplexData) {
8392 QualType Ty = std::prev(x: I)
8393 ->getAssociatedDeclaration()
8394 ->getType()
8395 .getNonReferenceType();
8396 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8397 FirstPointerInComplexData = false;
8398 }
8399 }
8400 }
8401
8402 auto Next = std::next(x: I);
8403
8404 // We need to generate the addresses and sizes if this is the last
8405 // component, if the component is a pointer or if it is an array section
8406 // whose length can't be proved to be one. If this is a pointer, it
8407 // becomes the base address for the following components.
8408
8409 // A final array section, is one whose length can't be proved to be one.
8410 // If the map item is non-contiguous then we don't treat any array section
8411 // as final array section.
8412 bool IsFinalArraySection =
8413 !IsNonContiguous &&
8414 isFinalArraySectionExpression(E: I->getAssociatedExpression());
8415
8416 // If we have a declaration for the mapping use that, otherwise use
8417 // the base declaration of the map clause.
8418 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
8419 ? I->getAssociatedDeclaration()
8420 : BaseDecl;
8421 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
8422 : MapExpr;
8423
8424 // Get information on whether the element is a pointer. Have to do a
8425 // special treatment for array sections given that they are built-in
8426 // types.
8427 const auto *OASE =
8428 dyn_cast<ArraySectionExpr>(Val: I->getAssociatedExpression());
8429 const auto *OAShE =
8430 dyn_cast<OMPArrayShapingExpr>(Val: I->getAssociatedExpression());
8431 const auto *UO = dyn_cast<UnaryOperator>(Val: I->getAssociatedExpression());
8432 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8433 bool IsPointer =
8434 OAShE ||
8435 (OASE && ArraySectionExpr::getBaseOriginalType(Base: OASE)
8436 .getCanonicalType()
8437 ->isAnyPointerType()) ||
8438 I->getAssociatedExpression()->getType()->isAnyPointerType();
8439 bool IsMemberReference = isa<MemberExpr>(Val: I->getAssociatedExpression()) &&
8440 MapDecl &&
8441 MapDecl->getType()->isLValueReferenceType();
8442 bool IsNonDerefPointer = IsPointer &&
8443 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
8444 !IsNonContiguous;
8445
8446 if (OASE)
8447 ++DimSize;
8448
8449 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
8450 IsFinalArraySection) {
8451 // If this is not the last component, we expect the pointer to be
8452 // associated with an array expression or member expression.
8453 assert((Next == CE ||
8454 isa<MemberExpr>(Next->getAssociatedExpression()) ||
8455 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
8456 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
8457 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
8458 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
8459 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
8460 "Unexpected expression");
8461
8462 Address LB = Address::invalid();
8463 Address LowestElem = Address::invalid();
8464 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
8465 const MemberExpr *E) {
8466 const Expr *BaseExpr = E->getBase();
8467 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a
8468 // scalar.
8469 LValue BaseLV;
8470 if (E->isArrow()) {
8471 LValueBaseInfo BaseInfo;
8472 TBAAAccessInfo TBAAInfo;
8473 Address Addr =
8474 CGF.EmitPointerWithAlignment(Addr: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
8475 QualType PtrTy = BaseExpr->getType()->getPointeeType();
8476 BaseLV = CGF.MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
8477 } else {
8478 BaseLV = CGF.EmitOMPSharedLValue(E: BaseExpr);
8479 }
8480 return BaseLV;
8481 };
8482 if (OAShE) {
8483 LowestElem = LB =
8484 Address(CGF.EmitScalarExpr(E: OAShE->getBase()),
8485 CGF.ConvertTypeForMem(
8486 T: OAShE->getBase()->getType()->getPointeeType()),
8487 CGF.getContext().getTypeAlignInChars(
8488 T: OAShE->getBase()->getType()));
8489 } else if (IsMemberReference) {
8490 const auto *ME = cast<MemberExpr>(Val: I->getAssociatedExpression());
8491 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
8492 LowestElem = CGF.EmitLValueForFieldInitialization(
8493 Base: BaseLVal, Field: cast<FieldDecl>(Val: MapDecl))
8494 .getAddress();
8495 LB = CGF.EmitLoadOfReferenceLValue(RefAddr: LowestElem, RefTy: MapDecl->getType())
8496 .getAddress();
8497 } else {
8498 LowestElem = LB =
8499 CGF.EmitOMPSharedLValue(E: I->getAssociatedExpression())
8500 .getAddress();
8501 }
8502
8503 // Save the final LowestElem, to use it as the pointee in attach maps,
8504 // if emitted.
8505 if (Next == CE)
8506 FinalLowestElem = LowestElem;
8507
8508 // If this component is a pointer inside the base struct then we don't
8509 // need to create any entry for it - it will be combined with the object
8510 // it is pointing to into a single PTR_AND_OBJ entry.
8511 bool IsMemberPointerOrAddr =
8512 EncounteredME &&
8513 (((IsPointer || ForDeviceAddr) &&
8514 I->getAssociatedExpression() == EncounteredME) ||
8515 (IsPrevMemberReference && !IsPointer) ||
8516 (IsMemberReference && Next != CE &&
8517 !Next->getAssociatedExpression()->getType()->isPointerType()));
8518 if (!OverlappedElements.empty() && Next == CE) {
8519 // Handle base element with the info for overlapped elements.
8520 assert(!PartialStruct.Base.isValid() && "The base element is set.");
8521 assert(!IsPointer &&
8522 "Unexpected base element with the pointer type.");
8523 // Mark the whole struct as the struct that requires allocation on the
8524 // device.
8525 PartialStruct.LowestElem = {0, LowestElem};
8526 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
8527 T: I->getAssociatedExpression()->getType());
8528 Address HB = CGF.Builder.CreateConstGEP(
8529 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8530 Addr: LowestElem, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty),
8531 Index: TypeSize.getQuantity() - 1);
8532 PartialStruct.HighestElem = {
8533 std::numeric_limits<decltype(
8534 PartialStruct.HighestElem.first)>::max(),
8535 HB};
8536 PartialStruct.Base = BP;
8537 PartialStruct.LB = LB;
8538 assert(
8539 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8540 "Overlapped elements must be used only once for the variable.");
8541 std::swap(a&: PartialStruct.PreliminaryMapData, b&: CombinedInfo);
8542 // Emit data for non-overlapped data.
8543 OpenMPOffloadMappingFlags Flags =
8544 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8545 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8546 /*AddPtrFlag=*/false,
8547 /*AddIsTargetParamFlag=*/false, IsNonContiguous);
8548 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8549 MapExpr, BP, LB, IsNonContiguous,
8550 DimSize);
8551 // Do bitcopy of all non-overlapped structure elements.
8552 for (OMPClauseMappableExprCommon::MappableExprComponentListRef
8553 Component : OverlappedElements) {
8554 for (const OMPClauseMappableExprCommon::MappableComponent &MC :
8555 Component) {
8556 if (const ValueDecl *VD = MC.getAssociatedDeclaration()) {
8557 if (const auto *FD = dyn_cast<FieldDecl>(Val: VD)) {
8558 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8559 }
8560 }
8561 }
8562 }
8563 CopyGaps.copyUntilEnd(HB);
8564 break;
8565 }
8566 llvm::Value *Size = getExprTypeSize(E: I->getAssociatedExpression());
8567 // Skip adding an entry in the CurInfo of this combined entry if the
8568 // whole struct is currently being mapped. The struct needs to be added
8569 // in the first position before any data internal to the struct is being
8570 // mapped.
8571 // Skip adding an entry in the CurInfo of this combined entry if the
8572 // PartialStruct.PreliminaryMapData.BasePointers has been mapped.
8573 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8574 (Next == CE && MapType != OMPC_MAP_unknown)) {
8575 if (!IsMappingWholeStruct) {
8576 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8577 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
8578 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8579 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8580 CombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8581 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8582 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8583 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize
8584 : 1);
8585 } else {
8586 StructBaseCombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8587 StructBaseCombinedInfo.BasePointers.push_back(
8588 Elt: BP.emitRawPointer(CGF));
8589 StructBaseCombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8590 StructBaseCombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8591 StructBaseCombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8592 StructBaseCombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8593 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8594 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8595 Elt: IsNonContiguous ? DimSize : 1);
8596 }
8597
8598 // If Mapper is valid, the last component inherits the mapper.
8599 bool HasMapper = Mapper && Next == CE;
8600 if (!IsMappingWholeStruct)
8601 CombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper : nullptr);
8602 else
8603 StructBaseCombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper
8604 : nullptr);
8605
8606 // We need to add a pointer flag for each map that comes from the
8607 // same expression except for the first one. We also need to signal
8608 // this map is the first one that relates with the current capture
8609 // (there is a set of entries for each capture).
8610 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
8611 MapType, MapModifiers, MotionModifiers, IsImplicit,
8612 AddPtrFlag: !IsExpressionFirstInfo || RequiresReference ||
8613 FirstPointerInComplexData || IsMemberReference,
8614 AddIsTargetParamFlag: IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
8615
8616 if (!IsExpressionFirstInfo || IsMemberReference) {
8617 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
8618 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
8619 if (IsPointer || (IsMemberReference && Next != CE))
8620 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8621 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8622 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8623 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8624 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8625
8626 if (ShouldBeMemberOf) {
8627 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
8628 // should be later updated with the correct value of MEMBER_OF.
8629 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8630 // From now on, all subsequent PTR_AND_OBJ entries should not be
8631 // marked as MEMBER_OF.
8632 ShouldBeMemberOf = false;
8633 }
8634 }
8635
8636 if (!IsMappingWholeStruct) {
8637 CombinedInfo.Types.push_back(Elt: Flags);
8638 // HasAttachPtr marks pointee entries, which have a base attach-ptr.
8639 CombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8640 } else {
8641 StructBaseCombinedInfo.Types.push_back(Elt: Flags);
8642 StructBaseCombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8643 }
8644 }
8645
8646 // If we have encountered a member expression so far, keep track of the
8647 // mapped member. If the parent is "*this", then the value declaration
8648 // is nullptr.
8649 if (EncounteredME) {
8650 const auto *FD = cast<FieldDecl>(Val: EncounteredME->getMemberDecl());
8651 unsigned FieldIndex = FD->getFieldIndex();
8652
8653 // Update info about the lowest and highest elements for this struct
8654 if (!PartialStruct.Base.isValid()) {
8655 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8656 if (IsFinalArraySection && OASE) {
8657 Address HB =
8658 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8659 .getAddress();
8660 PartialStruct.HighestElem = {FieldIndex, HB};
8661 } else {
8662 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8663 }
8664 PartialStruct.Base = BP;
8665 PartialStruct.LB = BP;
8666 } else if (FieldIndex < PartialStruct.LowestElem.first) {
8667 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8668 } else if (FieldIndex > PartialStruct.HighestElem.first) {
8669 if (IsFinalArraySection && OASE) {
8670 Address HB =
8671 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8672 .getAddress();
8673 PartialStruct.HighestElem = {FieldIndex, HB};
8674 } else {
8675 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8676 }
8677 }
8678 }
8679
8680 // Need to emit combined struct for array sections.
8681 if (IsFinalArraySection || IsNonContiguous)
8682 PartialStruct.IsArraySection = true;
8683
8684 // If we have a final array section, we are done with this expression.
8685 if (IsFinalArraySection)
8686 break;
8687
8688 // The pointer becomes the base for the next element.
8689 if (Next != CE)
8690 BP = IsMemberReference ? LowestElem : LB;
8691 if (!IsPartialMapped)
8692 IsExpressionFirstInfo = false;
8693 IsCaptureFirstInfo = false;
8694 FirstPointerInComplexData = false;
8695 IsPrevMemberReference = IsMemberReference;
8696 } else if (FirstPointerInComplexData) {
8697 QualType Ty = Components.rbegin()
8698 ->getAssociatedDeclaration()
8699 ->getType()
8700 .getNonReferenceType();
8701 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8702 FirstPointerInComplexData = false;
8703 }
8704 }
8705 // If ran into the whole component - allocate the space for the whole
8706 // record.
8707 if (!EncounteredME)
8708 PartialStruct.HasCompleteRecord = true;
8709
8710 // Populate ATTACH information for later processing by emitAttachEntry.
8711 if (shouldEmitAttachEntry(PointerExpr: AttachPtrExpr, MapBaseDecl: BaseDecl, CGF, CurDir)) {
8712 AttachInfo.AttachPtrAddr = AttachPtrAddr;
8713 AttachInfo.AttachPteeAddr = FinalLowestElem;
8714 AttachInfo.AttachPtrDecl = BaseDecl;
8715 AttachInfo.AttachMapExpr = MapExpr;
8716 }
8717
8718 if (!IsNonContiguous)
8719 return;
8720
8721 const ASTContext &Context = CGF.getContext();
8722
8723 // For supporting stride in array section, we need to initialize the first
8724 // dimension size as 1, first offset as 0, and first count as 1
8725 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 0)};
8726 MapValuesArrayTy CurCounts;
8727 MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8728 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8729 uint64_t ElementTypeSize;
8730
8731 // Collect Size information for each dimension and get the element size as
8732 // the first Stride. For example, for `int arr[10][10]`, the DimSizes
8733 // should be [10, 10] and the first stride is 4 btyes.
8734 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8735 Components) {
8736 const Expr *AssocExpr = Component.getAssociatedExpression();
8737 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8738
8739 if (!OASE)
8740 continue;
8741
8742 QualType Ty = ArraySectionExpr::getBaseOriginalType(Base: OASE->getBase());
8743 auto *CAT = Context.getAsConstantArrayType(T: Ty);
8744 auto *VAT = Context.getAsVariableArrayType(T: Ty);
8745
8746 // We need all the dimension size except for the last dimension.
8747 assert((VAT || CAT || &Component == &*Components.begin()) &&
8748 "Should be either ConstantArray or VariableArray if not the "
8749 "first Component");
8750
8751 // Get element size if CurCounts is empty.
8752 if (CurCounts.empty()) {
8753 const Type *ElementType = nullptr;
8754 if (CAT)
8755 ElementType = CAT->getElementType().getTypePtr();
8756 else if (VAT)
8757 ElementType = VAT->getElementType().getTypePtr();
8758 else if (&Component == &*Components.begin()) {
8759 // If the base is a raw pointer (e.g. T *data with data[a:b:c]),
8760 // there was no earlier CAT/VAT/array handling to establish
8761 // ElementType. Capture the pointee type now so that subsequent
8762 // components (offset/length/stride) have a concrete element type to
8763 // work with. This makes pointer-backed sections behave consistently
8764 // with CAT/VAT/array bases.
8765 if (const auto *PtrType = Ty->getAs<PointerType>())
8766 ElementType = PtrType->getPointeeType().getTypePtr();
8767 } else {
8768 // Any component after the first should never have a raw pointer type;
8769 // by this point. ElementType must already be known (set above or in
8770 // prior array / CAT / VAT handling).
8771 assert(!Ty->isPointerType() &&
8772 "Non-first components should not be raw pointers");
8773 }
8774
8775 // At this stage, if ElementType was a base pointer and we are in the
8776 // first iteration, it has been computed.
8777 if (ElementType) {
8778 // For the case that having pointer as base, we need to remove one
8779 // level of indirection.
8780 if (&Component != &*Components.begin())
8781 ElementType = ElementType->getPointeeOrArrayElementType();
8782 ElementTypeSize =
8783 Context.getTypeSizeInChars(T: ElementType).getQuantity();
8784 CurCounts.push_back(
8785 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: ElementTypeSize));
8786 }
8787 }
8788 // Get dimension value except for the last dimension since we don't need
8789 // it.
8790 if (DimSizes.size() < Components.size() - 1) {
8791 if (CAT)
8792 DimSizes.push_back(
8793 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: CAT->getZExtSize()));
8794 else if (VAT)
8795 DimSizes.push_back(Elt: CGF.Builder.CreateIntCast(
8796 V: CGF.EmitScalarExpr(E: VAT->getSizeExpr()), DestTy: CGF.Int64Ty,
8797 /*IsSigned=*/isSigned: false));
8798 }
8799 }
8800
8801 // Skip the dummy dimension since we have already have its information.
8802 auto *DI = DimSizes.begin() + 1;
8803 // Product of dimension.
8804 llvm::Value *DimProd =
8805 llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: ElementTypeSize);
8806
8807 // Collect info for non-contiguous. Notice that offset, count, and stride
8808 // are only meaningful for array-section, so we insert a null for anything
8809 // other than array-section.
8810 // Also, the size of offset, count, and stride are not the same as
8811 // pointers, base_pointers, sizes, or dims. Instead, the size of offset,
8812 // count, and stride are the same as the number of non-contiguous
8813 // declaration in target update to/from clause.
8814 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8815 Components) {
8816 const Expr *AssocExpr = Component.getAssociatedExpression();
8817
8818 if (const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr)) {
8819 llvm::Value *Offset = CGF.Builder.CreateIntCast(
8820 V: CGF.EmitScalarExpr(E: AE->getIdx()), DestTy: CGF.Int64Ty,
8821 /*isSigned=*/false);
8822 CurOffsets.push_back(Elt: Offset);
8823 CurCounts.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/1));
8824 CurStrides.push_back(Elt: CurStrides.back());
8825 continue;
8826 }
8827
8828 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8829
8830 if (!OASE)
8831 continue;
8832
8833 // Offset
8834 const Expr *OffsetExpr = OASE->getLowerBound();
8835 llvm::Value *Offset = nullptr;
8836 if (!OffsetExpr) {
8837 // If offset is absent, then we just set it to zero.
8838 Offset = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
8839 } else {
8840 Offset = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: OffsetExpr),
8841 DestTy: CGF.Int64Ty,
8842 /*isSigned=*/false);
8843 }
8844
8845 // Count
8846 const Expr *CountExpr = OASE->getLength();
8847 llvm::Value *Count = nullptr;
8848 if (!CountExpr) {
8849 // In Clang, once a high dimension is an array section, we construct all
8850 // the lower dimension as array section, however, for case like
8851 // arr[0:2][2], Clang construct the inner dimension as an array section
8852 // but it actually is not in an array section form according to spec.
8853 if (!OASE->getColonLocFirst().isValid() &&
8854 !OASE->getColonLocSecond().isValid()) {
8855 Count = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 1);
8856 } else {
8857 // OpenMP 5.0, 2.1.5 Array Sections, Description.
8858 // When the length is absent it defaults to ⌈(size −
8859 // lower-bound)/stride⌉, where size is the size of the array
8860 // dimension.
8861 const Expr *StrideExpr = OASE->getStride();
8862 llvm::Value *Stride =
8863 StrideExpr
8864 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8865 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8866 : nullptr;
8867 if (Stride)
8868 Count = CGF.Builder.CreateUDiv(
8869 LHS: CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset), RHS: Stride);
8870 else
8871 Count = CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset);
8872 }
8873 } else {
8874 Count = CGF.EmitScalarExpr(E: CountExpr);
8875 }
8876 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: CGF.Int64Ty, /*isSigned=*/false);
8877 CurCounts.push_back(Elt: Count);
8878
8879 // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size
8880 // Offset_n' = Offset_n * (D_0 * D_1 ... * D_n-1) * Unit size
8881 // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example:
8882 // Offset Count Stride
8883 // D0 0 4 1 (int) <- dummy dimension
8884 // D1 0 2 8 (2 * (1) * 4)
8885 // D2 100 2 20 (1 * (1 * 5) * 4)
8886 // D3 0 2 200 (2 * (1 * 5 * 4) * 4)
8887 const Expr *StrideExpr = OASE->getStride();
8888 llvm::Value *Stride =
8889 StrideExpr
8890 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8891 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8892 : nullptr;
8893 DimProd = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: *(DI - 1));
8894 if (Stride)
8895 CurStrides.push_back(Elt: CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Stride));
8896 else
8897 CurStrides.push_back(Elt: DimProd);
8898
8899 Offset = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Offset);
8900 CurOffsets.push_back(Elt: Offset);
8901
8902 if (DI != DimSizes.end())
8903 ++DI;
8904 }
8905
8906 CombinedInfo.NonContigInfo.Offsets.push_back(Elt: CurOffsets);
8907 CombinedInfo.NonContigInfo.Counts.push_back(Elt: CurCounts);
8908 CombinedInfo.NonContigInfo.Strides.push_back(Elt: CurStrides);
8909 }
8910
8911 /// Return the adjusted map modifiers if the declaration a capture refers to
8912 /// appears in a first-private clause. This is expected to be used only with
8913 /// directives that start with 'target'.
8914 OpenMPOffloadMappingFlags
8915 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
8916 assert(Cap.capturesVariable() && "Expected capture by reference only!");
8917
8918 // A first private variable captured by reference will use only the
8919 // 'private ptr' and 'map to' flag. Return the right flags if the captured
8920 // declaration is known as first-private in this handler.
8921 if (FirstPrivateDecls.count(Val: Cap.getCapturedVar())) {
8922 if (Cap.getCapturedVar()->getType()->isAnyPointerType())
8923 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8924 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8925 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8926 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8927 }
8928 auto I = LambdasMap.find(Val: Cap.getCapturedVar()->getCanonicalDecl());
8929 if (I != LambdasMap.end())
8930 // for map(to: lambda): using user specified map type.
8931 return getMapTypeBits(
8932 MapType: I->getSecond()->getMapType(), MapModifiers: I->getSecond()->getMapTypeModifiers(),
8933 /*MotionModifiers=*/{}, IsImplicit: I->getSecond()->isImplicit(),
8934 /*AddPtrFlag=*/false,
8935 /*AddIsTargetParamFlag=*/false,
8936 /*isNonContiguous=*/IsNonContiguous: false);
8937 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8938 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8939 }
8940
8941 void getPlainLayout(const CXXRecordDecl *RD,
8942 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8943 bool AsBase) const {
8944 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
8945
8946 llvm::StructType *St =
8947 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
8948
8949 unsigned NumElements = St->getNumElements();
8950 llvm::SmallVector<
8951 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8952 RecordLayout(NumElements);
8953
8954 // Fill bases.
8955 for (const auto &I : RD->bases()) {
8956 if (I.isVirtual())
8957 continue;
8958
8959 QualType BaseTy = I.getType();
8960 const auto *Base = BaseTy->getAsCXXRecordDecl();
8961 // Ignore empty bases.
8962 if (CodeGenUtils::isEmptyRecordForLayout(Ctx: CGF.getContext(), T: BaseTy) ||
8963 CGF.getContext()
8964 .getASTRecordLayout(D: Base)
8965 .getNonVirtualSize()
8966 .isZero())
8967 continue;
8968
8969 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(RD: Base);
8970 RecordLayout[FieldIndex] = Base;
8971 }
8972 // Fill in virtual bases.
8973 for (const auto &I : RD->vbases()) {
8974 QualType BaseTy = I.getType();
8975 // Ignore empty bases.
8976 if (CodeGenUtils::isEmptyRecordForLayout(Ctx: CGF.getContext(), T: BaseTy))
8977 continue;
8978
8979 const auto *Base = BaseTy->getAsCXXRecordDecl();
8980 unsigned FieldIndex = RL.getVirtualBaseIndex(base: Base);
8981 if (RecordLayout[FieldIndex])
8982 continue;
8983 RecordLayout[FieldIndex] = Base;
8984 }
8985 // Fill in all the fields.
8986 assert(!RD->isUnion() && "Unexpected union.");
8987 for (const auto *Field : RD->fields()) {
8988 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
8989 // will fill in later.)
8990 if (!Field->isBitField() &&
8991 !CodeGenUtils::isEmptyFieldForLayout(Ctx: CGF.getContext(), FD: Field)) {
8992 unsigned FieldIndex = RL.getLLVMFieldNo(FD: Field);
8993 RecordLayout[FieldIndex] = Field;
8994 }
8995 }
8996 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
8997 &Data : RecordLayout) {
8998 if (Data.isNull())
8999 continue;
9000 if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Val: Data))
9001 getPlainLayout(RD: Base, Layout, /*AsBase=*/true);
9002 else
9003 Layout.push_back(Elt: cast<const FieldDecl *>(Val: Data));
9004 }
9005 }
9006
9007 /// Returns the address corresponding to \p PointerExpr.
9008 static Address getAttachPtrAddr(const Expr *PointerExpr,
9009 CodeGenFunction &CGF) {
9010 assert(PointerExpr && "Cannot get addr from null attach-ptr expr");
9011 Address AttachPtrAddr = Address::invalid();
9012
9013 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: PointerExpr)) {
9014 // If the pointer is a variable, we can use its address directly.
9015 AttachPtrAddr = CGF.EmitLValue(E: DRE).getAddress();
9016 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(Val: PointerExpr)) {
9017 AttachPtrAddr =
9018 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/true).getAddress();
9019 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: PointerExpr)) {
9020 AttachPtrAddr = CGF.EmitLValue(E: ASE).getAddress();
9021 } else if (auto *ME = dyn_cast<MemberExpr>(Val: PointerExpr)) {
9022 AttachPtrAddr = CGF.EmitMemberExpr(E: ME).getAddress();
9023 } else if (auto *UO = dyn_cast<UnaryOperator>(Val: PointerExpr)) {
9024 assert(UO->getOpcode() == UO_Deref &&
9025 "Unexpected unary-operator on attach-ptr-expr");
9026 AttachPtrAddr = CGF.EmitLValue(E: UO).getAddress();
9027 }
9028 assert(AttachPtrAddr.isValid() &&
9029 "Failed to get address for attach pointer expression");
9030 return AttachPtrAddr;
9031 }
9032
9033 /// Get the address of the attach pointer, and a load from it, to get the
9034 /// pointee base address.
9035 /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair
9036 /// contains invalid addresses if \p AttachPtrExpr is null.
9037 static std::pair<Address, Address>
9038 getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr,
9039 CodeGenFunction &CGF) {
9040
9041 if (!AttachPtrExpr)
9042 return {Address::invalid(), Address::invalid()};
9043
9044 Address AttachPtrAddr = getAttachPtrAddr(PointerExpr: AttachPtrExpr, CGF);
9045 assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr");
9046
9047 QualType AttachPtrType =
9048 OMPClauseMappableExprCommon::getComponentExprElementType(Exp: AttachPtrExpr)
9049 .getCanonicalType();
9050
9051 Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer(
9052 Ptr: AttachPtrAddr, PtrTy: AttachPtrType->castAs<PointerType>());
9053 assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr");
9054
9055 return {AttachPtrAddr, AttachPteeBaseAddr};
9056 }
9057
9058 /// Returns whether an attach entry should be emitted for a map on
9059 /// \p MapBaseDecl on the directive \p CurDir.
9060 static bool
9061 shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl,
9062 CodeGenFunction &CGF,
9063 llvm::PointerUnion<const OMPExecutableDirective *,
9064 const OMPDeclareMapperDecl *>
9065 CurDir) {
9066 if (!PointerExpr)
9067 return false;
9068
9069 // Pointer attachment is needed at map-entering time or for declare
9070 // mappers.
9071 return isa<const OMPDeclareMapperDecl *>(Val: CurDir) ||
9072 isOpenMPTargetMapEnteringDirective(
9073 DKind: cast<const OMPExecutableDirective *>(Val&: CurDir)
9074 ->getDirectiveKind());
9075 }
9076
9077 /// Computes the attach-ptr expr for \p Components, and updates various maps
9078 /// with the information.
9079 /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr()
9080 /// with the OpenMPDirectiveKind extracted from \p CurDir.
9081 /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and
9082 /// AttachPtrExprMap.
9083 void collectAttachPtrExprInfo(
9084 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
9085 llvm::PointerUnion<const OMPExecutableDirective *,
9086 const OMPDeclareMapperDecl *>
9087 CurDir) {
9088
9089 OpenMPDirectiveKind CurDirectiveID =
9090 isa<const OMPDeclareMapperDecl *>(Val: CurDir)
9091 ? OMPD_declare_mapper
9092 : cast<const OMPExecutableDirective *>(Val&: CurDir)->getDirectiveKind();
9093
9094 const auto &[AttachPtrExpr, Depth] =
9095 OMPClauseMappableExprCommon::findAttachPtrExpr(Components,
9096 CurDirKind: CurDirectiveID);
9097
9098 AttachPtrComputationOrderMap.try_emplace(
9099 Key: AttachPtrExpr, Args: AttachPtrComputationOrderMap.size());
9100 AttachPtrComponentDepthMap.try_emplace(Key: AttachPtrExpr, Args: Depth);
9101 AttachPtrExprMap.try_emplace(Key: Components, Args: AttachPtrExpr);
9102 }
9103
9104 /// Generate all the base pointers, section pointers, sizes, map types, and
9105 /// mappers for the extracted mappable expressions (all included in \a
9106 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9107 /// pair of the relevant declaration and index where it occurs is appended to
9108 /// the device pointers info array.
9109 void generateAllInfoForClauses(
9110 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
9111 llvm::OpenMPIRBuilder &OMPBuilder,
9112 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9113 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9114 // We have to process the component lists that relate with the same
9115 // declaration in a single chunk so that we can generate the map flags
9116 // correctly. Therefore, we organize all lists in a map.
9117 enum MapKind { Present, Allocs, Other, Total };
9118 llvm::MapVector<CanonicalDeclPtr<const Decl>,
9119 SmallVector<SmallVector<MapInfo, 8>, 4>>
9120 Info;
9121
9122 // Helper function to fill the information map for the different supported
9123 // clauses.
9124 auto &&InfoGen =
9125 [&Info, &SkipVarSet](
9126 const ValueDecl *D, MapKind Kind,
9127 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
9128 OpenMPMapClauseKind MapType,
9129 ArrayRef<OpenMPMapModifierKind> MapModifiers,
9130 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
9131 bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,
9132 const Expr *VarRef = nullptr, bool ForDeviceAddr = false) {
9133 if (SkipVarSet.contains(V: D))
9134 return;
9135 auto It = Info.try_emplace(Key: D, Args: Total).first;
9136 It->second[Kind].emplace_back(
9137 Args&: L, Args&: MapType, Args&: MapModifiers, Args&: MotionModifiers, Args&: ReturnDevicePointer,
9138 Args&: IsImplicit, Args&: Mapper, Args&: VarRef, Args&: ForDeviceAddr);
9139 };
9140
9141 for (const auto *Cl : Clauses) {
9142 const auto *C = dyn_cast<OMPMapClause>(Val: Cl);
9143 if (!C)
9144 continue;
9145 MapKind Kind = Other;
9146 if (llvm::is_contained(Range: C->getMapTypeModifiers(),
9147 Element: OMPC_MAP_MODIFIER_present))
9148 Kind = Present;
9149 else if (C->getMapType() == OMPC_MAP_alloc)
9150 Kind = Allocs;
9151 const auto *EI = C->getVarRefs().begin();
9152 for (const auto L : C->component_lists()) {
9153 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
9154 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), C->getMapType(),
9155 C->getMapTypeModifiers(), {},
9156 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
9157 E);
9158 ++EI;
9159 }
9160 }
9161 for (const auto *Cl : Clauses) {
9162 const auto *C = dyn_cast<OMPToClause>(Val: Cl);
9163 if (!C)
9164 continue;
9165 MapKind Kind = Other;
9166 if (llvm::is_contained(Range: C->getMotionModifiers(),
9167 Element: OMPC_MOTION_MODIFIER_present))
9168 Kind = Present;
9169 if (llvm::is_contained(Range: C->getMotionModifiers(),
9170 Element: OMPC_MOTION_MODIFIER_iterator)) {
9171 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9172 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
9173 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
9174 CGF.EmitVarDecl(D: *VD);
9175 }
9176 }
9177
9178 const auto *EI = C->getVarRefs().begin();
9179 for (const auto L : C->component_lists()) {
9180 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_to, {},
9181 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
9182 C->isImplicit(), std::get<2>(t: L), *EI);
9183 ++EI;
9184 }
9185 }
9186 for (const auto *Cl : Clauses) {
9187 const auto *C = dyn_cast<OMPFromClause>(Val: Cl);
9188 if (!C)
9189 continue;
9190 MapKind Kind = Other;
9191 if (llvm::is_contained(Range: C->getMotionModifiers(),
9192 Element: OMPC_MOTION_MODIFIER_present))
9193 Kind = Present;
9194 if (llvm::is_contained(Range: C->getMotionModifiers(),
9195 Element: OMPC_MOTION_MODIFIER_iterator)) {
9196 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9197 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
9198 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
9199 CGF.EmitVarDecl(D: *VD);
9200 }
9201 }
9202
9203 const auto *EI = C->getVarRefs().begin();
9204 for (const auto L : C->component_lists()) {
9205 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_from, {},
9206 C->getMotionModifiers(),
9207 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
9208 *EI);
9209 ++EI;
9210 }
9211 }
9212
9213 // Look at the use_device_ptr and use_device_addr clauses information and
9214 // mark the existing map entries as such. If there is no map information for
9215 // an entry in the use_device_ptr and use_device_addr list, we create one
9216 // with map type 'return_param' and zero size section. It is the user's
9217 // fault if that was not mapped before. If there is no map information, then
9218 // we defer the emission of that entry until all the maps for the same VD
9219 // have been handled.
9220 MapCombinedInfoTy UseDeviceDataCombinedInfo;
9221
9222 auto &&UseDeviceDataCombinedInfoGen =
9223 [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,
9224 CodeGenFunction &CGF, bool IsDevAddr,
9225 bool HasUdpFbNullify = false) {
9226 UseDeviceDataCombinedInfo.Exprs.push_back(Elt: VD);
9227 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Args&: Ptr);
9228 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(Args&: VD);
9229 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
9230 Args: IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
9231 // FIXME: For use_device_addr on array-sections, this should
9232 // be the starting address of the section.
9233 // e.g. int *p;
9234 // ... use_device_addr(p[3])
9235 // &p[0], &p[3], /*size=*/0, RETURN_PARAM
9236 UseDeviceDataCombinedInfo.Pointers.push_back(Elt: Ptr);
9237 UseDeviceDataCombinedInfo.Sizes.push_back(
9238 Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
9239 OpenMPOffloadMappingFlags Flags =
9240 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9241 if (HasUdpFbNullify)
9242 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9243 UseDeviceDataCombinedInfo.Types.push_back(Elt: Flags);
9244 UseDeviceDataCombinedInfo.HasAttachPtr.push_back(Elt: false);
9245 UseDeviceDataCombinedInfo.Mappers.push_back(Elt: nullptr);
9246 };
9247
9248 auto &&MapInfoGen =
9249 [&UseDeviceDataCombinedInfoGen](
9250 CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
9251 OMPClauseMappableExprCommon::MappableExprComponentListRef
9252 Components,
9253 bool IsDevAddr, bool IEIsAttachPtrForDevAddr = false,
9254 bool HasUdpFbNullify = false) {
9255 // We didn't find any match in our map information - generate a zero
9256 // size array section.
9257 llvm::Value *Ptr;
9258 if (IsDevAddr && !IEIsAttachPtrForDevAddr) {
9259 if (IE->isGLValue())
9260 Ptr = CGF.EmitLValue(E: IE).getPointer(CGF);
9261 else
9262 Ptr = CGF.EmitScalarExpr(E: IE);
9263 } else {
9264 Ptr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: IE), Loc: IE->getExprLoc());
9265 }
9266 bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr;
9267 // For the purpose of address-translation, treat something like the
9268 // following:
9269 // int *p;
9270 // ... use_device_addr(p[1])
9271 // equivalent to
9272 // ... use_device_ptr(p)
9273 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, /*IsDevAddr=*/IsDevAddr &&
9274 !TreatDevAddrAsDevPtr,
9275 HasUdpFbNullify);
9276 };
9277
9278 auto &&IsMapInfoExist =
9279 [&Info, this](CodeGenFunction &CGF, const ValueDecl *VD, const Expr *IE,
9280 const Expr *DesiredAttachPtrExpr, bool IsDevAddr,
9281 bool HasUdpFbNullify = false) -> bool {
9282 // We potentially have map information for this declaration already.
9283 // Look for the first set of components that refer to it. If found,
9284 // return true.
9285 // If the first component is a member expression, we have to look into
9286 // 'this', which maps to null in the map of map information. Otherwise
9287 // look directly for the information.
9288 auto It = Info.find(Key: isa<MemberExpr>(Val: IE) ? nullptr : VD);
9289 if (It != Info.end()) {
9290 bool Found = false;
9291 for (auto &Data : It->second) {
9292 MapInfo *CI = nullptr;
9293 // We potentially have multiple maps for the same decl. We need to
9294 // only consider those for which the attach-ptr matches the desired
9295 // attach-ptr.
9296 auto *It = llvm::find_if(Range&: Data, P: [&](const MapInfo &MI) {
9297 if (MI.Components.back().getAssociatedDeclaration() != VD)
9298 return false;
9299
9300 const Expr *MapAttachPtr = getAttachPtrExpr(Components: MI.Components);
9301 bool Match = AttachPtrComparator.areEqual(LHS: MapAttachPtr,
9302 RHS: DesiredAttachPtrExpr);
9303 return Match;
9304 });
9305
9306 if (It != Data.end())
9307 CI = &*It;
9308
9309 if (CI) {
9310 if (IsDevAddr) {
9311 CI->ForDeviceAddr = true;
9312 CI->ReturnDevicePointer = true;
9313 CI->HasUdpFbNullify = HasUdpFbNullify;
9314 Found = true;
9315 break;
9316 } else {
9317 auto PrevCI = std::next(x: CI->Components.rbegin());
9318 const auto *VarD = dyn_cast<VarDecl>(Val: VD);
9319 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: CI->Components);
9320 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
9321 isa<MemberExpr>(Val: IE) ||
9322 !VD->getType().getNonReferenceType()->isPointerType() ||
9323 PrevCI == CI->Components.rend() ||
9324 isa<MemberExpr>(Val: PrevCI->getAssociatedExpression()) || !VarD ||
9325 VarD->hasLocalStorage() ||
9326 (isa_and_nonnull<DeclRefExpr>(Val: AttachPtrExpr) &&
9327 VD == cast<DeclRefExpr>(Val: AttachPtrExpr)->getDecl())) {
9328 CI->ForDeviceAddr = IsDevAddr;
9329 CI->ReturnDevicePointer = true;
9330 CI->HasUdpFbNullify = HasUdpFbNullify;
9331 Found = true;
9332 break;
9333 }
9334 }
9335 }
9336 }
9337 return Found;
9338 }
9339 return false;
9340 };
9341
9342 // Look at the use_device_ptr clause information and mark the existing map
9343 // entries as such. If there is no map information for an entry in the
9344 // use_device_ptr list, we create one with map type 'alloc' and zero size
9345 // section. It is the user fault if that was not mapped before. If there is
9346 // no map information and the pointer is a struct member, then we defer the
9347 // emission of that entry until the whole struct has been processed.
9348 for (const auto *Cl : Clauses) {
9349 const auto *C = dyn_cast<OMPUseDevicePtrClause>(Val: Cl);
9350 if (!C)
9351 continue;
9352 bool HasUdpFbNullify =
9353 C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify;
9354 for (const auto L : C->component_lists()) {
9355 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9356 std::get<1>(t: L);
9357 assert(!Components.empty() &&
9358 "Not expecting empty list of components!");
9359 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
9360 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9361 const Expr *IE = Components.back().getAssociatedExpression();
9362 // For use_device_ptr, we match an existing map clause if its attach-ptr
9363 // is same as the use_device_ptr operand. e.g.
9364 // map expr | use_device_ptr expr | current behavior
9365 // ---------|---------------------|-----------------
9366 // p[1] | p | match
9367 // ps->a | ps | match
9368 // p | p | no match
9369 const Expr *UDPOperandExpr =
9370 Components.front().getAssociatedExpression();
9371 if (IsMapInfoExist(CGF, VD, IE,
9372 /*DesiredAttachPtrExpr=*/UDPOperandExpr,
9373 /*IsDevAddr=*/false, HasUdpFbNullify))
9374 continue;
9375 MapInfoGen(CGF, IE, VD, Components, /*IsDevAddr=*/false,
9376 /*IEIsAttachPtrForDevAddr=*/false, HasUdpFbNullify);
9377 }
9378 }
9379
9380 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
9381 for (const auto *Cl : Clauses) {
9382 const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Val: Cl);
9383 if (!C)
9384 continue;
9385 for (const auto L : C->component_lists()) {
9386 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9387 std::get<1>(t: L);
9388 assert(!std::get<1>(L).empty() &&
9389 "Not expecting empty list of components!");
9390 const ValueDecl *VD = std::get<1>(t: L).back().getAssociatedDeclaration();
9391 if (!Processed.insert(V: VD).second)
9392 continue;
9393 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9394 // For use_device_addr, we match an existing map clause if the
9395 // use_device_addr operand's attach-ptr matches the map operand's
9396 // attach-ptr.
9397 // We chould also restrict to only match cases when there is a full
9398 // match between the map/use_device_addr clause exprs, but that may be
9399 // unnecessary.
9400 //
9401 // map expr | use_device_addr expr | current | possible restrictive/
9402 // | | behavior | safer behavior
9403 // ---------|----------------------|-----------|-----------------------
9404 // p | p | match | match
9405 // p[0] | p[0] | match | match
9406 // p[0:1] | p[0] | match | no match
9407 // p[0:1] | p[2:1] | match | no match
9408 // p[1] | p[0] | match | no match
9409 // ps->a | ps->b | match | no match
9410 // p | p[0] | no match | no match
9411 // pp | pp[0][0] | no match | no match
9412 const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components);
9413 const Expr *IE = std::get<1>(t: L).back().getAssociatedExpression();
9414 assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) &&
9415 "use_device_addr operand has an attach-ptr, but does not match "
9416 "last component's expr.");
9417 if (IsMapInfoExist(CGF, VD, IE,
9418 /*DesiredAttachPtrExpr=*/UDAAttachPtrExpr,
9419 /*IsDevAddr=*/true))
9420 continue;
9421 MapInfoGen(CGF, IE, VD, Components,
9422 /*IsDevAddr=*/true,
9423 /*IEIsAttachPtrForDevAddr=*/UDAAttachPtrExpr != nullptr);
9424 }
9425 }
9426
9427 for (const auto &Data : Info) {
9428 MapCombinedInfoTy CurInfo;
9429 const Decl *D = Data.first;
9430 const ValueDecl *VD = cast_or_null<ValueDecl>(Val: D);
9431 // Group component lists by their AttachPtrExpr and process them in order
9432 // of increasing complexity (nullptr first, then simple expressions like
9433 // p, then more complex ones like p[0], etc.)
9434 //
9435 // This is similar to how generateInfoForCaptureFromClauseInfo handles
9436 // grouping for target constructs.
9437 SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs;
9438
9439 // First, collect all MapData entries with their attach-ptr exprs.
9440 for (const auto &M : Data.second) {
9441 for (const MapInfo &L : M) {
9442 assert(!L.Components.empty() &&
9443 "Not expecting declaration with no component lists.");
9444
9445 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: L.Components);
9446 AttachPtrMapInfoPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
9447 }
9448 }
9449
9450 // Next, sort by increasing order of their complexity.
9451 llvm::stable_sort(Range&: AttachPtrMapInfoPairs,
9452 C: [this](const auto &LHS, const auto &RHS) {
9453 return AttachPtrComparator(LHS.first, RHS.first);
9454 });
9455
9456 // And finally, process them all in order, grouping those with
9457 // equivalent attach-ptr exprs together.
9458 auto *It = AttachPtrMapInfoPairs.begin();
9459 while (It != AttachPtrMapInfoPairs.end()) {
9460 const Expr *AttachPtrExpr = It->first;
9461
9462 SmallVector<MapInfo, 8> GroupLists;
9463 while (It != AttachPtrMapInfoPairs.end() &&
9464 (It->first == AttachPtrExpr ||
9465 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
9466 GroupLists.push_back(Elt: It->second);
9467 ++It;
9468 }
9469 assert(!GroupLists.empty() && "GroupLists should not be empty");
9470
9471 StructRangeInfoTy PartialStruct;
9472 AttachInfoTy AttachInfo;
9473 MapCombinedInfoTy GroupCurInfo;
9474 // Current group's struct base information:
9475 MapCombinedInfoTy GroupStructBaseCurInfo;
9476 for (const MapInfo &L : GroupLists) {
9477 // Remember the current base pointer index.
9478 unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size();
9479 unsigned StructBasePointersIdx =
9480 GroupStructBaseCurInfo.BasePointers.size();
9481
9482 GroupCurInfo.NonContigInfo.IsNonContiguous =
9483 L.Components.back().isNonContiguous();
9484 generateInfoForComponentList(
9485 MapType: L.MapType, MapModifiers: L.MapModifiers, MotionModifiers: L.MotionModifiers, Components: L.Components,
9486 CombinedInfo&: GroupCurInfo, StructBaseCombinedInfo&: GroupStructBaseCurInfo, PartialStruct, AttachInfo,
9487 /*IsFirstComponentList=*/false, IsImplicit: L.IsImplicit,
9488 /*GenerateAllInfoForClauses*/ true, Mapper: L.Mapper, ForDeviceAddr: L.ForDeviceAddr, BaseDecl: VD,
9489 MapExpr: L.VarRef, /*OverlappedElements*/ {});
9490
9491 // If this entry relates to a device pointer, set the relevant
9492 // declaration and add the 'return pointer' flag.
9493 if (L.ReturnDevicePointer) {
9494 // Check whether a value was added to either GroupCurInfo or
9495 // GroupStructBaseCurInfo and error if no value was added to either
9496 // of them:
9497 assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() ||
9498 StructBasePointersIdx <
9499 GroupStructBaseCurInfo.BasePointers.size()) &&
9500 "Unexpected number of mapped base pointers.");
9501
9502 // Choose a base pointer index which is always valid:
9503 const ValueDecl *RelevantVD =
9504 L.Components.back().getAssociatedDeclaration();
9505 assert(RelevantVD &&
9506 "No relevant declaration related with device pointer??");
9507
9508 // If GroupStructBaseCurInfo has been updated this iteration then
9509 // work on the first new entry added to it i.e. make sure that when
9510 // multiple values are added to any of the lists, the first value
9511 // added is being modified by the assignments below (not the last
9512 // value added).
9513 auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info,
9514 unsigned Idx) {
9515 Info.DevicePtrDecls[Idx] = RelevantVD;
9516 Info.DevicePointers[Idx] = L.ForDeviceAddr
9517 ? DeviceInfoTy::Address
9518 : DeviceInfoTy::Pointer;
9519 Info.Types[Idx] |=
9520 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9521 if (L.HasUdpFbNullify)
9522 Info.Types[Idx] |=
9523 OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9524 };
9525
9526 if (StructBasePointersIdx <
9527 GroupStructBaseCurInfo.BasePointers.size())
9528 SetDevicePointerInfo(GroupStructBaseCurInfo,
9529 StructBasePointersIdx);
9530 else
9531 SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx);
9532 }
9533 }
9534
9535 // Unify entries in one list making sure the struct mapping precedes the
9536 // individual fields:
9537 MapCombinedInfoTy GroupUnionCurInfo;
9538 GroupUnionCurInfo.append(CurInfo&: GroupStructBaseCurInfo);
9539 GroupUnionCurInfo.append(CurInfo&: GroupCurInfo);
9540
9541 // If there is an entry in PartialStruct it means we have a struct with
9542 // individual members mapped. Emit an extra combined entry.
9543 if (PartialStruct.Base.isValid()) {
9544 // Prepend a synthetic dimension of length 1 to represent the
9545 // aggregated struct object. Using 1 (not 0, as 0 produced an
9546 // incorrect non-contiguous descriptor (DimSize==1), causing the
9547 // non-contiguous motion clause path to be skipped.) is important:
9548 // * It preserves the correct rank so targetDataUpdate() computes
9549 // DimSize == 2 for cases like strided array sections originating
9550 // from user-defined mappers (e.g. test with s.data[0:8:2]).
9551 GroupUnionCurInfo.NonContigInfo.Dims.insert(
9552 I: GroupUnionCurInfo.NonContigInfo.Dims.begin(), Elt: 1);
9553 emitCombinedEntry(
9554 CombinedInfo&: CurInfo, CurTypes&: GroupUnionCurInfo.Types, PartialStruct, AttachInfo,
9555 /*IsMapThis=*/!VD, OMPBuilder, VD,
9556 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size(),
9557 /*NotTargetParams=*/true);
9558 }
9559
9560 // Append this group's results to the overall CurInfo in the correct
9561 // order: combined-entry -> original-field-entries -> attach-entry
9562 CurInfo.append(CurInfo&: GroupUnionCurInfo);
9563 if (AttachInfo.isValid())
9564 emitAttachEntry(CGF, CombinedInfo&: CurInfo, AttachInfo);
9565 }
9566
9567 // We need to append the results of this capture to what we already have.
9568 CombinedInfo.append(CurInfo);
9569 }
9570 // Append data for use_device_ptr/addr clauses.
9571 CombinedInfo.append(CurInfo&: UseDeviceDataCombinedInfo);
9572 }
9573
9574public:
9575 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
9576 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9577 // Extract firstprivate clause information.
9578 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
9579 for (const auto *D : C->varlist()) {
9580 const ValueDecl *VD = cast<DeclRefExpr>(Val: D)->getDecl();
9581 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
9582 VD = cast<VarDecl>(Val: BD->getDecomposedDecl());
9583 FirstPrivateDecls.try_emplace(Key: cast<VarDecl>(Val: VD), Args: C->isImplicit());
9584 }
9585 // Extract implicit firstprivates from uses_allocators clauses.
9586 for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
9587 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
9588 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
9589 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: D.AllocatorTraits))
9590 FirstPrivateDecls.try_emplace(Key: cast<VarDecl>(Val: DRE->getDecl()),
9591 /*Implicit=*/Args: true);
9592 else if (const auto *VD = dyn_cast<VarDecl>(
9593 Val: cast<DeclRefExpr>(Val: D.Allocator->IgnoreParenImpCasts())
9594 ->getDecl()))
9595 FirstPrivateDecls.try_emplace(Key: VD, /*Implicit=*/Args: true);
9596 }
9597 }
9598 // Extract defaultmap clause information.
9599 for (const auto *C : Dir.getClausesOfKind<OMPDefaultmapClause>())
9600 if (C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate)
9601 DefaultmapFirstprivateKinds.insert(V: C->getDefaultmapKind());
9602 // Extract device pointer clause information.
9603 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9604 for (auto L : C->component_lists())
9605 DevPointersMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9606 // Extract device addr clause information.
9607 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9608 for (auto L : C->component_lists())
9609 HasDevAddrsMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9610 // Extract map information.
9611 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) {
9612 if (C->getMapType() != OMPC_MAP_to)
9613 continue;
9614 for (auto L : C->component_lists()) {
9615 const ValueDecl *VD = std::get<0>(t&: L);
9616 const auto *RD = VD ? VD->getType()
9617 .getCanonicalType()
9618 .getNonReferenceType()
9619 ->getAsCXXRecordDecl()
9620 : nullptr;
9621 if (RD && RD->isLambda())
9622 LambdasMap.try_emplace(Key: std::get<0>(t&: L), Args&: C);
9623 }
9624 }
9625
9626 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9627 for (auto L : C->component_lists()) {
9628 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9629 std::get<1>(L);
9630 if (!Components.empty())
9631 collectAttachPtrExprInfo(Components, CurDir);
9632 }
9633 };
9634
9635 // Populate the AttachPtrExprMap for all component lists from map-related
9636 // clauses.
9637 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>())
9638 CollectAttachPtrExprsForClauseComponents(C);
9639 for (const auto *C : Dir.getClausesOfKind<OMPToClause>())
9640 CollectAttachPtrExprsForClauseComponents(C);
9641 for (const auto *C : Dir.getClausesOfKind<OMPFromClause>())
9642 CollectAttachPtrExprsForClauseComponents(C);
9643 for (const auto *C : Dir.getClausesOfKind<OMPUseDevicePtrClause>())
9644 CollectAttachPtrExprsForClauseComponents(C);
9645 for (const auto *C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>())
9646 CollectAttachPtrExprsForClauseComponents(C);
9647 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9648 CollectAttachPtrExprsForClauseComponents(C);
9649 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9650 CollectAttachPtrExprsForClauseComponents(C);
9651 }
9652
9653 /// Constructor for the declare mapper directive.
9654 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
9655 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9656 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9657 for (auto L : C->component_lists()) {
9658 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9659 std::get<1>(L);
9660 if (!Components.empty())
9661 collectAttachPtrExprInfo(Components, CurDir);
9662 }
9663 };
9664
9665 // Populate the AttachPtrExprMap for all component lists from map-related
9666 // clauses in the declare mapper directive, to enable attach-style mapping
9667 // for mappers.
9668 for (const auto *Cl : Dir.clauses()) {
9669 if (const auto *C = dyn_cast<OMPMapClause>(Val: Cl))
9670 CollectAttachPtrExprsForClauseComponents(C);
9671 else if (const auto *C = dyn_cast<OMPToClause>(Val: Cl))
9672 CollectAttachPtrExprsForClauseComponents(C);
9673 else if (const auto *C = dyn_cast<OMPFromClause>(Val: Cl))
9674 CollectAttachPtrExprsForClauseComponents(C);
9675 }
9676 }
9677
9678 /// Generate code for the combined entry if we have a partially mapped struct
9679 /// and take care of the mapping flags of the arguments corresponding to
9680 /// individual struct members.
9681 /// If a valid \p AttachInfo exists, its pointee addr will be updated to point
9682 /// to the combined-entry's begin address, if emitted.
9683 /// \p PartialStruct contains attach base-pointer information.
9684 /// \returns The index of the combined entry if one was added, std::nullopt
9685 /// otherwise.
9686 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9687 MapFlagsArrayTy &CurTypes,
9688 const StructRangeInfoTy &PartialStruct,
9689 AttachInfoTy &AttachInfo, bool IsMapThis,
9690 llvm::OpenMPIRBuilder &OMPBuilder, const ValueDecl *VD,
9691 unsigned OffsetForMemberOfFlag,
9692 bool NotTargetParams) const {
9693 if (CurTypes.size() == 1 &&
9694 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9695 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9696 !PartialStruct.IsArraySection)
9697 return;
9698 Address LBAddr = PartialStruct.LowestElem.second;
9699 Address HBAddr = PartialStruct.HighestElem.second;
9700 if (PartialStruct.HasCompleteRecord) {
9701 LBAddr = PartialStruct.LB;
9702 HBAddr = PartialStruct.LB;
9703 }
9704 CombinedInfo.Exprs.push_back(Elt: VD);
9705 // Base is the base of the struct
9706 CombinedInfo.BasePointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9707 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9708 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9709 // Pointer is the address of the lowest element
9710 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
9711 const CXXMethodDecl *MD =
9712 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(Val: CGF.CurFuncDecl) : nullptr;
9713 const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
9714 bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false;
9715 // There should not be a mapper for a combined entry.
9716 if (HasBaseClass) {
9717 // OpenMP 5.2 148:21:
9718 // If the target construct is within a class non-static member function,
9719 // and a variable is an accessible data member of the object for which the
9720 // non-static data member function is invoked, the variable is treated as
9721 // if the this[:1] expression had appeared in a map clause with a map-type
9722 // of tofrom.
9723 // Emit this[:1]
9724 CombinedInfo.Pointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9725 QualType Ty = MD->getFunctionObjectParameterType();
9726 llvm::Value *Size =
9727 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty), DestTy: CGF.Int64Ty,
9728 /*isSigned=*/true);
9729 CombinedInfo.Sizes.push_back(Elt: Size);
9730 } else {
9731 CombinedInfo.Pointers.push_back(Elt: LB);
9732 // Size is (addr of {highest+1} element) - (addr of lowest element)
9733 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
9734 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
9735 Ty: HBAddr.getElementType(), Ptr: HB, /*Idx0=*/1);
9736 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(V: LB, DestTy: CGF.VoidPtrTy);
9737 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(V: HAddr, DestTy: CGF.VoidPtrTy);
9738 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(LHS: CHAddr, RHS: CLAddr);
9739 llvm::Value *Size = CGF.Builder.CreateIntCast(V: Diff, DestTy: CGF.Int64Ty,
9740 /*isSigned=*/false);
9741 CombinedInfo.Sizes.push_back(Elt: Size);
9742 }
9743 CombinedInfo.Mappers.push_back(Elt: nullptr);
9744 // Map type is always TARGET_PARAM, if generate info for captures.
9745 CombinedInfo.Types.push_back(
9746 Elt: NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9747 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9748 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9749 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9750 // A combined entry has a base attach-ptr if its constituents do. e.g.:
9751 // map(s2.s1p->x, s2.s1p->y)
9752 // combined entry:
9753 // s2.s1p[0], s2.s1p->x, sizeof(s1p->x..y), ALLOC
9754 // here s2.s1p is the attach-ptr for the combined entry.
9755 // See the inline comments in emitUserDefinedMapper's definition for how
9756 // entries with an attach-ptr are treated.
9757 CombinedInfo.HasAttachPtr.push_back(Elt: AttachInfo.isValid());
9758 // If any element has the present modifier, then make sure the runtime
9759 // doesn't attempt to allocate the struct.
9760 if (CurTypes.end() !=
9761 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9762 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9763 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9764 }))
9765 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9766 // Remove TARGET_PARAM flag from the first element
9767 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9768 // If any element has the ompx_hold modifier, then make sure the runtime
9769 // uses the hold reference count for the struct as a whole so that it won't
9770 // be unmapped by an extra dynamic reference count decrement. Add it to all
9771 // elements as well so the runtime knows which reference count to check
9772 // when determining whether it's time for device-to-host transfers of
9773 // individual elements.
9774 if (CurTypes.end() !=
9775 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9776 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9777 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9778 })) {
9779 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9780 for (auto &M : CurTypes)
9781 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9782 }
9783
9784 // All other current entries will be MEMBER_OF the combined entry
9785 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9786 // 0xFFFF in the MEMBER_OF field, or ATTACH entries since they are expected
9787 // to be handled by themselves, after all other maps).
9788 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9789 Position: OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9790 for (auto &M : CurTypes)
9791 OMPBuilder.setCorrectMemberOfFlag(Flags&: M, MemberOfFlag);
9792
9793 // When we are emitting a combined entry. If there were any pending
9794 // attachments to be done, we do them to the begin address of the combined
9795 // entry. Note that this means only one attachment per combined-entry will
9796 // be done. So, for instance, if we have:
9797 // S *ps;
9798 // ... map(ps->a, ps->b)
9799 // When we are emitting a combined entry. If AttachInfo is valid,
9800 // update the pointee address to point to the begin address of the combined
9801 // entry. This ensures that if we have multiple maps like:
9802 // `map(ps->a, ps->b)`, we still get a single ATTACH entry, like:
9803 //
9804 // &ps[0], &ps->a, sizeof(ps->a to ps->b), ALLOC // combined-entry
9805 // &ps[0], &ps->a, sizeof(ps->a), TO | FROM
9806 // &ps[0], &ps->b, sizeof(ps->b), TO | FROM
9807 // &ps, &ps->a, sizeof(void*), ATTACH // Use combined-entry's LB
9808 if (AttachInfo.isValid())
9809 AttachInfo.AttachPteeAddr = LBAddr;
9810 }
9811
9812 /// Generate all the base pointers, section pointers, sizes, map types, and
9813 /// mappers for the extracted mappable expressions (all included in \a
9814 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9815 /// pair of the relevant declaration and index where it occurs is appended to
9816 /// the device pointers info array.
9817 void generateAllInfo(
9818 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9819 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9820 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9821 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9822 "Expect a executable directive");
9823 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9824 generateAllInfoForClauses(Clauses: CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9825 SkipVarSet);
9826 }
9827
9828 /// Generate all the base pointers, section pointers, sizes, map types, and
9829 /// mappers for the extracted map clauses of user-defined mapper (all included
9830 /// in \a CombinedInfo).
9831 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9832 llvm::OpenMPIRBuilder &OMPBuilder) const {
9833 assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&
9834 "Expect a declare mapper directive");
9835 const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(Val: CurDir);
9836 generateAllInfoForClauses(Clauses: CurMapperDir->clauses(), CombinedInfo,
9837 OMPBuilder);
9838 }
9839
9840 /// Emit capture info for lambdas for variables captured by reference.
9841 void generateInfoForLambdaCaptures(
9842 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9843 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
9844 QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();
9845 const auto *RD = VDType->getAsCXXRecordDecl();
9846 if (!RD || !RD->isLambda())
9847 return;
9848 Address VDAddr(Arg, CGF.ConvertTypeForMem(T: VDType),
9849 CGF.getContext().getDeclAlign(D: VD));
9850 LValue VDLVal = CGF.MakeAddrLValue(Addr: VDAddr, T: VDType);
9851 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9852 FieldDecl *ThisCapture = nullptr;
9853 RD->getCaptureFields(Captures, ThisCapture);
9854 if (ThisCapture) {
9855 LValue ThisLVal =
9856 CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: ThisCapture);
9857 LValue ThisLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: ThisCapture);
9858 LambdaPointers.try_emplace(Key: ThisLVal.getPointer(CGF),
9859 Args: VDLVal.getPointer(CGF));
9860 CombinedInfo.Exprs.push_back(Elt: VD);
9861 CombinedInfo.BasePointers.push_back(Elt: ThisLVal.getPointer(CGF));
9862 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9863 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9864 CombinedInfo.Pointers.push_back(Elt: ThisLValVal.getPointer(CGF));
9865 CombinedInfo.Sizes.push_back(
9866 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy),
9867 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9868 CombinedInfo.Types.push_back(
9869 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9870 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9871 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9872 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9873 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9874 CombinedInfo.Mappers.push_back(Elt: nullptr);
9875 }
9876 for (const LambdaCapture &LC : RD->captures()) {
9877 if (!LC.capturesVariable())
9878 continue;
9879 const VarDecl *VD = cast<VarDecl>(Val: LC.getCapturedVar());
9880 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
9881 continue;
9882 auto It = Captures.find(Val: VD);
9883 assert(It != Captures.end() && "Found lambda capture without field.");
9884 LValue VarLVal = CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: It->second);
9885 if (LC.getCaptureKind() == LCK_ByRef) {
9886 LValue VarLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: It->second);
9887 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9888 Args: VDLVal.getPointer(CGF));
9889 CombinedInfo.Exprs.push_back(Elt: VD);
9890 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9891 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9892 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9893 CombinedInfo.Pointers.push_back(Elt: VarLValVal.getPointer(CGF));
9894 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9895 V: CGF.getTypeSize(
9896 Ty: VD->getType().getCanonicalType().getNonReferenceType()),
9897 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9898 } else {
9899 RValue VarRVal = CGF.EmitLoadOfLValue(V: VarLVal, Loc: RD->getLocation());
9900 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9901 Args: VDLVal.getPointer(CGF));
9902 CombinedInfo.Exprs.push_back(Elt: VD);
9903 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9904 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9905 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9906 CombinedInfo.Pointers.push_back(Elt: VarRVal.getScalarVal());
9907 CombinedInfo.Sizes.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0));
9908 }
9909 CombinedInfo.Types.push_back(
9910 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9911 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9912 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9913 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9914 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9915 CombinedInfo.Mappers.push_back(Elt: nullptr);
9916 }
9917 }
9918
9919 /// Set correct indices for lambdas captures.
9920 void adjustMemberOfForLambdaCaptures(
9921 llvm::OpenMPIRBuilder &OMPBuilder,
9922 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9923 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9924 MapFlagsArrayTy &Types) const {
9925 for (unsigned I = 0, E = Types.size(); I < E; ++I) {
9926 // Set correct member_of idx for all implicit lambda captures.
9927 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9928 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9929 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9930 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9931 continue;
9932 llvm::Value *BasePtr = LambdaPointers.lookup(Val: BasePointers[I]);
9933 assert(BasePtr && "Unable to find base lambda address.");
9934 int TgtIdx = -1;
9935 for (unsigned J = I; J > 0; --J) {
9936 unsigned Idx = J - 1;
9937 if (Pointers[Idx] != BasePtr)
9938 continue;
9939 TgtIdx = Idx;
9940 break;
9941 }
9942 assert(TgtIdx != -1 && "Unable to find parent lambda.");
9943 // All other current entries will be MEMBER_OF the combined entry
9944 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9945 // 0xFFFF in the MEMBER_OF field).
9946 OpenMPOffloadMappingFlags MemberOfFlag =
9947 OMPBuilder.getMemberOfFlag(Position: TgtIdx);
9948 OMPBuilder.setCorrectMemberOfFlag(Flags&: Types[I], MemberOfFlag);
9949 }
9950 }
9951
9952 /// Populate component lists for non-lambda captured variables from map,
9953 /// is_device_ptr and has_device_addr clause info.
9954 void populateComponentListsForNonLambdaCaptureFromClauses(
9955 const ValueDecl *VD, MapDataArrayTy &DeclComponentLists,
9956 SmallVectorImpl<
9957 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9958 &StorageForImplicitlyAddedComponentLists) const {
9959 if (VD && LambdasMap.count(Val: VD))
9960 return;
9961
9962 // For member fields list in is_device_ptr, store it in
9963 // DeclComponentLists for generating components info.
9964 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9965 auto It = DevPointersMap.find(Val: VD);
9966 if (It != DevPointersMap.end())
9967 for (const auto &MCL : It->second)
9968 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_to, Args: Unknown,
9969 /*IsImpicit = */ Args: true, Args: nullptr,
9970 Args: nullptr);
9971 auto I = HasDevAddrsMap.find(Val: VD);
9972 if (I != HasDevAddrsMap.end())
9973 for (const auto &MCL : I->second)
9974 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_tofrom, Args: Unknown,
9975 /*IsImpicit = */ Args: true, Args: nullptr,
9976 Args: nullptr);
9977 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9978 "Expect a executable directive");
9979 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9980 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
9981 const auto *EI = C->getVarRefs().begin();
9982 for (const auto L : C->decl_component_lists(VD)) {
9983 const ValueDecl *VDecl, *Mapper;
9984 // The Expression is not correct if the mapping is implicit
9985 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
9986 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
9987 std::tie(args&: VDecl, args&: Components, args&: Mapper) = L;
9988 assert(VDecl == VD && "We got information for the wrong declaration??");
9989 assert(!Components.empty() &&
9990 "Not expecting declaration with no component lists.");
9991 DeclComponentLists.emplace_back(Args&: Components, Args: C->getMapType(),
9992 Args: C->getMapTypeModifiers(),
9993 Args: C->isImplicit(), Args&: Mapper, Args&: E);
9994 ++EI;
9995 }
9996 }
9997
9998 // For the target construct, if there's a map with a base-pointer that's
9999 // a member of an implicitly captured struct, of the current class,
10000 // we need to emit an implicit map on the pointer.
10001 if (isOpenMPTargetExecutionDirective(DKind: CurExecDir->getDirectiveKind()))
10002 addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10003 CapturedVD: VD, DeclComponentLists, ComponentVectorStorage&: StorageForImplicitlyAddedComponentLists);
10004
10005 llvm::stable_sort(Range&: DeclComponentLists, C: [](const MapData &LHS,
10006 const MapData &RHS) {
10007 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(t: LHS);
10008 OpenMPMapClauseKind MapType = std::get<1>(t: RHS);
10009 bool HasPresent =
10010 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
10011 bool HasAllocs = MapType == OMPC_MAP_alloc;
10012 MapModifiers = std::get<2>(t: RHS);
10013 MapType = std::get<1>(t: LHS);
10014 bool HasPresentR =
10015 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
10016 bool HasAllocsR = MapType == OMPC_MAP_alloc;
10017 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
10018 });
10019 }
10020
10021 /// On a target construct, if there's an implicit map on a struct, or that of
10022 /// this[:], and an explicit map with a member of that struct/class as the
10023 /// base-pointer, we need to make sure that base-pointer is implicitly mapped,
10024 /// to make sure we don't map the full struct/class. For example:
10025 ///
10026 /// \code
10027 /// struct S {
10028 /// int dummy[10000];
10029 /// int *p;
10030 /// void f1() {
10031 /// #pragma omp target map(p[0:1])
10032 /// (void)this;
10033 /// }
10034 /// }; S s;
10035 ///
10036 /// void f2() {
10037 /// #pragma omp target map(s.p[0:10])
10038 /// (void)s;
10039 /// }
10040 /// \endcode
10041 ///
10042 /// Only `this-p` and `s.p` should be mapped in the two cases above.
10043 //
10044 // OpenMP 6.0: 7.9.6 map clause, pg 285
10045 // If a list item with an implicitly determined data-mapping attribute does
10046 // not have any corresponding storage in the device data environment prior to
10047 // a task encountering the construct associated with the map clause, and one
10048 // or more contiguous parts of the original storage are either list items or
10049 // base pointers to list items that are explicitly mapped on the construct,
10050 // only those parts of the original storage will have corresponding storage in
10051 // the device data environment as a result of the map clauses on the
10052 // construct.
10053 void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10054 const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists,
10055 SmallVectorImpl<
10056 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
10057 &ComponentVectorStorage) const {
10058 bool IsThisCapture = CapturedVD == nullptr;
10059
10060 for (const auto &ComponentsAndAttachPtr : AttachPtrExprMap) {
10061 OMPClauseMappableExprCommon::MappableExprComponentListRef
10062 ComponentsWithAttachPtr = ComponentsAndAttachPtr.first;
10063 const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second;
10064 if (!AttachPtrExpr)
10065 continue;
10066
10067 const auto *ME = dyn_cast<MemberExpr>(Val: AttachPtrExpr);
10068 if (!ME)
10069 continue;
10070
10071 const Expr *Base = ME->getBase()->IgnoreParenImpCasts();
10072
10073 // If we are handling a "this" capture, then we are looking for
10074 // attach-ptrs of form `this->p`, either explicitly or implicitly.
10075 if (IsThisCapture && !ME->isImplicitCXXThis() && !isa<CXXThisExpr>(Val: Base))
10076 continue;
10077
10078 if (!IsThisCapture && (!isa<DeclRefExpr>(Val: Base) ||
10079 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
10080 continue;
10081
10082 // For non-this captures, we are looking for attach-ptrs of form
10083 // `s.p`.
10084 // For non-this captures, we are looking for attach-ptrs like `s.p`.
10085 if (!IsThisCapture && (ME->isArrow() || !isa<DeclRefExpr>(Val: Base) ||
10086 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
10087 continue;
10088
10089 // Check if we have an existing map on either:
10090 // this[:], s, this->p, or s.p, in which case, we don't need to add
10091 // an implicit one for the attach-ptr s.p/this->p.
10092 bool FoundExistingMap = false;
10093 for (const MapData &ExistingL : DeclComponentLists) {
10094 OMPClauseMappableExprCommon::MappableExprComponentListRef
10095 ExistingComponents = std::get<0>(t: ExistingL);
10096
10097 if (ExistingComponents.empty())
10098 continue;
10099
10100 // First check if we have a map like map(this->p) or map(s.p).
10101 const auto &FirstComponent = ExistingComponents.front();
10102 const Expr *FirstExpr = FirstComponent.getAssociatedExpression();
10103
10104 if (!FirstExpr)
10105 continue;
10106
10107 // First check if we have a map like map(this->p) or map(s.p).
10108 if (AttachPtrComparator.areEqual(LHS: FirstExpr, RHS: AttachPtrExpr)) {
10109 FoundExistingMap = true;
10110 break;
10111 }
10112
10113 // Check if we have a map like this[0:1]
10114 if (IsThisCapture) {
10115 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: FirstExpr)) {
10116 if (isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts())) {
10117 FoundExistingMap = true;
10118 break;
10119 }
10120 }
10121 continue;
10122 }
10123
10124 // When the attach-ptr is something like `s.p`, check if
10125 // `s` itself is mapped explicitly.
10126 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: FirstExpr)) {
10127 if (DRE->getDecl() == CapturedVD) {
10128 FoundExistingMap = true;
10129 break;
10130 }
10131 }
10132 }
10133
10134 if (FoundExistingMap)
10135 continue;
10136
10137 // If no base map is found, we need to create an implicit map for the
10138 // attach-pointer expr.
10139
10140 ComponentVectorStorage.emplace_back();
10141 auto &AttachPtrComponents = ComponentVectorStorage.back();
10142
10143 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
10144 bool SeenAttachPtrComponent = false;
10145 // For creating a map on the attach-ptr `s.p/this->p`, we copy all
10146 // components from the component-list which has `s.p/this->p`
10147 // as the attach-ptr, starting from the component which matches
10148 // `s.p/this->p`. This way, we'll have component-lists of
10149 // `s.p` -> `s`, and `this->p` -> `this`.
10150 for (size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) {
10151 const auto &Component = ComponentsWithAttachPtr[i];
10152 const Expr *ComponentExpr = Component.getAssociatedExpression();
10153
10154 if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr)
10155 continue;
10156 SeenAttachPtrComponent = true;
10157
10158 AttachPtrComponents.emplace_back(Args: Component.getAssociatedExpression(),
10159 Args: Component.getAssociatedDeclaration(),
10160 Args: Component.isNonContiguous());
10161 }
10162 assert(!AttachPtrComponents.empty() &&
10163 "Could not populate component-lists for mapping attach-ptr");
10164
10165 DeclComponentLists.emplace_back(
10166 Args&: AttachPtrComponents, Args: OMPC_MAP_tofrom, Args: Unknown,
10167 /*IsImplicit=*/Args: true, /*mapper=*/Args: nullptr, Args&: AttachPtrExpr);
10168 }
10169 }
10170
10171 /// For a capture that has an associated clause, generate the base pointers,
10172 /// section pointers, sizes, map types, and mappers (all included in
10173 /// \a CurCaptureVarInfo).
10174 void generateInfoForCaptureFromClauseInfo(
10175 const MapDataArrayTy &DeclComponentListsFromClauses,
10176 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
10177 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
10178 unsigned OffsetForMemberOfFlag) const {
10179 assert(!Cap->capturesVariableArrayType() &&
10180 "Not expecting to generate map info for a variable array type!");
10181
10182 // We need to know when we generating information for the first component
10183 const ValueDecl *VD = Cap->capturesThis()
10184 ? nullptr
10185 : Cap->getCapturedVar()->getCanonicalDecl();
10186
10187 // for map(to: lambda): skip here, processing it in
10188 // generateDefaultMapInfo
10189 if (LambdasMap.count(Val: VD))
10190 return;
10191
10192 // If this declaration appears in a is_device_ptr clause we just have to
10193 // pass the pointer by value. If it is a reference to a declaration, we just
10194 // pass its value.
10195 if (VD && (DevPointersMap.count(Val: VD) || HasDevAddrsMap.count(Val: VD))) {
10196 CurCaptureVarInfo.Exprs.push_back(Elt: VD);
10197 CurCaptureVarInfo.BasePointers.emplace_back(Args&: Arg);
10198 CurCaptureVarInfo.DevicePtrDecls.emplace_back(Args&: VD);
10199 CurCaptureVarInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::Pointer);
10200 CurCaptureVarInfo.Pointers.push_back(Elt: Arg);
10201 CurCaptureVarInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10202 V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy), DestTy: CGF.Int64Ty,
10203 /*isSigned=*/true));
10204 CurCaptureVarInfo.Types.push_back(
10205 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10206 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
10207 CurCaptureVarInfo.HasAttachPtr.push_back(Elt: false);
10208 CurCaptureVarInfo.Mappers.push_back(Elt: nullptr);
10209 return;
10210 }
10211
10212 auto GenerateInfoForComponentLists =
10213 [&](ArrayRef<MapData> DeclComponentListsFromClauses,
10214 bool IsEligibleForTargetParamFlag) {
10215 MapCombinedInfoTy CurInfoForComponentLists;
10216 StructRangeInfoTy PartialStruct;
10217 AttachInfoTy AttachInfo;
10218
10219 if (DeclComponentListsFromClauses.empty())
10220 return;
10221
10222 generateInfoForCaptureFromComponentLists(
10223 VD, DeclComponentLists: DeclComponentListsFromClauses, CurComponentListInfo&: CurInfoForComponentLists,
10224 PartialStruct, AttachInfo, IsListEligibleForTargetParamFlag: IsEligibleForTargetParamFlag);
10225
10226 // If there is an entry in PartialStruct it means we have a
10227 // struct with individual members mapped. Emit an extra combined
10228 // entry.
10229 if (PartialStruct.Base.isValid()) {
10230 CurCaptureVarInfo.append(CurInfo&: PartialStruct.PreliminaryMapData);
10231 emitCombinedEntry(
10232 CombinedInfo&: CurCaptureVarInfo, CurTypes&: CurInfoForComponentLists.Types,
10233 PartialStruct, AttachInfo, IsMapThis: Cap->capturesThis(), OMPBuilder,
10234 /*VD=*/nullptr, OffsetForMemberOfFlag,
10235 /*NotTargetParams*/ !IsEligibleForTargetParamFlag);
10236 }
10237
10238 // We do the appends to get the entries in the following order:
10239 // combined-entry -> individual-field-entries -> attach-entry,
10240 CurCaptureVarInfo.append(CurInfo&: CurInfoForComponentLists);
10241 if (AttachInfo.isValid())
10242 emitAttachEntry(CGF, CombinedInfo&: CurCaptureVarInfo, AttachInfo);
10243 };
10244
10245 // Group component lists by their AttachPtrExpr and process them in order
10246 // of increasing complexity (nullptr first, then simple expressions like p,
10247 // then more complex ones like p[0], etc.)
10248 //
10249 // This ensure that we:
10250 // * handle maps that can contribute towards setting the kernel argument,
10251 // (e.g. map(ps), or map(ps[0])), before any that cannot (e.g. ps->pt->d).
10252 // * allocate a single contiguous storage for all exprs with the same
10253 // captured var and having the same attach-ptr.
10254 //
10255 // Example: The map clauses below should be handled grouped together based
10256 // on their attachable-base-pointers:
10257 // map-clause | attachable-base-pointer
10258 // --------------------------+------------------------
10259 // map(p, ps) | nullptr
10260 // map(p[0]) | p
10261 // map(p[0]->b, p[0]->c) | p[0]
10262 // map(ps->d, ps->e, ps->pt) | ps
10263 // map(ps->pt->d, ps->pt->e) | ps->pt
10264
10265 // First, collect all MapData entries with their attach-ptr exprs.
10266 SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs;
10267
10268 for (const MapData &L : DeclComponentListsFromClauses) {
10269 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
10270 std::get<0>(t: L);
10271 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
10272 AttachPtrMapDataPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
10273 }
10274
10275 // Next, sort by increasing order of their complexity.
10276 llvm::stable_sort(Range&: AttachPtrMapDataPairs,
10277 C: [this](const auto &LHS, const auto &RHS) {
10278 return AttachPtrComparator(LHS.first, RHS.first);
10279 });
10280
10281 bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty();
10282 bool IsFirstGroup = true;
10283
10284 // And finally, process them all in order, grouping those with
10285 // equivalent attach-ptr exprs together.
10286 auto *It = AttachPtrMapDataPairs.begin();
10287 while (It != AttachPtrMapDataPairs.end()) {
10288 const Expr *AttachPtrExpr = It->first;
10289
10290 MapDataArrayTy GroupLists;
10291 while (It != AttachPtrMapDataPairs.end() &&
10292 (It->first == AttachPtrExpr ||
10293 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
10294 GroupLists.push_back(Elt: It->second);
10295 ++It;
10296 }
10297 assert(!GroupLists.empty() && "GroupLists should not be empty");
10298
10299 // Determine if this group of component-lists is eligible for TARGET_PARAM
10300 // flag. Only the first group processed should be eligible, and only if no
10301 // default mapping was done.
10302 bool IsEligibleForTargetParamFlag =
10303 IsFirstGroup && NoDefaultMappingDoneForVD;
10304
10305 GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag);
10306 IsFirstGroup = false;
10307 }
10308 }
10309
10310 /// Generate the base pointers, section pointers, sizes, map types, and
10311 /// mappers associated to \a DeclComponentLists for a given capture
10312 /// \a VD (all included in \a CurComponentListInfo).
10313 void generateInfoForCaptureFromComponentLists(
10314 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
10315 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
10316 AttachInfoTy &AttachInfo, bool IsListEligibleForTargetParamFlag) const {
10317 // Find overlapping elements (including the offset from the base element).
10318 llvm::SmallDenseMap<
10319 const MapData *,
10320 llvm::SmallVector<
10321 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
10322 4>
10323 OverlappedData;
10324 size_t Count = 0;
10325 for (const MapData &L : DeclComponentLists) {
10326 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10327 OpenMPMapClauseKind MapType;
10328 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10329 bool IsImplicit;
10330 const ValueDecl *Mapper;
10331 const Expr *VarRef;
10332 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10333 L;
10334 ++Count;
10335 for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(N: Count)) {
10336 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
10337 std::tie(args&: Components1, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper,
10338 args&: VarRef) = L1;
10339 auto CI = Components.rbegin();
10340 auto CE = Components.rend();
10341 auto SI = Components1.rbegin();
10342 auto SE = Components1.rend();
10343 for (; CI != CE && SI != SE; ++CI, ++SI) {
10344 if (CI->getAssociatedExpression()->getStmtClass() !=
10345 SI->getAssociatedExpression()->getStmtClass())
10346 break;
10347 // Are we dealing with different variables/fields?
10348 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
10349 break;
10350 }
10351 // Found overlapping if, at least for one component, reached the head
10352 // of the components list.
10353 if (CI == CE || SI == SE) {
10354 // Ignore it if it is the same component.
10355 if (CI == CE && SI == SE)
10356 continue;
10357 const auto It = (SI == SE) ? CI : SI;
10358 // If one component is a pointer and another one is a kind of
10359 // dereference of this pointer (array subscript, section, dereference,
10360 // etc.), it is not an overlapping.
10361 // Same, if one component is a base and another component is a
10362 // dereferenced pointer memberexpr with the same base.
10363 if (!isa<MemberExpr>(Val: It->getAssociatedExpression()) ||
10364 (std::prev(x: It)->getAssociatedDeclaration() &&
10365 std::prev(x: It)
10366 ->getAssociatedDeclaration()
10367 ->getType()
10368 ->isPointerType()) ||
10369 (It->getAssociatedDeclaration() &&
10370 It->getAssociatedDeclaration()->getType()->isPointerType() &&
10371 std::next(x: It) != CE && std::next(x: It) != SE))
10372 continue;
10373 const MapData &BaseData = CI == CE ? L : L1;
10374 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
10375 SI == SE ? Components : Components1;
10376 OverlappedData[&BaseData].push_back(Elt: SubData);
10377 }
10378 }
10379 }
10380 // Sort the overlapped elements for each item.
10381 llvm::SmallVector<const FieldDecl *, 4> Layout;
10382 if (!OverlappedData.empty()) {
10383 const Type *BaseType = VD->getType().getCanonicalType().getTypePtr();
10384 const Type *OrigType = BaseType->getPointeeOrArrayElementType();
10385 while (BaseType != OrigType) {
10386 BaseType = OrigType->getCanonicalTypeInternal().getTypePtr();
10387 OrigType = BaseType->getPointeeOrArrayElementType();
10388 }
10389
10390 if (const auto *CRD = BaseType->getAsCXXRecordDecl())
10391 getPlainLayout(RD: CRD, Layout, /*AsBase=*/false);
10392 else {
10393 const auto *RD = BaseType->getAsRecordDecl();
10394 Layout.append(in_start: RD->field_begin(), in_end: RD->field_end());
10395 }
10396 }
10397 for (auto &Pair : OverlappedData) {
10398 llvm::stable_sort(
10399 Range&: Pair.getSecond(),
10400 C: [&Layout](
10401 OMPClauseMappableExprCommon::MappableExprComponentListRef First,
10402 OMPClauseMappableExprCommon::MappableExprComponentListRef
10403 Second) {
10404 auto CI = First.rbegin();
10405 auto CE = First.rend();
10406 auto SI = Second.rbegin();
10407 auto SE = Second.rend();
10408 for (; CI != CE && SI != SE; ++CI, ++SI) {
10409 if (CI->getAssociatedExpression()->getStmtClass() !=
10410 SI->getAssociatedExpression()->getStmtClass())
10411 break;
10412 // Are we dealing with different variables/fields?
10413 if (CI->getAssociatedDeclaration() !=
10414 SI->getAssociatedDeclaration())
10415 break;
10416 }
10417
10418 // Lists contain the same elements.
10419 if (CI == CE && SI == SE)
10420 return false;
10421
10422 // List with less elements is less than list with more elements.
10423 if (CI == CE || SI == SE)
10424 return CI == CE;
10425
10426 const auto *FD1 = cast<FieldDecl>(Val: CI->getAssociatedDeclaration());
10427 const auto *FD2 = cast<FieldDecl>(Val: SI->getAssociatedDeclaration());
10428 if (FD1->getParent() == FD2->getParent())
10429 return FD1->getFieldIndex() < FD2->getFieldIndex();
10430 const auto *It =
10431 llvm::find_if(Range&: Layout, P: [FD1, FD2](const FieldDecl *FD) {
10432 return FD == FD1 || FD == FD2;
10433 });
10434 return *It == FD1;
10435 });
10436 }
10437
10438 // Associated with a capture, because the mapping flags depend on it.
10439 // Go through all of the elements with the overlapped elements.
10440 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
10441 MapCombinedInfoTy StructBaseCombinedInfo;
10442 for (const auto &Pair : OverlappedData) {
10443 const MapData &L = *Pair.getFirst();
10444 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10445 OpenMPMapClauseKind MapType;
10446 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10447 bool IsImplicit;
10448 const ValueDecl *Mapper;
10449 const Expr *VarRef;
10450 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10451 L;
10452 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
10453 OverlappedComponents = Pair.getSecond();
10454 generateInfoForComponentList(
10455 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10456 StructBaseCombinedInfo, PartialStruct, AttachInfo, IsFirstComponentList: AddTargetParamFlag,
10457 IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
10458 /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef, OverlappedElements: OverlappedComponents);
10459 AddTargetParamFlag = false;
10460 }
10461 // Go through other elements without overlapped elements.
10462 for (const MapData &L : DeclComponentLists) {
10463 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10464 OpenMPMapClauseKind MapType;
10465 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10466 bool IsImplicit;
10467 const ValueDecl *Mapper;
10468 const Expr *VarRef;
10469 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10470 L;
10471 auto It = OverlappedData.find(Val: &L);
10472 if (It == OverlappedData.end())
10473 generateInfoForComponentList(
10474 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10475 StructBaseCombinedInfo, PartialStruct, AttachInfo,
10476 IsFirstComponentList: AddTargetParamFlag, IsImplicit, /*GenerateAllInfoForClauses*/ false,
10477 Mapper, /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef,
10478 /*OverlappedElements*/ {});
10479 AddTargetParamFlag = false;
10480 }
10481 }
10482
10483 /// Check if a variable should be treated as firstprivate due to explicit
10484 /// firstprivate clause or defaultmap(firstprivate:...).
10485 bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const {
10486 // Check explicit firstprivate clauses (not implicit from defaultmap)
10487 auto I = FirstPrivateDecls.find(Val: VD);
10488 if (I != FirstPrivateDecls.end() && !I->getSecond())
10489 return true; // Explicit firstprivate only
10490
10491 // Check defaultmap(firstprivate:scalar) for scalar types
10492 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_scalar)) {
10493 if (Type->isScalarType())
10494 return true;
10495 }
10496
10497 // Check defaultmap(firstprivate:pointer) for pointer types
10498 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_pointer)) {
10499 if (Type->isAnyPointerType())
10500 return true;
10501 }
10502
10503 // Check defaultmap(firstprivate:aggregate) for aggregate types
10504 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_aggregate)) {
10505 if (Type->isAggregateType())
10506 return true;
10507 }
10508
10509 // Check defaultmap(firstprivate:all) for all types
10510 return DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_all);
10511 }
10512
10513 /// Generate the default map information for a given capture \a CI,
10514 /// record field declaration \a RI and captured value \a CV.
10515 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
10516 const FieldDecl &RI, llvm::Value *CV,
10517 MapCombinedInfoTy &CombinedInfo) const {
10518 bool IsImplicit = true;
10519 // Do the default mapping.
10520 if (CI.capturesThis()) {
10521 CombinedInfo.Exprs.push_back(Elt: nullptr);
10522 CombinedInfo.BasePointers.push_back(Elt: CV);
10523 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10524 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10525 CombinedInfo.Pointers.push_back(Elt: CV);
10526 const auto *PtrTy = cast<PointerType>(Val: RI.getType().getTypePtr());
10527 CombinedInfo.Sizes.push_back(
10528 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: PtrTy->getPointeeType()),
10529 DestTy: CGF.Int64Ty, /*isSigned=*/true));
10530 // Default map type.
10531 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TO |
10532 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
10533 } else if (CI.capturesVariableByCopy()) {
10534 const VarDecl *VD = CI.getCapturedVar();
10535 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10536 CombinedInfo.BasePointers.push_back(Elt: CV);
10537 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10538 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10539 CombinedInfo.Pointers.push_back(Elt: CV);
10540 bool IsFirstprivate =
10541 isEffectivelyFirstprivate(VD, Type: RI.getType().getNonReferenceType());
10542
10543 if (!RI.getType()->isAnyPointerType()) {
10544 // We have to signal to the runtime captures passed by value that are
10545 // not pointers.
10546 CombinedInfo.Types.push_back(
10547 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10548 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10549 V: CGF.getTypeSize(Ty: RI.getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10550 } else if (IsFirstprivate) {
10551 // Firstprivate pointers should be passed by value (as literals)
10552 // without performing a present table lookup at runtime.
10553 CombinedInfo.Types.push_back(
10554 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10555 // Use zero size for pointer literals (just passing the pointer value)
10556 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10557 } else {
10558 // Pointers are implicitly mapped with a zero size and no flags
10559 // (other than first map that is added for all implicit maps).
10560 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_NONE);
10561 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10562 }
10563 auto I = FirstPrivateDecls.find(Val: VD);
10564 if (I != FirstPrivateDecls.end())
10565 IsImplicit = I->getSecond();
10566 } else {
10567 assert(CI.capturesVariable() && "Expected captured reference.");
10568 const auto *PtrTy = cast<ReferenceType>(Val: RI.getType().getTypePtr());
10569 QualType ElementType = PtrTy->getPointeeType();
10570 const VarDecl *VD = CI.getCapturedVar();
10571 bool IsFirstprivate = isEffectivelyFirstprivate(VD, Type: ElementType);
10572 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10573 CombinedInfo.BasePointers.push_back(Elt: CV);
10574 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10575 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10576
10577 // For firstprivate pointers, pass by value instead of dereferencing
10578 if (IsFirstprivate && ElementType->isAnyPointerType()) {
10579 // Treat as a literal value (pass the pointer value itself)
10580 CombinedInfo.Pointers.push_back(Elt: CV);
10581 // Use zero size for pointer literals
10582 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10583 CombinedInfo.Types.push_back(
10584 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10585 } else {
10586 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10587 V: CGF.getTypeSize(Ty: ElementType), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10588 // The default map type for a scalar/complex type is 'to' because by
10589 // default the value doesn't have to be retrieved. For an aggregate
10590 // type, the default is 'tofrom'.
10591 CombinedInfo.Types.push_back(Elt: getMapModifiersForPrivateClauses(Cap: CI));
10592 CombinedInfo.Pointers.push_back(Elt: CV);
10593 }
10594 auto I = FirstPrivateDecls.find(Val: VD);
10595 if (I != FirstPrivateDecls.end())
10596 IsImplicit = I->getSecond();
10597 }
10598 // Every default map produces a single argument which is a target parameter.
10599 CombinedInfo.Types.back() |=
10600 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
10601
10602 // Add flag stating this is an implicit map.
10603 if (IsImplicit)
10604 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
10605
10606 CombinedInfo.HasAttachPtr.push_back(Elt: false);
10607 // No user-defined mapper for default mapping.
10608 CombinedInfo.Mappers.push_back(Elt: nullptr);
10609 }
10610};
10611} // anonymous namespace
10612
10613// Try to extract the base declaration from a `this->x` expression if possible.
10614static ValueDecl *getDeclFromThisExpr(const Expr *E) {
10615 if (!E)
10616 return nullptr;
10617
10618 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenCasts()))
10619 if (const MemberExpr *ME =
10620 dyn_cast<MemberExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))
10621 return ME->getMemberDecl();
10622 return nullptr;
10623}
10624
10625/// Emit a string constant containing the names of the values mapped to the
10626/// offloading runtime library.
10627static llvm::Constant *
10628emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
10629 MappableExprsHandler::MappingExprInfo &MapExprs) {
10630
10631 uint32_t SrcLocStrSize;
10632 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
10633 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10634
10635 SourceLocation Loc;
10636 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
10637 if (const ValueDecl *VD = getDeclFromThisExpr(E: MapExprs.getMapExpr()))
10638 Loc = VD->getLocation();
10639 else
10640 Loc = MapExprs.getMapExpr()->getExprLoc();
10641 } else {
10642 Loc = MapExprs.getMapDecl()->getLocation();
10643 }
10644
10645 std::string ExprName;
10646 if (MapExprs.getMapExpr()) {
10647 PrintingPolicy P(CGF.getContext().getLangOpts());
10648 llvm::raw_string_ostream OS(ExprName);
10649 MapExprs.getMapExpr()->printPretty(OS, Helper: nullptr, Policy: P);
10650 } else {
10651 ExprName = MapExprs.getMapDecl()->getNameAsString();
10652 }
10653
10654 std::string FileName;
10655 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
10656 if (auto *DbgInfo = CGF.getDebugInfo())
10657 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
10658 else
10659 FileName = PLoc.getFilename();
10660 return OMPBuilder.getOrCreateSrcLocStr(FunctionName: FileName, FileName: ExprName, Line: PLoc.getLine(),
10661 Column: PLoc.getColumn(), SrcLocStrSize);
10662}
10663/// Emit the arrays used to pass the captures and map information to the
10664/// offloading runtime library. If there is no map or capture information,
10665/// return nullptr by reference.
10666static void emitOffloadingArraysAndArgs(
10667 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10668 CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder,
10669 bool IsNonContiguous = false, bool ForEndCall = false) {
10670 CodeGenModule &CGM = CGF.CGM;
10671
10672 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10673 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
10674 InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
10675
10676 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
10677 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10678 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
10679 }
10680 };
10681
10682 auto CustomMapperCB = [&](unsigned int I) {
10683 llvm::Function *MFunc = nullptr;
10684 if (CombinedInfo.Mappers[I]) {
10685 Info.HasMapper = true;
10686 MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
10687 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10688 }
10689 return MFunc;
10690 };
10691 cantFail(Err: OMPBuilder.emitOffloadingArraysAndArgs(
10692 AllocaIP, CodeGenIP, Info, RTArgs&: Info.RTArgs, CombinedInfo, CustomMapperCB,
10693 IsNonContiguous, ForEndCall, DeviceAddrCB));
10694}
10695
10696/// Check for inner distribute directive.
10697static const OMPExecutableDirective *
10698getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
10699 const auto *CS = D.getInnermostCapturedStmt();
10700 const auto *Body =
10701 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
10702 const Stmt *ChildStmt =
10703 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10704
10705 if (const auto *NestedDir =
10706 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10707 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
10708 switch (D.getDirectiveKind()) {
10709 case OMPD_target:
10710 // For now, treat 'target' with nested 'teams loop' as if it's
10711 // distributed (target teams distribute).
10712 if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop)
10713 return NestedDir;
10714 if (DKind == OMPD_teams) {
10715 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
10716 /*IgnoreCaptured=*/true);
10717 if (!Body)
10718 return nullptr;
10719 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10720 if (const auto *NND =
10721 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10722 DKind = NND->getDirectiveKind();
10723 if (isOpenMPDistributeDirective(DKind))
10724 return NND;
10725 }
10726 }
10727 return nullptr;
10728 case OMPD_target_teams:
10729 if (isOpenMPDistributeDirective(DKind))
10730 return NestedDir;
10731 return nullptr;
10732 case OMPD_target_parallel:
10733 case OMPD_target_simd:
10734 case OMPD_target_parallel_for:
10735 case OMPD_target_parallel_for_simd:
10736 return nullptr;
10737 case OMPD_target_teams_distribute:
10738 case OMPD_target_teams_distribute_simd:
10739 case OMPD_target_teams_distribute_parallel_for:
10740 case OMPD_target_teams_distribute_parallel_for_simd:
10741 case OMPD_parallel:
10742 case OMPD_for:
10743 case OMPD_parallel_for:
10744 case OMPD_parallel_master:
10745 case OMPD_parallel_sections:
10746 case OMPD_for_simd:
10747 case OMPD_parallel_for_simd:
10748 case OMPD_cancel:
10749 case OMPD_cancellation_point:
10750 case OMPD_ordered_standalone:
10751 case OMPD_ordered_blockassoc:
10752 case OMPD_threadprivate:
10753 case OMPD_allocate:
10754 case OMPD_task:
10755 case OMPD_simd:
10756 case OMPD_tile:
10757 case OMPD_unroll:
10758 case OMPD_sections:
10759 case OMPD_section:
10760 case OMPD_single:
10761 case OMPD_master:
10762 case OMPD_critical:
10763 case OMPD_taskyield:
10764 case OMPD_barrier:
10765 case OMPD_taskwait:
10766 case OMPD_taskgroup:
10767 case OMPD_atomic:
10768 case OMPD_flush:
10769 case OMPD_depobj:
10770 case OMPD_scan:
10771 case OMPD_teams:
10772 case OMPD_target_data:
10773 case OMPD_target_exit_data:
10774 case OMPD_target_enter_data:
10775 case OMPD_distribute:
10776 case OMPD_distribute_simd:
10777 case OMPD_distribute_parallel_for:
10778 case OMPD_distribute_parallel_for_simd:
10779 case OMPD_teams_distribute:
10780 case OMPD_teams_distribute_simd:
10781 case OMPD_teams_distribute_parallel_for:
10782 case OMPD_teams_distribute_parallel_for_simd:
10783 case OMPD_target_update:
10784 case OMPD_declare_simd:
10785 case OMPD_declare_variant:
10786 case OMPD_begin_declare_variant:
10787 case OMPD_end_declare_variant:
10788 case OMPD_declare_target:
10789 case OMPD_end_declare_target:
10790 case OMPD_declare_reduction:
10791 case OMPD_declare_mapper:
10792 case OMPD_taskloop:
10793 case OMPD_taskloop_simd:
10794 case OMPD_master_taskloop:
10795 case OMPD_master_taskloop_simd:
10796 case OMPD_parallel_master_taskloop:
10797 case OMPD_parallel_master_taskloop_simd:
10798 case OMPD_requires:
10799 case OMPD_metadirective:
10800 case OMPD_unknown:
10801 default:
10802 llvm_unreachable("Unexpected directive.");
10803 }
10804 }
10805
10806 return nullptr;
10807}
10808
10809/// Emit the user-defined mapper function. The code generation follows the
10810/// pattern in the example below.
10811/// \code
10812/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
10813/// void *base, void *begin,
10814/// int64_t size, int64_t type,
10815/// void *name = nullptr) {
10816/// // Allocate space for an array section first.
10817/// if ((size > 1 || (base != begin)) && !maptype.IsDelete)
10818/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10819/// size*sizeof(Ty), clearToFromMember(type));
10820/// // Map members.
10821/// for (unsigned i = 0; i < size; i++) {
10822/// N = __tgt_mapper_num_components(rt_mapper_handle);
10823/// // For each component specified by this mapper:
10824/// for (auto c : begin[i]->all_components) {
10825/// // MEMBER_OF grouping: tie this component to the current array element
10826/// // (component N) by adding N<<48. Exceptions:
10827/// // - ATTACH entries are not members of any struct storage range.
10828/// // - Pointee entries (reached via a pointer member) occupy separate
10829/// // storage; their inner MEMBER_OF bits are shifted by N instead.
10830/// if (c.isAttach() || c.isPointee())
10831/// member_type = c.arg_type + (c.hasInnerMemberOf() ? N<<48 : 0);
10832/// else
10833/// member_type = c.arg_type + N<<48;
10834/// // Map-type-modifying bits (ALWAYS, DELETE, CLOSE) from the outer map
10835/// // clause are propagated to each component, except ATTACH entries
10836/// // (ATTACH|ALWAYS is reserved for attach(always), and other modifier
10837/// // bits have no meaning for ATTACH). PRESENT is additionally
10838/// // propagated to components with HasAttachPtr (the pointee data) at
10839/// // OpenMP >= 6.0.
10840/// present_bit = (v60 && c.hasAttachPtr()) ? PRESENT : 0;
10841/// imported_modifier_bits =
10842/// type & (ALWAYS | DELETE | CLOSE | present_bit);
10843/// effective_type = c.isAttach() ? member_type
10844/// : member_type | imported_modifier_bits;
10845/// if (c.hasMapper())
10846/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
10847/// effective_type, c.arg_name);
10848/// else
10849/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
10850/// c.arg_begin, c.arg_size, effective_type,
10851/// c.arg_name);
10852/// }
10853/// }
10854/// // Delete the array section.
10855/// if (size > 1 && maptype.IsDelete)
10856/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10857/// size*sizeof(Ty), clearToFromMember(type));
10858/// }
10859/// \endcode
10860void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
10861 CodeGenFunction *CGF) {
10862 if (UDMMap.count(Val: D) > 0)
10863 return;
10864 ASTContext &C = CGM.getContext();
10865 QualType Ty = D->getType();
10866 auto *MapperVarDecl =
10867 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getMapperVarRef())->getDecl());
10868 CharUnits ElementSize = C.getTypeSizeInChars(T: Ty);
10869 llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
10870
10871 CodeGenFunction MapperCGF(CGM);
10872 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10873 auto PrivatizeAndGenMapInfoCB =
10874 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10875 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10876 MapperCGF.Builder.restoreIP(IP: CodeGenIP);
10877
10878 // Privatize the declared variable of mapper to be the current array
10879 // element.
10880 Address PtrCurrent(
10881 PtrPHI, ElemTy,
10882 Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign())
10883 .getAlignment()
10884 .alignmentOfArrayElement(elementSize: ElementSize));
10885 CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
10886 Scope.addPrivate(LocalVD: MapperVarDecl, Addr: PtrCurrent);
10887 (void)Scope.Privatize();
10888
10889 // Get map clause information.
10890 MappableExprsHandler MEHandler(*D, MapperCGF);
10891 MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder);
10892
10893 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10894 return emitMappingInformation(CGF&: MapperCGF, OMPBuilder, MapExprs&: MapExpr);
10895 };
10896 if (CGM.getCodeGenOpts().getDebugInfo() !=
10897 llvm::codegenoptions::NoDebugInfo) {
10898 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10899 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10900 F: FillInfoMap);
10901 }
10902
10903 return CombinedInfo;
10904 };
10905
10906 auto CustomMapperCB = [&](unsigned I) {
10907 llvm::Function *MapperFunc = nullptr;
10908 if (CombinedInfo.Mappers[I]) {
10909 // Call the corresponding mapper function.
10910 MapperFunc = getOrCreateUserDefinedMapperFunc(
10911 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10912 assert(MapperFunc && "Expect a valid mapper function is available.");
10913 }
10914 return MapperFunc;
10915 };
10916
10917 SmallString<64> TyStr;
10918 llvm::raw_svector_ostream Out(TyStr);
10919 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out);
10920 std::string Name = getName(Parts: {"omp_mapper", TyStr, D->getName()});
10921
10922 // Propagate the PRESENT modifier to the pointee entries (those with
10923 // HasAttachPtr) only for OpenMP >= 6.0; before 6.0 the present modifier does
10924 // not apply to the pointee (see the OpenMP 6.0 erratum on the present motion
10925 // vs. map-type modifier divergence).
10926 bool PropagatePresentToPointee = CGM.getLangOpts().OpenMP >= 60;
10927 llvm::Function *NewFn = cantFail(ValOrErr: OMPBuilder.emitUserDefinedMapper(
10928 PrivAndGenMapInfoCB: PrivatizeAndGenMapInfoCB, ElemTy, FuncName: Name, CustomMapperCB,
10929 /*PreserveMemberOfFlags=*/false, PropagatePresentToPointee));
10930 UDMMap.try_emplace(Key: D, Args&: NewFn);
10931 if (CGF)
10932 FunctionUDMMap[CGF->CurFn].push_back(Elt: D);
10933}
10934
10935llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc(
10936 const OMPDeclareMapperDecl *D) {
10937 auto I = UDMMap.find(Val: D);
10938 if (I != UDMMap.end())
10939 return I->second;
10940 emitUserDefinedMapper(D);
10941 return UDMMap.lookup(Val: D);
10942}
10943
10944llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall(
10945 CodeGenFunction &CGF, const OMPExecutableDirective &D,
10946 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10947 const OMPLoopDirective &D)>
10948 SizeEmitter) {
10949 OpenMPDirectiveKind Kind = D.getDirectiveKind();
10950 const OMPExecutableDirective *TD = &D;
10951 // Get nested teams distribute kind directive, if any. For now, treat
10952 // 'target_teams_loop' as if it's really a target_teams_distribute.
10953 if ((!isOpenMPDistributeDirective(DKind: Kind) || !isOpenMPTeamsDirective(DKind: Kind)) &&
10954 Kind != OMPD_target_teams_loop)
10955 TD = getNestedDistributeDirective(Ctx&: CGM.getContext(), D);
10956 if (!TD)
10957 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10958
10959 const auto *LD = cast<OMPLoopDirective>(Val: TD);
10960 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
10961 return NumIterations;
10962 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10963}
10964
10965static void
10966emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10967 const OMPExecutableDirective &D,
10968 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10969 bool RequiresOuterTask, const CapturedStmt &CS,
10970 bool OffloadingMandatory, CodeGenFunction &CGF) {
10971 if (OffloadingMandatory) {
10972 CGF.Builder.CreateUnreachable();
10973 } else {
10974 if (RequiresOuterTask) {
10975 CapturedVars.clear();
10976 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
10977 }
10978 llvm::SmallVector<llvm::Value *, 16> Args(CapturedVars.begin(),
10979 CapturedVars.end());
10980 Args.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy()));
10981 OMPRuntime->emitOutlinedFunctionCall(CGF, Loc: D.getBeginLoc(), OutlinedFn,
10982 Args);
10983 }
10984}
10985
10986static llvm::Value *emitDeviceID(
10987 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
10988 CodeGenFunction &CGF) {
10989 // Emit device ID if any.
10990 llvm::Value *DeviceID;
10991 if (Device.getPointer()) {
10992 assert((Device.getInt() == OMPC_DEVICE_unknown ||
10993 Device.getInt() == OMPC_DEVICE_device_num) &&
10994 "Expected device_num modifier.");
10995 llvm::Value *DevVal = CGF.EmitScalarExpr(E: Device.getPointer());
10996 DeviceID =
10997 CGF.Builder.CreateIntCast(V: DevVal, DestTy: CGF.Int64Ty, /*isSigned=*/true);
10998 } else {
10999 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11000 }
11001 return DeviceID;
11002}
11003
11004static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>
11005emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) {
11006 llvm::Value *DynGP = CGF.Builder.getInt32(C: 0);
11007 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11008
11009 if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) {
11010 CodeGenFunction::RunCleanupsScope DynGPScope(CGF);
11011 llvm::Value *DynGPVal =
11012 CGF.EmitScalarExpr(E: DynGPClause->getSize(), /*IgnoreResultAssign=*/true);
11013 DynGP = CGF.Builder.CreateIntCast(V: DynGPVal, DestTy: CGF.Int32Ty,
11014 /*isSigned=*/false);
11015 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();
11016 switch (FallbackModifier) {
11017 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:
11018 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11019 break;
11020 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:
11021 DynGPFallback = OMPDynGroupprivateFallbackType::Null;
11022 break;
11023 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:
11024 case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown:
11025 // This is the default for dyn_groupprivate.
11026 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;
11027 break;
11028 default:
11029 llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate");
11030 }
11031 } else if (auto *OMPXDynCGClause =
11032 D.getSingleClause<OMPXDynCGroupMemClause>()) {
11033 CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF);
11034 llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(E: OMPXDynCGClause->getSize(),
11035 /*IgnoreResultAssign=*/true);
11036 DynGP = CGF.Builder.CreateIntCast(V: DynCGMemVal, DestTy: CGF.Int32Ty,
11037 /*isSigned=*/false);
11038 }
11039 return {DynGP, DynGPFallback};
11040}
11041
11042static void genMapInfoForCaptures(
11043 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
11044 const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11045 llvm::OpenMPIRBuilder &OMPBuilder,
11046 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet,
11047 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11048
11049 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
11050 auto RI = CS.getCapturedRecordDecl()->field_begin();
11051 auto *CV = CapturedVars.begin();
11052 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
11053 CE = CS.capture_end();
11054 CI != CE; ++CI, ++RI, ++CV) {
11055 MappableExprsHandler::MapCombinedInfoTy CurInfo;
11056
11057 // VLA sizes are passed to the outlined region by copy and do not have map
11058 // information associated.
11059 if (CI->capturesVariableArrayType()) {
11060 CurInfo.Exprs.push_back(Elt: nullptr);
11061 CurInfo.BasePointers.push_back(Elt: *CV);
11062 CurInfo.DevicePtrDecls.push_back(Elt: nullptr);
11063 CurInfo.DevicePointers.push_back(
11064 Elt: MappableExprsHandler::DeviceInfoTy::None);
11065 CurInfo.Pointers.push_back(Elt: *CV);
11066 CurInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
11067 V: CGF.getTypeSize(Ty: RI->getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
11068 // Copy to the device as an argument. No need to retrieve it.
11069 CurInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
11070 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11071 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
11072 CurInfo.HasAttachPtr.push_back(Elt: false);
11073 CurInfo.Mappers.push_back(Elt: nullptr);
11074 } else {
11075 const ValueDecl *CapturedVD =
11076 CI->capturesThis() ? nullptr
11077 : CI->getCapturedVar()->getCanonicalDecl();
11078 bool HasEntryWithCVAsAttachPtr = false;
11079 if (CapturedVD)
11080 HasEntryWithCVAsAttachPtr =
11081 MEHandler.hasAttachEntryForCapturedVar(VD: CapturedVD);
11082
11083 // Populate component lists for the captured variable from clauses.
11084 MappableExprsHandler::MapDataArrayTy DeclComponentLists;
11085 SmallVector<
11086 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>, 4>
11087 StorageForImplicitlyAddedComponentLists;
11088 MEHandler.populateComponentListsForNonLambdaCaptureFromClauses(
11089 VD: CapturedVD, DeclComponentLists,
11090 StorageForImplicitlyAddedComponentLists);
11091
11092 // OpenMP 6.0, 15.8, target construct, restrictions:
11093 // * A list item in a map clause that is specified on a target construct
11094 // must have a base variable or base pointer.
11095 //
11096 // Map clauses on a target construct must either have a base pointer, or a
11097 // base-variable. So, if we don't have a base-pointer, that means that it
11098 // must have a base-variable, i.e. we have a map like `map(s)`, `map(s.x)`
11099 // etc. In such cases, we do not need to handle default map generation
11100 // for `s`.
11101 bool HasEntryWithoutAttachPtr =
11102 llvm::any_of(Range&: DeclComponentLists, P: [&](const auto &MapData) {
11103 OMPClauseMappableExprCommon::MappableExprComponentListRef
11104 Components = std::get<0>(MapData);
11105 return !MEHandler.getAttachPtrExpr(Components);
11106 });
11107
11108 // Generate default map info first if there's no direct map with CV as
11109 // the base-variable, or attach pointer.
11110 if (DeclComponentLists.empty() ||
11111 (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr))
11112 MEHandler.generateDefaultMapInfo(CI: *CI, RI: **RI, CV: *CV, CombinedInfo&: CurInfo);
11113
11114 // If we have any information in the map clause, we use it, otherwise we
11115 // just do a default mapping.
11116 MEHandler.generateInfoForCaptureFromClauseInfo(
11117 DeclComponentListsFromClauses: DeclComponentLists, Cap: CI, Arg: *CV, CurCaptureVarInfo&: CurInfo, OMPBuilder,
11118 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());
11119
11120 if (!CI->capturesThis())
11121 MappedVarSet.insert(V: CI->getCapturedVar());
11122 else
11123 MappedVarSet.insert(V: nullptr);
11124
11125 // Generate correct mapping for variables captured by reference in
11126 // lambdas.
11127 if (CI->capturesVariable())
11128 MEHandler.generateInfoForLambdaCaptures(VD: CI->getCapturedVar(), Arg: *CV,
11129 CombinedInfo&: CurInfo, LambdaPointers);
11130 }
11131 // We expect to have at least an element of information for this capture.
11132 assert(!CurInfo.BasePointers.empty() &&
11133 "Non-existing map pointer for capture!");
11134 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
11135 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
11136 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
11137 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
11138 "Inconsistent map information sizes!");
11139
11140 // We need to append the results of this capture to what we already have.
11141 CombinedInfo.append(CurInfo);
11142 }
11143 // Adjust MEMBER_OF flags for the lambdas captures.
11144 MEHandler.adjustMemberOfForLambdaCaptures(
11145 OMPBuilder, LambdaPointers, BasePointers&: CombinedInfo.BasePointers,
11146 Pointers&: CombinedInfo.Pointers, Types&: CombinedInfo.Types);
11147}
11148static void
11149genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
11150 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
11151 llvm::OpenMPIRBuilder &OMPBuilder,
11152 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet =
11153 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) {
11154
11155 CodeGenModule &CGM = CGF.CGM;
11156 // Map any list items in a map clause that were not captures because they
11157 // weren't referenced within the construct.
11158 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkipVarSet: SkippedVarSet);
11159
11160 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11161 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
11162 };
11163 if (CGM.getCodeGenOpts().getDebugInfo() !=
11164 llvm::codegenoptions::NoDebugInfo) {
11165 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
11166 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
11167 F: FillInfoMap);
11168 }
11169}
11170
11171static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
11172 const CapturedStmt &CS,
11173 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11174 llvm::OpenMPIRBuilder &OMPBuilder,
11175 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11176 // Get mappable expression information.
11177 MappableExprsHandler MEHandler(D, CGF);
11178 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
11179
11180 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,
11181 MappedVarSet, CombinedInfo);
11182 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, SkippedVarSet: MappedVarSet);
11183}
11184
11185template <typename ClauseTy>
11186static void
11187emitClauseForBareTargetDirective(CodeGenFunction &CGF,
11188 const OMPExecutableDirective &D,
11189 llvm::SmallVectorImpl<llvm::Value *> &Values) {
11190 const auto *C = D.getSingleClause<ClauseTy>();
11191 assert(!C->varlist_empty() &&
11192 "ompx_bare requires explicit num_teams and thread_limit");
11193 CodeGenFunction::RunCleanupsScope Scope(CGF);
11194 for (auto *E : C->varlist()) {
11195 llvm::Value *V = CGF.EmitScalarExpr(E);
11196 Values.push_back(
11197 Elt: CGF.Builder.CreateIntCast(V, DestTy: CGF.Int32Ty, /*isSigned=*/true));
11198 }
11199}
11200
11201static void emitTargetCallKernelLaunch(
11202 CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11203 const OMPExecutableDirective &D,
11204 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask,
11205 const CapturedStmt &CS, bool OffloadingMandatory,
11206 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11207 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
11208 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
11209 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11210 const OMPLoopDirective &D)>
11211 SizeEmitter,
11212 CodeGenFunction &CGF, CodeGenModule &CGM) {
11213 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();
11214
11215 // Fill up the arrays with all the captured variables.
11216 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11217 CGOpenMPRuntime::TargetDataInfo Info;
11218 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
11219
11220 // Append a null entry for the implicit dyn_ptr argument.
11221 using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
11222 auto *NullPtr = llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy());
11223 CombinedInfo.BasePointers.push_back(Elt: NullPtr);
11224 CombinedInfo.Pointers.push_back(Elt: NullPtr);
11225 CombinedInfo.DevicePointers.push_back(
11226 Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
11227 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.getInt64(C: 0));
11228 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11229 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
11230 CombinedInfo.HasAttachPtr.push_back(Elt: false);
11231 if (!CombinedInfo.Names.empty())
11232 CombinedInfo.Names.push_back(Elt: NullPtr);
11233 CombinedInfo.Exprs.push_back(Elt: nullptr);
11234 CombinedInfo.Mappers.push_back(Elt: nullptr);
11235 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
11236
11237 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11238 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11239
11240 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11241 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11242 CGF.VoidPtrTy, CGM.getPointerAlign());
11243 InputInfo.PointersArray =
11244 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11245 InputInfo.SizesArray =
11246 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11247 InputInfo.MappersArray =
11248 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11249 MapTypesArray = Info.RTArgs.MapTypesArray;
11250 MapNamesArray = Info.RTArgs.MapNamesArray;
11251
11252 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
11253 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11254 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
11255 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
11256 bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
11257
11258 if (IsReverseOffloading) {
11259 // Reverse offloading is not supported, so just execute on the host.
11260 // FIXME: This fallback solution is incorrect since it ignores the
11261 // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to
11262 // assert here and ensure SEMA emits an error.
11263 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11264 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11265 return;
11266 }
11267
11268 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
11269 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
11270
11271 llvm::Value *BasePointersArray =
11272 InputInfo.BasePointersArray.emitRawPointer(CGF);
11273 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
11274 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
11275 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
11276
11277 auto &&EmitTargetCallFallbackCB =
11278 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11279 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
11280 -> llvm::OpenMPIRBuilder::InsertPointTy {
11281 CGF.Builder.restoreIP(IP);
11282 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11283 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11284 return CGF.Builder.saveIP();
11285 };
11286
11287 bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
11288 SmallVector<llvm::Value *, 3> NumTeams;
11289 SmallVector<llvm::Value *, 3> NumThreads;
11290 if (IsBare) {
11291 emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, Values&: NumTeams);
11292 emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
11293 Values&: NumThreads);
11294 } else {
11295 NumTeams.push_back(Elt: OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
11296 NumThreads.push_back(
11297 Elt: OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
11298 }
11299
11300 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
11301 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11302 llvm::Value *NumIterations =
11303 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
11304 auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF);
11305 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
11306 CGF.AllocaInsertPt->getIterator());
11307
11308 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
11309 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
11310 nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray);
11311
11312 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
11313 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
11314 DynCGroupMem, HasNoWait, /*StrictBlocks=*/IsBare,
11315 /*StrictThreads=*/IsBare, DynCGroupMemFallback);
11316
11317 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11318 cantFail(ValOrErr: OMPRuntime->getOMPBuilder().emitKernelLaunch(
11319 Loc: CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
11320 RTLoc, AllocaIP));
11321 CGF.Builder.restoreIP(IP: AfterIP);
11322 };
11323
11324 if (RequiresOuterTask)
11325 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11326 else
11327 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11328}
11329
11330static void
11331emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11332 const OMPExecutableDirective &D,
11333 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11334 bool RequiresOuterTask, const CapturedStmt &CS,
11335 bool OffloadingMandatory, CodeGenFunction &CGF) {
11336
11337 // Notify that the host version must be executed.
11338 auto &&ElseGen =
11339 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11340 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11341 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11342 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11343 };
11344
11345 if (RequiresOuterTask) {
11346 CodeGenFunction::OMPTargetDataInfo InputInfo;
11347 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ElseGen, InputInfo);
11348 } else {
11349 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ElseGen);
11350 }
11351}
11352
11353void CGOpenMPRuntime::emitTargetCall(
11354 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11355 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11356 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11357 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11358 const OMPLoopDirective &D)>
11359 SizeEmitter) {
11360 if (!CGF.HaveInsertPoint())
11361 return;
11362
11363 const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice &&
11364 CGM.getLangOpts().OpenMPOffloadMandatory;
11365
11366 assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
11367
11368 const bool RequiresOuterTask =
11369 D.hasClausesOfKind<OMPDependClause>() ||
11370 D.hasClausesOfKind<OMPNowaitClause>() ||
11371 D.hasClausesOfKind<OMPInReductionClause>() ||
11372 (CGM.getLangOpts().OpenMP >= 51 &&
11373 needsTaskBasedThreadLimit(DKind: D.getDirectiveKind()) &&
11374 D.hasClausesOfKind<OMPThreadLimitClause>());
11375 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
11376 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
11377 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
11378 PrePostActionTy &) {
11379 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
11380 };
11381 emitInlinedDirective(CGF, InnerKind: OMPD_unknown, CodeGen: ArgsCodegen);
11382
11383 CodeGenFunction::OMPTargetDataInfo InputInfo;
11384 llvm::Value *MapTypesArray = nullptr;
11385 llvm::Value *MapNamesArray = nullptr;
11386
11387 auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,
11388 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11389 OutlinedFnID, &InputInfo, &MapTypesArray,
11390 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
11391 PrePostActionTy &) {
11392 emitTargetCallKernelLaunch(OMPRuntime: this, OutlinedFn, D, CapturedVars,
11393 RequiresOuterTask, CS, OffloadingMandatory,
11394 Device, OutlinedFnID, InputInfo, MapTypesArray,
11395 MapNamesArray, SizeEmitter, CGF, CGM);
11396 };
11397
11398 auto &&TargetElseGen =
11399 [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11400 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11401 emitTargetCallElse(OMPRuntime: this, OutlinedFn, D, CapturedVars, RequiresOuterTask,
11402 CS, OffloadingMandatory, CGF);
11403 };
11404
11405 // If we have a target function ID it means that we need to support
11406 // offloading, otherwise, just execute on the host. We need to execute on host
11407 // regardless of the conditional in the if clause if, e.g., the user do not
11408 // specify target triples.
11409 if (OutlinedFnID) {
11410 if (IfCond) {
11411 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen, ElseGen: TargetElseGen);
11412 } else {
11413 RegionCodeGenTy ThenRCG(TargetThenGen);
11414 ThenRCG(CGF);
11415 }
11416 } else {
11417 RegionCodeGenTy ElseRCG(TargetElseGen);
11418 ElseRCG(CGF);
11419 }
11420}
11421
11422void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
11423 StringRef ParentName) {
11424 if (!S)
11425 return;
11426
11427 // Register vtable from device for target data and target directives.
11428 // Add this block here since scanForTargetRegionsFunctions ignores
11429 // target data by checking if S is a executable directive (target).
11430 if (auto *E = dyn_cast<OMPExecutableDirective>(Val: S);
11431 E && isOpenMPTargetDataManagementDirective(DKind: E->getDirectiveKind())) {
11432 // Don't need to check if it's device compile
11433 // since scanForTargetRegionsFunctions currently only called
11434 // in device compilation.
11435 registerVTable(D: *E);
11436 }
11437
11438 // Codegen OMP target directives that offload compute to the device.
11439 bool RequiresDeviceCodegen =
11440 isa<OMPExecutableDirective>(Val: S) &&
11441 isOpenMPTargetExecutionDirective(
11442 DKind: cast<OMPExecutableDirective>(Val: S)->getDirectiveKind());
11443
11444 if (RequiresDeviceCodegen) {
11445 const auto &E = *cast<OMPExecutableDirective>(Val: S);
11446
11447 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
11448 CGM, OMPBuilder, BeginLoc: E.getBeginLoc(), ParentName);
11449
11450 // Is this a target region that should not be emitted as an entry point? If
11451 // so just signal we are done with this target region.
11452 if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
11453 return;
11454
11455 switch (E.getDirectiveKind()) {
11456 case OMPD_target:
11457 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
11458 S: cast<OMPTargetDirective>(Val: E));
11459 break;
11460 case OMPD_target_parallel:
11461 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
11462 CGM, ParentName, S: cast<OMPTargetParallelDirective>(Val: E));
11463 break;
11464 case OMPD_target_teams:
11465 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
11466 CGM, ParentName, S: cast<OMPTargetTeamsDirective>(Val: E));
11467 break;
11468 case OMPD_target_teams_distribute:
11469 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
11470 CGM, ParentName, S: cast<OMPTargetTeamsDistributeDirective>(Val: E));
11471 break;
11472 case OMPD_target_teams_distribute_simd:
11473 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
11474 CGM, ParentName, S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: E));
11475 break;
11476 case OMPD_target_parallel_for:
11477 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
11478 CGM, ParentName, S: cast<OMPTargetParallelForDirective>(Val: E));
11479 break;
11480 case OMPD_target_parallel_for_simd:
11481 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
11482 CGM, ParentName, S: cast<OMPTargetParallelForSimdDirective>(Val: E));
11483 break;
11484 case OMPD_target_simd:
11485 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
11486 CGM, ParentName, S: cast<OMPTargetSimdDirective>(Val: E));
11487 break;
11488 case OMPD_target_teams_distribute_parallel_for:
11489 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
11490 CGM, ParentName,
11491 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: E));
11492 break;
11493 case OMPD_target_teams_distribute_parallel_for_simd:
11494 CodeGenFunction::
11495 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
11496 CGM, ParentName,
11497 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: E));
11498 break;
11499 case OMPD_target_teams_loop:
11500 CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
11501 CGM, ParentName, S: cast<OMPTargetTeamsGenericLoopDirective>(Val: E));
11502 break;
11503 case OMPD_target_parallel_loop:
11504 CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
11505 CGM, ParentName, S: cast<OMPTargetParallelGenericLoopDirective>(Val: E));
11506 break;
11507 case OMPD_parallel:
11508 case OMPD_for:
11509 case OMPD_parallel_for:
11510 case OMPD_parallel_master:
11511 case OMPD_parallel_sections:
11512 case OMPD_for_simd:
11513 case OMPD_parallel_for_simd:
11514 case OMPD_cancel:
11515 case OMPD_cancellation_point:
11516 case OMPD_ordered_standalone:
11517 case OMPD_ordered_blockassoc:
11518 case OMPD_threadprivate:
11519 case OMPD_allocate:
11520 case OMPD_task:
11521 case OMPD_simd:
11522 case OMPD_tile:
11523 case OMPD_unroll:
11524 case OMPD_sections:
11525 case OMPD_section:
11526 case OMPD_single:
11527 case OMPD_master:
11528 case OMPD_critical:
11529 case OMPD_taskyield:
11530 case OMPD_barrier:
11531 case OMPD_taskwait:
11532 case OMPD_taskgroup:
11533 case OMPD_atomic:
11534 case OMPD_flush:
11535 case OMPD_depobj:
11536 case OMPD_scan:
11537 case OMPD_teams:
11538 case OMPD_target_data:
11539 case OMPD_target_exit_data:
11540 case OMPD_target_enter_data:
11541 case OMPD_distribute:
11542 case OMPD_distribute_simd:
11543 case OMPD_distribute_parallel_for:
11544 case OMPD_distribute_parallel_for_simd:
11545 case OMPD_teams_distribute:
11546 case OMPD_teams_distribute_simd:
11547 case OMPD_teams_distribute_parallel_for:
11548 case OMPD_teams_distribute_parallel_for_simd:
11549 case OMPD_target_update:
11550 case OMPD_declare_simd:
11551 case OMPD_declare_variant:
11552 case OMPD_begin_declare_variant:
11553 case OMPD_end_declare_variant:
11554 case OMPD_declare_target:
11555 case OMPD_end_declare_target:
11556 case OMPD_declare_reduction:
11557 case OMPD_declare_mapper:
11558 case OMPD_taskloop:
11559 case OMPD_taskloop_simd:
11560 case OMPD_master_taskloop:
11561 case OMPD_master_taskloop_simd:
11562 case OMPD_parallel_master_taskloop:
11563 case OMPD_parallel_master_taskloop_simd:
11564 case OMPD_requires:
11565 case OMPD_metadirective:
11566 case OMPD_unknown:
11567 default:
11568 llvm_unreachable("Unknown target directive for OpenMP device codegen.");
11569 }
11570 return;
11571 }
11572
11573 if (const auto *E = dyn_cast<OMPExecutableDirective>(Val: S)) {
11574 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
11575 return;
11576
11577 scanForTargetRegionsFunctions(S: E->getRawStmt(), ParentName);
11578 return;
11579 }
11580
11581 // If this is a lambda function, look into its body.
11582 if (const auto *L = dyn_cast<LambdaExpr>(Val: S))
11583 S = L->getBody();
11584
11585 // Keep looking for target regions recursively.
11586 for (const Stmt *II : S->children())
11587 scanForTargetRegionsFunctions(S: II, ParentName);
11588}
11589
11590static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) {
11591 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
11592 OMPDeclareTargetDeclAttr::getDeviceType(VD);
11593 if (!DevTy)
11594 return false;
11595 // Do not emit device_type(nohost) functions for the host.
11596 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
11597 return true;
11598 // Do not emit device_type(host) functions for the device.
11599 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
11600 return true;
11601 return false;
11602}
11603
11604bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
11605 // If emitting code for the host, we do not process FD here. Instead we do
11606 // the normal code generation.
11607 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
11608 if (const auto *FD = dyn_cast<FunctionDecl>(Val: GD.getDecl()))
11609 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11610 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11611 return true;
11612 return false;
11613 }
11614
11615 const ValueDecl *VD = cast<ValueDecl>(Val: GD.getDecl());
11616 // Try to detect target regions in the function.
11617 if (const auto *FD = dyn_cast<FunctionDecl>(Val: VD)) {
11618 StringRef Name = CGM.getMangledName(GD);
11619 scanForTargetRegionsFunctions(S: FD->getBody(), ParentName: Name);
11620 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11621 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11622 return true;
11623 }
11624
11625 // Do not emit function if it is not marked as declare target.
11626 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
11627 AlreadyEmittedTargetDecls.count(V: VD) == 0;
11628}
11629
11630bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
11631 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: GD.getDecl()),
11632 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11633 return true;
11634
11635 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
11636 return false;
11637
11638 // Check if there are Ctors/Dtors in this declaration and look for target
11639 // regions in it. We use the complete variant to produce the kernel name
11640 // mangling.
11641 QualType RDTy = cast<VarDecl>(Val: GD.getDecl())->getType();
11642 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
11643 for (const CXXConstructorDecl *Ctor : RD->ctors()) {
11644 StringRef ParentName =
11645 CGM.getMangledName(GD: GlobalDecl(Ctor, Ctor_Complete));
11646 scanForTargetRegionsFunctions(S: Ctor->getBody(), ParentName);
11647 }
11648 if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
11649 StringRef ParentName =
11650 CGM.getMangledName(GD: GlobalDecl(Dtor, Dtor_Complete));
11651 scanForTargetRegionsFunctions(S: Dtor->getBody(), ParentName);
11652 }
11653 }
11654
11655 // Do not emit variable if it is not marked as declare target.
11656 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11657 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
11658 VD: cast<VarDecl>(Val: GD.getDecl()));
11659 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
11660 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11661 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11662 HasRequiresUnifiedSharedMemory)) {
11663 DeferredGlobalVariables.insert(V: cast<VarDecl>(Val: GD.getDecl()));
11664 return true;
11665 }
11666 return false;
11667}
11668
11669void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
11670 llvm::Constant *Addr) {
11671 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
11672 !CGM.getLangOpts().OpenMPIsTargetDevice)
11673 return;
11674
11675 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11676 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11677
11678 // If this is an 'extern' declaration we defer to the canonical definition and
11679 // do not emit an offloading entry.
11680 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
11681 VD->hasExternalStorage())
11682 return;
11683
11684 // MT_Local variables use direct access with no host-device mapping.
11685 // No offload entry needed — the device global keeps its own initializer.
11686 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local)
11687 return;
11688
11689 if (!Res) {
11690 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11691 // Register non-target variables being emitted in device code (debug info
11692 // may cause this).
11693 StringRef VarName = CGM.getMangledName(GD: VD);
11694 EmittedNonTargetVariables.try_emplace(Key: VarName, Args&: Addr);
11695 }
11696 return;
11697 }
11698
11699 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
11700 auto LinkageForVariable = [&VD, this]() {
11701 return CGM.getLLVMLinkageVarDefinition(VD);
11702 };
11703
11704 std::vector<llvm::GlobalVariable *> GeneratedRefs;
11705 OMPBuilder.registerTargetGlobalVariable(
11706 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
11707 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
11708 IsExternallyVisible: VD->isExternallyVisible(),
11709 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
11710 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
11711 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
11712 TargetTriple: CGM.getLangOpts().OMPTargetTriples, GlobalInitializer: AddrOfGlobal, VariableLinkage: LinkageForVariable,
11713 LlvmPtrTy: CGM.getTypes().ConvertTypeForMem(
11714 T: CGM.getContext().getPointerType(T: VD->getType())),
11715 Addr);
11716
11717 for (auto *ref : GeneratedRefs)
11718 CGM.addCompilerUsedGlobal(GV: ref);
11719}
11720
11721bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
11722 if (isa<FunctionDecl>(Val: GD.getDecl()) ||
11723 isa<OMPDeclareReductionDecl>(Val: GD.getDecl()))
11724 return emitTargetFunctions(GD);
11725
11726 return emitTargetGlobalVariable(GD);
11727}
11728
11729void CGOpenMPRuntime::emitDeferredTargetDecls() const {
11730 for (const VarDecl *VD : DeferredGlobalVariables) {
11731 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11732 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11733 if (!Res)
11734 continue;
11735 // MT_Local and MT_To/MT_Enter without USM are always emitted.
11736 if (*Res == OMPDeclareTargetDeclAttr::MT_Local ||
11737 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11738 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11739 !HasRequiresUnifiedSharedMemory)) {
11740 CGM.EmitGlobal(D: VD);
11741 } else {
11742 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
11743 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11744 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
11745 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
11746 HasRequiresUnifiedSharedMemory)) &&
11747 "Expected link clause or to clause with unified memory.");
11748 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
11749 }
11750 }
11751}
11752
11753void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
11754 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
11755 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
11756 " Expected target-based directive.");
11757}
11758
11759void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) {
11760 for (const OMPClause *Clause : D->clauselists()) {
11761 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
11762 HasRequiresUnifiedSharedMemory = true;
11763 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
11764 } else if (const auto *AC =
11765 dyn_cast<OMPAtomicDefaultMemOrderClause>(Val: Clause)) {
11766 switch (AC->getAtomicDefaultMemOrderKind()) {
11767 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
11768 RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;
11769 break;
11770 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
11771 RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
11772 break;
11773 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
11774 RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
11775 break;
11776 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown:
11777 break;
11778 }
11779 }
11780 }
11781}
11782
11783llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {
11784 return RequiresAtomicOrdering;
11785}
11786
11787bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
11788 LangAS &AS) {
11789 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
11790 return false;
11791 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
11792 switch(A->getAllocatorType()) {
11793 case OMPAllocateDeclAttr::OMPNullMemAlloc:
11794 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
11795 // Not supported, fallback to the default mem space.
11796 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
11797 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
11798 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
11799 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
11800 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
11801 case OMPAllocateDeclAttr::OMPConstMemAlloc:
11802 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
11803 AS = LangAS::Default;
11804 return true;
11805 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
11806 llvm_unreachable("Expected predefined allocator for the variables with the "
11807 "static storage.");
11808 }
11809 return false;
11810}
11811
11812bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {
11813 return HasRequiresUnifiedSharedMemory;
11814}
11815
11816CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
11817 CodeGenModule &CGM)
11818 : CGM(CGM) {
11819 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11820 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
11821 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
11822 }
11823}
11824
11825CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
11826 if (CGM.getLangOpts().OpenMPIsTargetDevice)
11827 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
11828}
11829
11830bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
11831 if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal)
11832 return true;
11833
11834 const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
11835 // Do not emit function if it is marked as declare target as it was already
11836 // emitted.
11837 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: D)) {
11838 if (D->hasBody() && AlreadyEmittedTargetDecls.count(V: D) == 0) {
11839 if (auto *F = dyn_cast_or_null<llvm::Function>(
11840 Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD))))
11841 return !F->isDeclaration();
11842 return false;
11843 }
11844 return true;
11845 }
11846
11847 return !AlreadyEmittedTargetDecls.insert(V: D).second;
11848}
11849
11850void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
11851 const OMPExecutableDirective &D,
11852 SourceLocation Loc,
11853 llvm::Function *OutlinedFn,
11854 ArrayRef<llvm::Value *> CapturedVars) {
11855 if (!CGF.HaveInsertPoint())
11856 return;
11857
11858 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11859 CodeGenFunction::RunCleanupsScope Scope(CGF);
11860
11861 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
11862 llvm::Value *Args[] = {
11863 RTLoc,
11864 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
11865 OutlinedFn};
11866 llvm::SmallVector<llvm::Value *, 16> RealArgs;
11867 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
11868 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
11869
11870 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
11871 M&: CGM.getModule(), FnID: OMPRTL___kmpc_fork_teams);
11872 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
11873}
11874
11875void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11876 const Expr *NumTeams,
11877 const Expr *ThreadLimit,
11878 SourceLocation Loc) {
11879 if (!CGF.HaveInsertPoint())
11880 return;
11881
11882 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11883
11884 llvm::Value *NumTeamsVal =
11885 NumTeams
11886 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: NumTeams),
11887 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11888 : CGF.Builder.getInt32(C: 0);
11889
11890 llvm::Value *ThreadLimitVal =
11891 ThreadLimit
11892 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11893 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11894 : CGF.Builder.getInt32(C: 0);
11895
11896 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
11897 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
11898 ThreadLimitVal};
11899 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11900 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_num_teams),
11901 args: PushNumTeamsArgs);
11902}
11903
11904void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
11905 const Expr *ThreadLimit,
11906 SourceLocation Loc) {
11907 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11908 llvm::Value *ThreadLimitVal =
11909 ThreadLimit
11910 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11911 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11912 : CGF.Builder.getInt32(C: 0);
11913
11914 // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit)
11915 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
11916 ThreadLimitVal};
11917 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11918 M&: CGM.getModule(), FnID: OMPRTL___kmpc_set_thread_limit),
11919 args: ThreadLimitArgs);
11920}
11921
11922void CGOpenMPRuntime::emitTargetDataCalls(
11923 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11924 const Expr *Device, const RegionCodeGenTy &CodeGen,
11925 CGOpenMPRuntime::TargetDataInfo &Info) {
11926 if (!CGF.HaveInsertPoint())
11927 return;
11928
11929 // Action used to replace the default codegen action and turn privatization
11930 // off.
11931 PrePostActionTy NoPrivAction;
11932
11933 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
11934
11935 llvm::Value *IfCondVal = nullptr;
11936 if (IfCond)
11937 IfCondVal = CGF.EvaluateExprAsBool(E: IfCond);
11938
11939 // Emit device ID if any.
11940 llvm::Value *DeviceID = nullptr;
11941 if (Device) {
11942 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
11943 DestTy: CGF.Int64Ty, /*isSigned=*/true);
11944 } else {
11945 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11946 }
11947
11948 // Fill up the arrays with all the mapped variables.
11949 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11950 auto GenMapInfoCB =
11951 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
11952 CGF.Builder.restoreIP(IP: CodeGenIP);
11953 // Get map clause information.
11954 MappableExprsHandler MEHandler(D, CGF);
11955 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);
11956
11957 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11958 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
11959 };
11960 if (CGM.getCodeGenOpts().getDebugInfo() !=
11961 llvm::codegenoptions::NoDebugInfo) {
11962 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
11963 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
11964 F: FillInfoMap);
11965 }
11966
11967 return CombinedInfo;
11968 };
11969 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
11970 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
11971 CGF.Builder.restoreIP(IP: CodeGenIP);
11972 switch (BodyGenType) {
11973 case BodyGenTy::Priv:
11974 if (!Info.CaptureDeviceAddrMap.empty())
11975 CodeGen(CGF);
11976 break;
11977 case BodyGenTy::DupNoPriv:
11978 if (!Info.CaptureDeviceAddrMap.empty()) {
11979 CodeGen.setAction(NoPrivAction);
11980 CodeGen(CGF);
11981 }
11982 break;
11983 case BodyGenTy::NoPriv:
11984 if (Info.CaptureDeviceAddrMap.empty()) {
11985 CodeGen.setAction(NoPrivAction);
11986 CodeGen(CGF);
11987 }
11988 break;
11989 }
11990 return InsertPointTy(CGF.Builder.GetInsertPoint());
11991 };
11992
11993 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
11994 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
11995 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
11996 }
11997 };
11998
11999 auto CustomMapperCB = [&](unsigned int I) {
12000 llvm::Function *MFunc = nullptr;
12001 if (CombinedInfo.Mappers[I]) {
12002 Info.HasMapper = true;
12003 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
12004 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
12005 }
12006 return MFunc;
12007 };
12008
12009 // Source location for the ident struct
12010 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
12011
12012 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
12013 InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
12014 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.Builder);
12015 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
12016 cantFail(ValOrErr: OMPBuilder.createTargetData(
12017 Loc: OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
12018 IfCond: IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
12019 /*MapperFunc=*/nullptr, BodyGenCB: BodyCB, DeviceAddrCB, SrcLocInfo: RTLoc));
12020 CGF.Builder.restoreIP(IP: AfterIP);
12021}
12022
12023void CGOpenMPRuntime::emitTargetDataStandAloneCall(
12024 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
12025 const Expr *Device) {
12026 if (!CGF.HaveInsertPoint())
12027 return;
12028
12029 assert((isa<OMPTargetEnterDataDirective>(D) ||
12030 isa<OMPTargetExitDataDirective>(D) ||
12031 isa<OMPTargetUpdateDirective>(D)) &&
12032 "Expecting either target enter, exit data, or update directives.");
12033
12034 CodeGenFunction::OMPTargetDataInfo InputInfo;
12035 llvm::Value *MapTypesArray = nullptr;
12036 llvm::Value *MapNamesArray = nullptr;
12037 // Generate the code for the opening of the data environment.
12038 auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,
12039 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
12040 // Emit device ID if any.
12041 llvm::Value *DeviceID = nullptr;
12042 if (Device) {
12043 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
12044 DestTy: CGF.Int64Ty, /*isSigned=*/true);
12045 } else {
12046 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
12047 }
12048
12049 // Emit the number of elements in the offloading arrays.
12050 llvm::Constant *PointerNum =
12051 CGF.Builder.getInt32(C: InputInfo.NumberOfTargetItems);
12052
12053 // Source location for the ident struct
12054 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
12055
12056 SmallVector<llvm::Value *, 13> OffloadingArgs(
12057 {RTLoc, DeviceID, PointerNum,
12058 InputInfo.BasePointersArray.emitRawPointer(CGF),
12059 InputInfo.PointersArray.emitRawPointer(CGF),
12060 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
12061 InputInfo.MappersArray.emitRawPointer(CGF)});
12062
12063 // Select the right runtime function call for each standalone
12064 // directive.
12065 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
12066 RuntimeFunction RTLFn;
12067 switch (D.getDirectiveKind()) {
12068 case OMPD_target_enter_data:
12069 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
12070 : OMPRTL___tgt_target_data_begin_mapper;
12071 break;
12072 case OMPD_target_exit_data:
12073 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
12074 : OMPRTL___tgt_target_data_end_mapper;
12075 break;
12076 case OMPD_target_update:
12077 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
12078 : OMPRTL___tgt_target_data_update_mapper;
12079 break;
12080 case OMPD_parallel:
12081 case OMPD_for:
12082 case OMPD_parallel_for:
12083 case OMPD_parallel_master:
12084 case OMPD_parallel_sections:
12085 case OMPD_for_simd:
12086 case OMPD_parallel_for_simd:
12087 case OMPD_cancel:
12088 case OMPD_cancellation_point:
12089 case OMPD_ordered_standalone:
12090 case OMPD_ordered_blockassoc:
12091 case OMPD_threadprivate:
12092 case OMPD_allocate:
12093 case OMPD_task:
12094 case OMPD_simd:
12095 case OMPD_tile:
12096 case OMPD_unroll:
12097 case OMPD_sections:
12098 case OMPD_section:
12099 case OMPD_single:
12100 case OMPD_master:
12101 case OMPD_critical:
12102 case OMPD_taskyield:
12103 case OMPD_barrier:
12104 case OMPD_taskwait:
12105 case OMPD_taskgroup:
12106 case OMPD_atomic:
12107 case OMPD_flush:
12108 case OMPD_depobj:
12109 case OMPD_scan:
12110 case OMPD_teams:
12111 case OMPD_target_data:
12112 case OMPD_distribute:
12113 case OMPD_distribute_simd:
12114 case OMPD_distribute_parallel_for:
12115 case OMPD_distribute_parallel_for_simd:
12116 case OMPD_teams_distribute:
12117 case OMPD_teams_distribute_simd:
12118 case OMPD_teams_distribute_parallel_for:
12119 case OMPD_teams_distribute_parallel_for_simd:
12120 case OMPD_declare_simd:
12121 case OMPD_declare_variant:
12122 case OMPD_begin_declare_variant:
12123 case OMPD_end_declare_variant:
12124 case OMPD_declare_target:
12125 case OMPD_end_declare_target:
12126 case OMPD_declare_reduction:
12127 case OMPD_declare_mapper:
12128 case OMPD_taskloop:
12129 case OMPD_taskloop_simd:
12130 case OMPD_master_taskloop:
12131 case OMPD_master_taskloop_simd:
12132 case OMPD_parallel_master_taskloop:
12133 case OMPD_parallel_master_taskloop_simd:
12134 case OMPD_target:
12135 case OMPD_target_simd:
12136 case OMPD_target_teams_distribute:
12137 case OMPD_target_teams_distribute_simd:
12138 case OMPD_target_teams_distribute_parallel_for:
12139 case OMPD_target_teams_distribute_parallel_for_simd:
12140 case OMPD_target_teams:
12141 case OMPD_target_parallel:
12142 case OMPD_target_parallel_for:
12143 case OMPD_target_parallel_for_simd:
12144 case OMPD_requires:
12145 case OMPD_metadirective:
12146 case OMPD_unknown:
12147 default:
12148 llvm_unreachable("Unexpected standalone target data directive.");
12149 break;
12150 }
12151 if (HasNowait) {
12152 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
12153 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
12154 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
12155 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
12156 }
12157 CGF.EmitRuntimeCall(
12158 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID: RTLFn),
12159 args: OffloadingArgs);
12160 };
12161
12162 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
12163 &MapNamesArray](CodeGenFunction &CGF,
12164 PrePostActionTy &) {
12165 // Fill up the arrays with all the mapped variables.
12166 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
12167 CGOpenMPRuntime::TargetDataInfo Info;
12168 MappableExprsHandler MEHandler(D, CGF);
12169 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
12170 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
12171 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
12172
12173 bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
12174 D.hasClausesOfKind<OMPNowaitClause>();
12175
12176 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
12177 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
12178 CGF.VoidPtrTy, CGM.getPointerAlign());
12179 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
12180 CGM.getPointerAlign());
12181 InputInfo.SizesArray =
12182 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
12183 InputInfo.MappersArray =
12184 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
12185 MapTypesArray = Info.RTArgs.MapTypesArray;
12186 MapNamesArray = Info.RTArgs.MapNamesArray;
12187 if (RequiresOuterTask)
12188 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
12189 else
12190 emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
12191 };
12192
12193 if (IfCond) {
12194 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen,
12195 ElseGen: [](CodeGenFunction &CGF, PrePostActionTy &) {});
12196 } else {
12197 RegionCodeGenTy ThenRCG(TargetThenGen);
12198 ThenRCG(CGF);
12199 }
12200}
12201
12202static unsigned
12203evaluateCDTSize(const FunctionDecl *FD,
12204 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12205 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
12206 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
12207 // of that clause. The VLEN value must be power of 2.
12208 // In other case the notion of the function`s "characteristic data type" (CDT)
12209 // is used to compute the vector length.
12210 // CDT is defined in the following order:
12211 // a) For non-void function, the CDT is the return type.
12212 // b) If the function has any non-uniform, non-linear parameters, then the
12213 // CDT is the type of the first such parameter.
12214 // c) If the CDT determined by a) or b) above is struct, union, or class
12215 // type which is pass-by-value (except for the type that maps to the
12216 // built-in complex data type), the characteristic data type is int.
12217 // d) If none of the above three cases is applicable, the CDT is int.
12218 // The VLEN is then determined based on the CDT and the size of vector
12219 // register of that ISA for which current vector version is generated. The
12220 // VLEN is computed using the formula below:
12221 // VLEN = sizeof(vector_register) / sizeof(CDT),
12222 // where vector register size specified in section 3.2.1 Registers and the
12223 // Stack Frame of original AMD64 ABI document.
12224 QualType RetType = FD->getReturnType();
12225 if (RetType.isNull())
12226 return 0;
12227 ASTContext &C = FD->getASTContext();
12228 QualType CDT;
12229 if (!RetType.isNull() && !RetType->isVoidType()) {
12230 CDT = RetType;
12231 } else {
12232 unsigned Offset = 0;
12233 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD)) {
12234 if (ParamAttrs[Offset].Kind ==
12235 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector)
12236 CDT = C.getPointerType(T: C.getCanonicalTagType(TD: MD->getParent()));
12237 ++Offset;
12238 }
12239 if (CDT.isNull()) {
12240 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12241 if (ParamAttrs[I + Offset].Kind ==
12242 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) {
12243 CDT = FD->getParamDecl(i: I)->getType();
12244 break;
12245 }
12246 }
12247 }
12248 }
12249 if (CDT.isNull())
12250 CDT = C.IntTy;
12251 CDT = CDT->getCanonicalTypeUnqualified();
12252 if (CDT->isRecordType() || CDT->isUnionType())
12253 CDT = C.IntTy;
12254 return C.getTypeSize(T: CDT);
12255}
12256
12257// This are the Functions that are needed to mangle the name of the
12258// vector functions generated by the compiler, according to the rules
12259// defined in the "Vector Function ABI specifications for AArch64",
12260// available at
12261// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
12262
12263/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
12264static bool getAArch64MTV(QualType QT,
12265 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) {
12266 QT = QT.getCanonicalType();
12267
12268 if (QT->isVoidType())
12269 return false;
12270
12271 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform)
12272 return false;
12273
12274 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12275 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)
12276 return false;
12277
12278 if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12279 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) &&
12280 !QT->isReferenceType())
12281 return false;
12282
12283 return true;
12284}
12285
12286/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
12287static bool getAArch64PBV(QualType QT, ASTContext &C) {
12288 QT = QT.getCanonicalType();
12289 unsigned Size = C.getTypeSize(T: QT);
12290
12291 // Only scalars and complex within 16 bytes wide set PVB to true.
12292 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
12293 return false;
12294
12295 if (QT->isFloatingType())
12296 return true;
12297
12298 if (QT->isIntegerType())
12299 return true;
12300
12301 if (QT->isPointerType())
12302 return true;
12303
12304 // TODO: Add support for complex types (section 3.1.2, item 2).
12305
12306 return false;
12307}
12308
12309/// Computes the lane size (LS) of a return type or of an input parameter,
12310/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
12311/// TODO: Add support for references, section 3.2.1, item 1.
12312static unsigned getAArch64LS(QualType QT,
12313 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind,
12314 ASTContext &C) {
12315 if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
12316 QualType PTy = QT.getCanonicalType()->getPointeeType();
12317 if (getAArch64PBV(QT: PTy, C))
12318 return C.getTypeSize(T: PTy);
12319 }
12320 if (getAArch64PBV(QT, C))
12321 return C.getTypeSize(T: QT);
12322
12323 return C.getTypeSize(T: C.getUIntPtrType());
12324}
12325
12326// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
12327// signature of the scalar function, as defined in 3.2.2 of the
12328// AAVFABI.
12329static std::tuple<unsigned, unsigned, bool>
12330getNDSWDS(const FunctionDecl *FD,
12331 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12332 QualType RetType = FD->getReturnType().getCanonicalType();
12333
12334 ASTContext &C = FD->getASTContext();
12335
12336 bool OutputBecomesInput = false;
12337
12338 llvm::SmallVector<unsigned, 8> Sizes;
12339 if (!RetType->isVoidType()) {
12340 Sizes.push_back(Elt: getAArch64LS(
12341 QT: RetType, Kind: llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C));
12342 if (!getAArch64PBV(QT: RetType, C) && getAArch64MTV(QT: RetType, Kind: {}))
12343 OutputBecomesInput = true;
12344 }
12345 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12346 QualType QT = FD->getParamDecl(i: I)->getType().getCanonicalType();
12347 Sizes.push_back(Elt: getAArch64LS(QT, Kind: ParamAttrs[I].Kind, C));
12348 }
12349
12350 assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
12351 // The LS of a function parameter / return value can only be a power
12352 // of 2, starting from 8 bits, up to 128.
12353 assert(llvm::all_of(Sizes,
12354 [](unsigned Size) {
12355 return Size == 8 || Size == 16 || Size == 32 ||
12356 Size == 64 || Size == 128;
12357 }) &&
12358 "Invalid size");
12359
12360 return std::make_tuple(args&: *llvm::min_element(Range&: Sizes), args&: *llvm::max_element(Range&: Sizes),
12361 args&: OutputBecomesInput);
12362}
12363
12364static llvm::OpenMPIRBuilder::DeclareSimdBranch
12365convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) {
12366 switch (State) {
12367 case OMPDeclareSimdDeclAttr::BS_Undefined:
12368 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined;
12369 case OMPDeclareSimdDeclAttr::BS_Inbranch:
12370 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch;
12371 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
12372 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch;
12373 }
12374 llvm_unreachable("unexpected declare simd branch state");
12375}
12376
12377// Check the values provided via `simdlen` by the user.
12378static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc,
12379 unsigned UserVLEN, unsigned WDS, char ISA) {
12380 // 1. A `simdlen(1)` doesn't produce vector signatures.
12381 if (UserVLEN == 1) {
12382 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_1_no_effect);
12383 return false;
12384 }
12385
12386 // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced
12387 // SIMD.
12388 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(Value: UserVLEN)) {
12389 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_requires_power_of_2);
12390 return false;
12391 }
12392
12393 // 3. Section 3.4.1: SVE fixed length must obey the architectural limits.
12394 if (ISA == 's' && UserVLEN != 0 &&
12395 ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) {
12396 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_must_fit_lanes) << WDS;
12397 return false;
12398 }
12399
12400 return true;
12401}
12402
12403void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
12404 llvm::Function *Fn) {
12405 ASTContext &C = CGM.getContext();
12406 FD = FD->getMostRecentDecl();
12407 while (FD) {
12408 // Map params to their positions in function decl.
12409 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
12410 if (isa<CXXMethodDecl>(Val: FD))
12411 ParamPositions.try_emplace(Key: FD, Args: 0);
12412 unsigned ParamPos = ParamPositions.size();
12413 for (const ParmVarDecl *P : FD->parameters()) {
12414 ParamPositions.try_emplace(Key: P->getCanonicalDecl(), Args&: ParamPos);
12415 ++ParamPos;
12416 }
12417 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
12418 llvm::SmallVector<llvm::OpenMPIRBuilder::DeclareSimdAttrTy, 8> ParamAttrs(
12419 ParamPositions.size());
12420 // Mark uniform parameters.
12421 for (const Expr *E : Attr->uniforms()) {
12422 E = E->IgnoreParenImpCasts();
12423 unsigned Pos;
12424 if (isa<CXXThisExpr>(Val: E)) {
12425 Pos = ParamPositions[FD];
12426 } else {
12427 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12428 ->getCanonicalDecl();
12429 auto It = ParamPositions.find(Val: PVD);
12430 assert(It != ParamPositions.end() && "Function parameter not found");
12431 Pos = It->second;
12432 }
12433 ParamAttrs[Pos].Kind =
12434 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform;
12435 }
12436 // Get alignment info.
12437 auto *NI = Attr->alignments_begin();
12438 for (const Expr *E : Attr->aligneds()) {
12439 E = E->IgnoreParenImpCasts();
12440 unsigned Pos;
12441 QualType ParmTy;
12442 if (isa<CXXThisExpr>(Val: E)) {
12443 Pos = ParamPositions[FD];
12444 ParmTy = E->getType();
12445 } else {
12446 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12447 ->getCanonicalDecl();
12448 auto It = ParamPositions.find(Val: PVD);
12449 assert(It != ParamPositions.end() && "Function parameter not found");
12450 Pos = It->second;
12451 ParmTy = PVD->getType();
12452 }
12453 ParamAttrs[Pos].Alignment =
12454 (*NI)
12455 ? (*NI)->EvaluateKnownConstInt(Ctx: C)
12456 : llvm::APSInt::getUnsigned(
12457 X: C.toCharUnitsFromBits(BitSize: C.getOpenMPDefaultSimdAlign(T: ParmTy))
12458 .getQuantity());
12459 ++NI;
12460 }
12461 // Mark linear parameters.
12462 auto *SI = Attr->steps_begin();
12463 auto *MI = Attr->modifiers_begin();
12464 for (const Expr *E : Attr->linears()) {
12465 E = E->IgnoreParenImpCasts();
12466 unsigned Pos;
12467 bool IsReferenceType = false;
12468 // Rescaling factor needed to compute the linear parameter
12469 // value in the mangled name.
12470 unsigned PtrRescalingFactor = 1;
12471 if (isa<CXXThisExpr>(Val: E)) {
12472 Pos = ParamPositions[FD];
12473 auto *P = cast<PointerType>(Val: E->getType());
12474 PtrRescalingFactor = CGM.getContext()
12475 .getTypeSizeInChars(T: P->getPointeeType())
12476 .getQuantity();
12477 } else {
12478 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12479 ->getCanonicalDecl();
12480 auto It = ParamPositions.find(Val: PVD);
12481 assert(It != ParamPositions.end() && "Function parameter not found");
12482 Pos = It->second;
12483 if (auto *P = dyn_cast<PointerType>(Val: PVD->getType()))
12484 PtrRescalingFactor = CGM.getContext()
12485 .getTypeSizeInChars(T: P->getPointeeType())
12486 .getQuantity();
12487 else if (PVD->getType()->isReferenceType()) {
12488 IsReferenceType = true;
12489 PtrRescalingFactor =
12490 CGM.getContext()
12491 .getTypeSizeInChars(T: PVD->getType().getNonReferenceType())
12492 .getQuantity();
12493 }
12494 }
12495 llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos];
12496 if (*MI == OMPC_LINEAR_ref)
12497 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef;
12498 else if (*MI == OMPC_LINEAR_uval)
12499 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal;
12500 else if (IsReferenceType)
12501 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal;
12502 else
12503 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear;
12504 // Assuming a stride of 1, for `linear` without modifiers.
12505 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: 1);
12506 if (*SI) {
12507 Expr::EvalResult Result;
12508 if (!(*SI)->EvaluateAsInt(Result, Ctx: C, AllowSideEffects: Expr::SE_AllowSideEffects)) {
12509 if (const auto *DRE =
12510 cast<DeclRefExpr>(Val: (*SI)->IgnoreParenImpCasts())) {
12511 if (const auto *StridePVD =
12512 dyn_cast<ParmVarDecl>(Val: DRE->getDecl())) {
12513 ParamAttr.HasVarStride = true;
12514 auto It = ParamPositions.find(Val: StridePVD->getCanonicalDecl());
12515 assert(It != ParamPositions.end() &&
12516 "Function parameter not found");
12517 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: It->second);
12518 }
12519 }
12520 } else {
12521 ParamAttr.StrideOrArg = Result.Val.getInt();
12522 }
12523 }
12524 // If we are using a linear clause on a pointer, we need to
12525 // rescale the value of linear_step with the byte size of the
12526 // pointee type.
12527 if (!ParamAttr.HasVarStride &&
12528 (ParamAttr.Kind ==
12529 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12530 ParamAttr.Kind ==
12531 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef))
12532 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
12533 ++SI;
12534 ++MI;
12535 }
12536 llvm::APSInt VLENVal;
12537 SourceLocation ExprLoc;
12538 const Expr *VLENExpr = Attr->getSimdlen();
12539 if (VLENExpr) {
12540 VLENVal = VLENExpr->EvaluateKnownConstInt(Ctx: C);
12541 ExprLoc = VLENExpr->getExprLoc();
12542 }
12543 llvm::OpenMPIRBuilder::DeclareSimdBranch State =
12544 convertDeclareSimdBranch(State: Attr->getBranchState());
12545 if (CGM.getTriple().isX86()) {
12546 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
12547 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12548 OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElements: NumElts, VLENVal, ParamAttrs,
12549 Branch: State);
12550 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
12551 unsigned VLEN = VLENVal.getExtValue();
12552 // Get basic data for building the vector signature.
12553 const auto Data = getNDSWDS(FD, ParamAttrs);
12554 const unsigned NDS = std::get<0>(t: Data);
12555 const unsigned WDS = std::get<1>(t: Data);
12556 const bool OutputBecomesInput = std::get<2>(t: Data);
12557 if (CGM.getTarget().hasFeature(Feature: "sve")) {
12558 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 's'))
12559 OMPBuilder.emitAArch64DeclareSimdFunction(
12560 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 's', NarrowestDataSize: NDS, OutputBecomesInput);
12561 } else if (CGM.getTarget().hasFeature(Feature: "neon")) {
12562 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 'n'))
12563 OMPBuilder.emitAArch64DeclareSimdFunction(
12564 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 'n', NarrowestDataSize: NDS, OutputBecomesInput);
12565 }
12566 }
12567 }
12568 FD = FD->getPreviousDecl();
12569 }
12570}
12571
12572namespace {
12573/// Cleanup action for doacross support.
12574class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
12575public:
12576 static const int DoacrossFinArgs = 2;
12577
12578private:
12579 llvm::FunctionCallee RTLFn;
12580 llvm::Value *Args[DoacrossFinArgs];
12581
12582public:
12583 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
12584 ArrayRef<llvm::Value *> CallArgs)
12585 : RTLFn(RTLFn) {
12586 assert(CallArgs.size() == DoacrossFinArgs);
12587 std::copy(first: CallArgs.begin(), last: CallArgs.end(), result: std::begin(arr&: Args));
12588 }
12589 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12590 if (!CGF.HaveInsertPoint())
12591 return;
12592 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12593 }
12594};
12595} // namespace
12596
12597void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
12598 const OMPLoopDirective &D,
12599 ArrayRef<Expr *> NumIterations) {
12600 if (!CGF.HaveInsertPoint())
12601 return;
12602
12603 ASTContext &C = CGM.getContext();
12604 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
12605 RecordDecl *RD;
12606 if (KmpDimTy.isNull()) {
12607 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
12608 // kmp_int64 lo; // lower
12609 // kmp_int64 up; // upper
12610 // kmp_int64 st; // stride
12611 // };
12612 RD = C.buildImplicitRecord(Name: "kmp_dim");
12613 RD->startDefinition();
12614 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12615 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12616 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12617 RD->completeDefinition();
12618 KmpDimTy = C.getCanonicalTagType(TD: RD);
12619 } else {
12620 RD = KmpDimTy->castAsRecordDecl();
12621 }
12622 llvm::APInt Size(/*numBits=*/32, NumIterations.size());
12623 QualType ArrayTy = C.getConstantArrayType(EltTy: KmpDimTy, ArySize: Size, SizeExpr: nullptr,
12624 ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12625
12626 Address DimsAddr = CGF.CreateMemTemp(T: ArrayTy, Name: "dims");
12627 CGF.EmitNullInitialization(DestPtr: DimsAddr, Ty: ArrayTy);
12628 enum { LowerFD = 0, UpperFD, StrideFD };
12629 // Fill dims with data.
12630 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
12631 LValue DimsLVal = CGF.MakeAddrLValue(
12632 Addr: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: I), T: KmpDimTy);
12633 // dims.upper = num_iterations;
12634 LValue UpperLVal = CGF.EmitLValueForField(
12635 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: UpperFD));
12636 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
12637 Src: CGF.EmitScalarExpr(E: NumIterations[I]), SrcTy: NumIterations[I]->getType(),
12638 DstTy: Int64Ty, Loc: NumIterations[I]->getExprLoc());
12639 CGF.EmitStoreOfScalar(value: NumIterVal, lvalue: UpperLVal);
12640 // dims.stride = 1;
12641 LValue StrideLVal = CGF.EmitLValueForField(
12642 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: StrideFD));
12643 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::getSigned(Ty: CGM.Int64Ty, /*V=*/1),
12644 lvalue: StrideLVal);
12645 }
12646
12647 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
12648 // kmp_int32 num_dims, struct kmp_dim * dims);
12649 llvm::Value *Args[] = {
12650 emitUpdateLocation(CGF, Loc: D.getBeginLoc()),
12651 getThreadID(CGF, Loc: D.getBeginLoc()),
12652 llvm::ConstantInt::getSigned(Ty: CGM.Int32Ty, V: NumIterations.size()),
12653 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12654 V: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: 0).emitRawPointer(CGF),
12655 DestTy: CGM.VoidPtrTy)};
12656
12657 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12658 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_init);
12659 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12660 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
12661 emitUpdateLocation(CGF, Loc: D.getEndLoc()), getThreadID(CGF, Loc: D.getEndLoc())};
12662 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12663 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_fini);
12664 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(Kind: NormalAndEHCleanup, A: FiniRTLFn,
12665 A: llvm::ArrayRef(FiniArgs));
12666}
12667
12668template <typename T>
12669static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
12670 const T *C, llvm::Value *ULoc,
12671 llvm::Value *ThreadID) {
12672 QualType Int64Ty =
12673 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
12674 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
12675 QualType ArrayTy = CGM.getContext().getConstantArrayType(
12676 EltTy: Int64Ty, ArySize: Size, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12677 Address CntAddr = CGF.CreateMemTemp(T: ArrayTy, Name: ".cnt.addr");
12678 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
12679 const Expr *CounterVal = C->getLoopData(I);
12680 assert(CounterVal);
12681 llvm::Value *CntVal = CGF.EmitScalarConversion(
12682 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
12683 Loc: CounterVal->getExprLoc());
12684 CGF.EmitStoreOfScalar(Value: CntVal, Addr: CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: I),
12685 /*Volatile=*/false, Ty: Int64Ty);
12686 }
12687 llvm::Value *Args[] = {
12688 ULoc, ThreadID,
12689 CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: 0).emitRawPointer(CGF)};
12690 llvm::FunctionCallee RTLFn;
12691 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
12692 OMPDoacrossKind<T> ODK;
12693 if (ODK.isSource(C)) {
12694 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12695 FnID: OMPRTL___kmpc_doacross_post);
12696 } else {
12697 assert(ODK.isSink(C) && "Expect sink modifier.");
12698 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12699 FnID: OMPRTL___kmpc_doacross_wait);
12700 }
12701 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12702}
12703
12704void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12705 const OMPDependClause *C) {
12706 return EmitDoacrossOrdered<OMPDependClause>(
12707 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12708 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12709}
12710
12711void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12712 const OMPDoacrossClause *C) {
12713 return EmitDoacrossOrdered<OMPDoacrossClause>(
12714 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12715 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12716}
12717
12718void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
12719 llvm::FunctionCallee Callee,
12720 ArrayRef<llvm::Value *> Args) const {
12721 assert(Loc.isValid() && "Outlined function call location must be valid.");
12722 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
12723
12724 if (auto *Fn = dyn_cast<llvm::Function>(Val: Callee.getCallee())) {
12725 if (Fn->doesNotThrow()) {
12726 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
12727 return;
12728 }
12729 }
12730 CGF.EmitRuntimeCall(callee: Callee, args: Args);
12731}
12732
12733void CGOpenMPRuntime::emitOutlinedFunctionCall(
12734 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
12735 ArrayRef<llvm::Value *> Args) const {
12736 emitCall(CGF, Loc, Callee: OutlinedFn, Args);
12737}
12738
12739void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
12740 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D))
12741 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: FD))
12742 HasEmittedDeclareTargetRegion = true;
12743}
12744
12745Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
12746 const VarDecl *NativeParam,
12747 const VarDecl *TargetParam) const {
12748 return CGF.GetAddrOfLocalVar(VD: NativeParam);
12749}
12750
12751/// Return allocator value from expression, or return a null allocator (default
12752/// when no allocator specified).
12753static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
12754 const Expr *Allocator) {
12755 llvm::Value *AllocVal;
12756 if (Allocator) {
12757 AllocVal = CGF.EmitScalarExpr(E: Allocator);
12758 // According to the standard, the original allocator type is a enum
12759 // (integer). Convert to pointer type, if required.
12760 AllocVal = CGF.EmitScalarConversion(Src: AllocVal, SrcTy: Allocator->getType(),
12761 DstTy: CGF.getContext().VoidPtrTy,
12762 Loc: Allocator->getExprLoc());
12763 } else {
12764 // If no allocator specified, it defaults to the null allocator.
12765 AllocVal = llvm::Constant::getNullValue(
12766 Ty: CGF.CGM.getTypes().ConvertType(T: CGF.getContext().VoidPtrTy));
12767 }
12768 return AllocVal;
12769}
12770
12771/// Return the alignment from an allocate directive if present.
12772static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {
12773 std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);
12774
12775 if (!AllocateAlignment)
12776 return nullptr;
12777
12778 return llvm::ConstantInt::get(Ty: CGM.SizeTy, V: AllocateAlignment->getQuantity());
12779}
12780
12781Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
12782 const VarDecl *VD) {
12783 if (!VD)
12784 return Address::invalid();
12785 Address UntiedAddr = Address::invalid();
12786 Address UntiedRealAddr = Address::invalid();
12787 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12788 if (It != FunctionToUntiedTaskStackMap.end()) {
12789 const UntiedLocalVarsAddressesMap &UntiedData =
12790 UntiedLocalVarsStack[It->second];
12791 auto I = UntiedData.find(Key: VD);
12792 if (I != UntiedData.end()) {
12793 UntiedAddr = I->second.first;
12794 UntiedRealAddr = I->second.second;
12795 }
12796 }
12797 const VarDecl *CVD = VD->getCanonicalDecl();
12798 if (CVD->hasAttr<OMPAllocateDeclAttr>()) {
12799 // Use the default allocation.
12800 if (!isAllocatableDecl(VD))
12801 return UntiedAddr;
12802 llvm::Value *Size;
12803 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
12804 if (CVD->getType()->isVariablyModifiedType()) {
12805 Size = CGF.getTypeSize(Ty: CVD->getType());
12806 // Align the size: ((size + align - 1) / align) * align
12807 Size = CGF.Builder.CreateNUWAdd(
12808 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
12809 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
12810 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
12811 } else {
12812 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
12813 Size = CGM.getSize(numChars: Sz.alignTo(Align));
12814 }
12815 llvm::Value *ThreadID = getThreadID(CGF, Loc: CVD->getBeginLoc());
12816 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
12817 const Expr *Allocator = AA->getAllocator();
12818 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
12819 llvm::Value *Alignment = getAlignmentValue(CGM, VD: CVD);
12820 SmallVector<llvm::Value *, 4> Args;
12821 Args.push_back(Elt: ThreadID);
12822 if (Alignment)
12823 Args.push_back(Elt: Alignment);
12824 Args.push_back(Elt: Size);
12825 Args.push_back(Elt: AllocVal);
12826 llvm::omp::RuntimeFunction FnID =
12827 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
12828 llvm::Value *Addr = CGF.EmitRuntimeCall(
12829 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args,
12830 name: getName(Parts: {CVD->getName(), ".void.addr"}));
12831 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12832 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free);
12833 QualType Ty = CGM.getContext().getPointerType(T: CVD->getType());
12834 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12835 V: Addr, DestTy: CGF.ConvertTypeForMem(T: Ty), Name: getName(Parts: {CVD->getName(), ".addr"}));
12836 if (UntiedAddr.isValid())
12837 CGF.EmitStoreOfScalar(Value: Addr, Addr: UntiedAddr, /*Volatile=*/false, Ty);
12838
12839 // Cleanup action for allocate support.
12840 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
12841 llvm::FunctionCallee RTLFn;
12842 SourceLocation::UIntTy LocEncoding;
12843 Address Addr;
12844 const Expr *AllocExpr;
12845
12846 public:
12847 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
12848 SourceLocation::UIntTy LocEncoding, Address Addr,
12849 const Expr *AllocExpr)
12850 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
12851 AllocExpr(AllocExpr) {}
12852 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12853 if (!CGF.HaveInsertPoint())
12854 return;
12855 llvm::Value *Args[3];
12856 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
12857 CGF, Loc: SourceLocation::getFromRawEncoding(Encoding: LocEncoding));
12858 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12859 V: Addr.emitRawPointer(CGF), DestTy: CGF.VoidPtrTy);
12860 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator: AllocExpr);
12861 Args[2] = AllocVal;
12862 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12863 }
12864 };
12865 Address VDAddr =
12866 UntiedRealAddr.isValid()
12867 ? UntiedRealAddr
12868 : Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
12869 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
12870 Kind: NormalAndEHCleanup, A: FiniRTLFn, A: CVD->getLocation().getRawEncoding(),
12871 A: VDAddr, A: Allocator);
12872 if (UntiedRealAddr.isValid())
12873 if (auto *Region =
12874 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
12875 Region->emitUntiedSwitch(CGF);
12876 return VDAddr;
12877 }
12878 return UntiedAddr;
12879}
12880
12881bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,
12882 const VarDecl *VD) const {
12883 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12884 if (It == FunctionToUntiedTaskStackMap.end())
12885 return false;
12886 return UntiedLocalVarsStack[It->second].count(Key: VD) > 0;
12887}
12888
12889CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII(
12890 CodeGenModule &CGM, const OMPLoopDirective &S)
12891 : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {
12892 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12893 if (!NeedToPush)
12894 return;
12895 NontemporalDeclsSet &DS =
12896 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12897 for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) {
12898 for (const Stmt *Ref : C->private_refs()) {
12899 const auto *SimpleRefExpr = cast<Expr>(Val: Ref)->IgnoreParenImpCasts();
12900 const ValueDecl *VD;
12901 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: SimpleRefExpr)) {
12902 VD = DRE->getDecl();
12903 } else {
12904 const auto *ME = cast<MemberExpr>(Val: SimpleRefExpr);
12905 assert((ME->isImplicitCXXThis() ||
12906 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
12907 "Expected member of current class.");
12908 VD = ME->getMemberDecl();
12909 }
12910 DS.insert(V: VD);
12911 }
12912 }
12913}
12914
12915CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() {
12916 if (!NeedToPush)
12917 return;
12918 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
12919}
12920
12921CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII(
12922 CodeGenFunction &CGF,
12923 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
12924 std::pair<Address, Address>> &LocalVars)
12925 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
12926 if (!NeedToPush)
12927 return;
12928 CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace(
12929 Key: CGF.CurFn, Args: CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
12930 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(Elt: LocalVars);
12931}
12932
12933CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() {
12934 if (!NeedToPush)
12935 return;
12936 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
12937}
12938
12939bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const {
12940 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12941
12942 return llvm::any_of(
12943 Range&: CGM.getOpenMPRuntime().NontemporalDeclsStack,
12944 P: [VD](const NontemporalDeclsSet &Set) { return Set.contains(V: VD); });
12945}
12946
12947void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
12948 const OMPExecutableDirective &S,
12949 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled)
12950 const {
12951 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
12952 // Vars in target/task regions must be excluded completely.
12953 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()) ||
12954 isOpenMPTaskingDirective(Kind: S.getDirectiveKind())) {
12955 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
12956 getOpenMPCaptureRegions(CaptureRegions, DKind: S.getDirectiveKind());
12957 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CaptureRegions.front());
12958 for (const CapturedStmt::Capture &Cap : CS->captures()) {
12959 if (Cap.capturesVariable() || Cap.capturesVariableByCopy())
12960 NeedToCheckForLPCs.insert(V: Cap.getCapturedVar());
12961 }
12962 }
12963 // Exclude vars in private clauses.
12964 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
12965 for (const Expr *Ref : C->varlist()) {
12966 if (!Ref->getType()->isScalarType())
12967 continue;
12968 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12969 if (!DRE)
12970 continue;
12971 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12972 }
12973 }
12974 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
12975 for (const Expr *Ref : C->varlist()) {
12976 if (!Ref->getType()->isScalarType())
12977 continue;
12978 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12979 if (!DRE)
12980 continue;
12981 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12982 }
12983 }
12984 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
12985 for (const Expr *Ref : C->varlist()) {
12986 if (!Ref->getType()->isScalarType())
12987 continue;
12988 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12989 if (!DRE)
12990 continue;
12991 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12992 }
12993 }
12994 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
12995 for (const Expr *Ref : C->varlist()) {
12996 if (!Ref->getType()->isScalarType())
12997 continue;
12998 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12999 if (!DRE)
13000 continue;
13001 NeedToCheckForLPCs.insert(V: DRE->getDecl());
13002 }
13003 }
13004 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
13005 for (const Expr *Ref : C->varlist()) {
13006 if (!Ref->getType()->isScalarType())
13007 continue;
13008 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
13009 if (!DRE)
13010 continue;
13011 NeedToCheckForLPCs.insert(V: DRE->getDecl());
13012 }
13013 }
13014 for (const Decl *VD : NeedToCheckForLPCs) {
13015 for (const LastprivateConditionalData &Data :
13016 llvm::reverse(C&: CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
13017 if (Data.DeclToUniqueName.count(Key: VD) > 0) {
13018 if (!Data.Disabled)
13019 NeedToAddForLPCsAsDisabled.insert(V: VD);
13020 break;
13021 }
13022 }
13023 }
13024}
13025
13026CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13027 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
13028 : CGM(CGF.CGM),
13029 Action((CGM.getLangOpts().OpenMP >= 50 &&
13030 llvm::any_of(Range: S.getClausesOfKind<OMPLastprivateClause>(),
13031 P: [](const OMPLastprivateClause *C) {
13032 return C->getKind() ==
13033 OMPC_LASTPRIVATE_conditional;
13034 }))
13035 ? ActionToDo::PushAsLastprivateConditional
13036 : ActionToDo::DoNotPush) {
13037 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13038 if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
13039 return;
13040 assert(Action == ActionToDo::PushAsLastprivateConditional &&
13041 "Expected a push action.");
13042 LastprivateConditionalData &Data =
13043 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
13044 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
13045 if (C->getKind() != OMPC_LASTPRIVATE_conditional)
13046 continue;
13047
13048 for (const Expr *Ref : C->varlist()) {
13049 Data.DeclToUniqueName.insert(KV: std::make_pair(
13050 x: cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts())->getDecl(),
13051 y: SmallString<16>(generateUniqueName(CGM, Prefix: "pl_cond", Ref))));
13052 }
13053 }
13054 Data.IVLVal = IVLVal;
13055 Data.Fn = CGF.CurFn;
13056}
13057
13058CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13059 CodeGenFunction &CGF, const OMPExecutableDirective &S)
13060 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
13061 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13062 if (CGM.getLangOpts().OpenMP < 50)
13063 return;
13064 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
13065 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
13066 if (!NeedToAddForLPCsAsDisabled.empty()) {
13067 Action = ActionToDo::DisableLastprivateConditional;
13068 LastprivateConditionalData &Data =
13069 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
13070 for (const Decl *VD : NeedToAddForLPCsAsDisabled)
13071 Data.DeclToUniqueName.try_emplace(Key: VD);
13072 Data.Fn = CGF.CurFn;
13073 Data.Disabled = true;
13074 }
13075}
13076
13077CGOpenMPRuntime::LastprivateConditionalRAII
13078CGOpenMPRuntime::LastprivateConditionalRAII::disable(
13079 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
13080 return LastprivateConditionalRAII(CGF, S);
13081}
13082
13083CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() {
13084 if (CGM.getLangOpts().OpenMP < 50)
13085 return;
13086 if (Action == ActionToDo::DisableLastprivateConditional) {
13087 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13088 "Expected list of disabled private vars.");
13089 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13090 }
13091 if (Action == ActionToDo::PushAsLastprivateConditional) {
13092 assert(
13093 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13094 "Expected list of lastprivate conditional vars.");
13095 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13096 }
13097}
13098
13099Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
13100 const VarDecl *VD) {
13101 ASTContext &C = CGM.getContext();
13102 auto I = LastprivateConditionalToTypes.try_emplace(Key: CGF.CurFn).first;
13103 QualType NewType;
13104 const FieldDecl *VDField;
13105 const FieldDecl *FiredField;
13106 LValue BaseLVal;
13107 auto VI = I->getSecond().find(Val: VD);
13108 if (VI == I->getSecond().end()) {
13109 RecordDecl *RD = C.buildImplicitRecord(Name: "lasprivate.conditional");
13110 RD->startDefinition();
13111 VDField = addFieldToRecordDecl(C, DC: RD, FieldTy: VD->getType().getNonReferenceType());
13112 FiredField = addFieldToRecordDecl(C, DC: RD, FieldTy: C.CharTy);
13113 RD->completeDefinition();
13114 NewType = C.getCanonicalTagType(TD: RD);
13115 Address Addr = CGF.CreateMemTemp(T: NewType, Align: C.getDeclAlign(D: VD), Name: VD->getName());
13116 BaseLVal = CGF.MakeAddrLValue(Addr, T: NewType, Source: AlignmentSource::Decl);
13117 I->getSecond().try_emplace(Key: VD, Args&: NewType, Args&: VDField, Args&: FiredField, Args&: BaseLVal);
13118 } else {
13119 NewType = std::get<0>(t&: VI->getSecond());
13120 VDField = std::get<1>(t&: VI->getSecond());
13121 FiredField = std::get<2>(t&: VI->getSecond());
13122 BaseLVal = std::get<3>(t&: VI->getSecond());
13123 }
13124 LValue FiredLVal =
13125 CGF.EmitLValueForField(Base: BaseLVal, Field: FiredField);
13126 CGF.EmitStoreOfScalar(
13127 value: llvm::ConstantInt::getNullValue(Ty: CGF.ConvertTypeForMem(T: C.CharTy)),
13128 lvalue: FiredLVal);
13129 return CGF.EmitLValueForField(Base: BaseLVal, Field: VDField).getAddress();
13130}
13131
13132namespace {
13133/// Checks if the lastprivate conditional variable is referenced in LHS.
13134class LastprivateConditionalRefChecker final
13135 : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {
13136 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM;
13137 const Expr *FoundE = nullptr;
13138 const Decl *FoundD = nullptr;
13139 StringRef UniqueDeclName;
13140 LValue IVLVal;
13141 llvm::Function *FoundFn = nullptr;
13142 SourceLocation Loc;
13143
13144public:
13145 bool VisitDeclRefExpr(const DeclRefExpr *E) {
13146 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
13147 llvm::reverse(C&: LPM)) {
13148 auto It = D.DeclToUniqueName.find(Key: E->getDecl());
13149 if (It == D.DeclToUniqueName.end())
13150 continue;
13151 if (D.Disabled)
13152 return false;
13153 FoundE = E;
13154 FoundD = E->getDecl()->getCanonicalDecl();
13155 UniqueDeclName = It->second;
13156 IVLVal = D.IVLVal;
13157 FoundFn = D.Fn;
13158 break;
13159 }
13160 return FoundE == E;
13161 }
13162 bool VisitMemberExpr(const MemberExpr *E) {
13163 if (!CodeGenFunction::IsWrappedCXXThis(E: E->getBase()))
13164 return false;
13165 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
13166 llvm::reverse(C&: LPM)) {
13167 auto It = D.DeclToUniqueName.find(Key: E->getMemberDecl());
13168 if (It == D.DeclToUniqueName.end())
13169 continue;
13170 if (D.Disabled)
13171 return false;
13172 FoundE = E;
13173 FoundD = E->getMemberDecl()->getCanonicalDecl();
13174 UniqueDeclName = It->second;
13175 IVLVal = D.IVLVal;
13176 FoundFn = D.Fn;
13177 break;
13178 }
13179 return FoundE == E;
13180 }
13181 bool VisitStmt(const Stmt *S) {
13182 for (const Stmt *Child : S->children()) {
13183 if (!Child)
13184 continue;
13185 if (const auto *E = dyn_cast<Expr>(Val: Child))
13186 if (!E->isGLValue())
13187 continue;
13188 if (Visit(S: Child))
13189 return true;
13190 }
13191 return false;
13192 }
13193 explicit LastprivateConditionalRefChecker(
13194 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
13195 : LPM(LPM) {}
13196 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
13197 getFoundData() const {
13198 return std::make_tuple(args: FoundE, args: FoundD, args: UniqueDeclName, args: IVLVal, args: FoundFn);
13199 }
13200};
13201} // namespace
13202
13203void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
13204 LValue IVLVal,
13205 StringRef UniqueDeclName,
13206 LValue LVal,
13207 SourceLocation Loc) {
13208 // Last updated loop counter for the lastprivate conditional var.
13209 // int<xx> last_iv = 0;
13210 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(T: IVLVal.getType());
13211 llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable(
13212 Ty: LLIVTy, Name: getName(Parts: {UniqueDeclName, "iv"}));
13213 cast<llvm::GlobalVariable>(Val: LastIV)->setAlignment(
13214 IVLVal.getAlignment().getAsAlign());
13215 LValue LastIVLVal =
13216 CGF.MakeNaturalAlignRawAddrLValue(V: LastIV, T: IVLVal.getType());
13217
13218 // Last value of the lastprivate conditional.
13219 // decltype(priv_a) last_a;
13220 llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
13221 Ty: CGF.ConvertTypeForMem(T: LVal.getType()), Name: UniqueDeclName);
13222 cast<llvm::GlobalVariable>(Val: Last)->setAlignment(
13223 LVal.getAlignment().getAsAlign());
13224 LValue LastLVal =
13225 CGF.MakeRawAddrLValue(V: Last, T: LVal.getType(), Alignment: LVal.getAlignment());
13226
13227 // Global loop counter. Required to handle inner parallel-for regions.
13228 // iv
13229 llvm::Value *IVVal = CGF.EmitLoadOfScalar(lvalue: IVLVal, Loc);
13230
13231 // #pragma omp critical(a)
13232 // if (last_iv <= iv) {
13233 // last_iv = iv;
13234 // last_a = priv_a;
13235 // }
13236 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
13237 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
13238 Action.Enter(CGF);
13239 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(lvalue: LastIVLVal, Loc);
13240 // (last_iv <= iv) ? Check if the variable is updated and store new
13241 // value in global var.
13242 llvm::Value *CmpRes;
13243 if (IVLVal.getType()->isSignedIntegerType()) {
13244 CmpRes = CGF.Builder.CreateICmpSLE(LHS: LastIVVal, RHS: IVVal);
13245 } else {
13246 assert(IVLVal.getType()->isUnsignedIntegerType() &&
13247 "Loop iteration variable must be integer.");
13248 CmpRes = CGF.Builder.CreateICmpULE(LHS: LastIVVal, RHS: IVVal);
13249 }
13250 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lp_cond_then");
13251 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "lp_cond_exit");
13252 CGF.Builder.CreateCondBr(Cond: CmpRes, True: ThenBB, False: ExitBB);
13253 // {
13254 CGF.EmitBlock(BB: ThenBB);
13255
13256 // last_iv = iv;
13257 CGF.EmitStoreOfScalar(value: IVVal, lvalue: LastIVLVal);
13258
13259 // last_a = priv_a;
13260 switch (CGF.getEvaluationKind(T: LVal.getType())) {
13261 case TEK_Scalar: {
13262 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
13263 CGF.EmitStoreOfScalar(value: PrivVal, lvalue: LastLVal);
13264 break;
13265 }
13266 case TEK_Complex: {
13267 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(src: LVal, loc: Loc);
13268 CGF.EmitStoreOfComplex(V: PrivVal, dest: LastLVal, /*isInit=*/false);
13269 break;
13270 }
13271 case TEK_Aggregate:
13272 llvm_unreachable(
13273 "Aggregates are not supported in lastprivate conditional.");
13274 }
13275 // }
13276 CGF.EmitBranch(Block: ExitBB);
13277 // There is no need to emit line number for unconditional branch.
13278 (void)ApplyDebugLocation::CreateEmpty(CGF);
13279 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
13280 };
13281
13282 if (CGM.getLangOpts().OpenMPSimd) {
13283 // Do not emit as a critical region as no parallel region could be emitted.
13284 RegionCodeGenTy ThenRCG(CodeGen);
13285 ThenRCG(CGF);
13286 } else {
13287 emitCriticalRegion(CGF, CriticalName: UniqueDeclName, CriticalOpGen: CodeGen, Loc);
13288 }
13289}
13290
13291void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
13292 const Expr *LHS) {
13293 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13294 return;
13295 LastprivateConditionalRefChecker Checker(LastprivateConditionalStack);
13296 if (!Checker.Visit(S: LHS))
13297 return;
13298 const Expr *FoundE;
13299 const Decl *FoundD;
13300 StringRef UniqueDeclName;
13301 LValue IVLVal;
13302 llvm::Function *FoundFn;
13303 std::tie(args&: FoundE, args&: FoundD, args&: UniqueDeclName, args&: IVLVal, args&: FoundFn) =
13304 Checker.getFoundData();
13305 if (FoundFn != CGF.CurFn) {
13306 // Special codegen for inner parallel regions.
13307 // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1;
13308 auto It = LastprivateConditionalToTypes[FoundFn].find(Val: FoundD);
13309 assert(It != LastprivateConditionalToTypes[FoundFn].end() &&
13310 "Lastprivate conditional is not found in outer region.");
13311 QualType StructTy = std::get<0>(t&: It->getSecond());
13312 const FieldDecl* FiredDecl = std::get<2>(t&: It->getSecond());
13313 LValue PrivLVal = CGF.EmitLValue(E: FoundE);
13314 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
13315 Addr: PrivLVal.getAddress(),
13316 Ty: CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: StructTy)),
13317 ElementTy: CGF.ConvertTypeForMem(T: StructTy));
13318 LValue BaseLVal =
13319 CGF.MakeAddrLValue(Addr: StructAddr, T: StructTy, Source: AlignmentSource::Decl);
13320 LValue FiredLVal = CGF.EmitLValueForField(Base: BaseLVal, Field: FiredDecl);
13321 CGF.EmitAtomicStore(rvalue: RValue::get(V: llvm::ConstantInt::get(
13322 Ty: CGF.ConvertTypeForMem(T: FiredDecl->getType()), V: 1)),
13323 lvalue: FiredLVal, AO: llvm::AtomicOrdering::Unordered,
13324 /*IsVolatile=*/true, /*isInit=*/false);
13325 return;
13326 }
13327
13328 // Private address of the lastprivate conditional in the current context.
13329 // priv_a
13330 LValue LVal = CGF.EmitLValue(E: FoundE);
13331 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
13332 Loc: FoundE->getExprLoc());
13333}
13334
13335void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional(
13336 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13337 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) {
13338 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13339 return;
13340 auto Range = llvm::reverse(C&: LastprivateConditionalStack);
13341 auto It = llvm::find_if(
13342 Range, P: [](const LastprivateConditionalData &D) { return !D.Disabled; });
13343 if (It == Range.end() || It->Fn != CGF.CurFn)
13344 return;
13345 auto LPCI = LastprivateConditionalToTypes.find(Val: It->Fn);
13346 assert(LPCI != LastprivateConditionalToTypes.end() &&
13347 "Lastprivates must be registered already.");
13348 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
13349 getOpenMPCaptureRegions(CaptureRegions, DKind: D.getDirectiveKind());
13350 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: CaptureRegions.back());
13351 for (const auto &Pair : It->DeclToUniqueName) {
13352 const auto *VD = cast<VarDecl>(Val: Pair.first->getCanonicalDecl());
13353 if (!CS->capturesVariable(Var: VD) || IgnoredDecls.contains(V: VD))
13354 continue;
13355 auto I = LPCI->getSecond().find(Val: Pair.first);
13356 assert(I != LPCI->getSecond().end() &&
13357 "Lastprivate must be rehistered already.");
13358 // bool Cmp = priv_a.Fired != 0;
13359 LValue BaseLVal = std::get<3>(t&: I->getSecond());
13360 LValue FiredLVal =
13361 CGF.EmitLValueForField(Base: BaseLVal, Field: std::get<2>(t&: I->getSecond()));
13362 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: FiredLVal, Loc: D.getBeginLoc());
13363 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Res);
13364 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lpc.then");
13365 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "lpc.done");
13366 // if (Cmp) {
13367 CGF.Builder.CreateCondBr(Cond: Cmp, True: ThenBB, False: DoneBB);
13368 CGF.EmitBlock(BB: ThenBB);
13369 Address Addr = CGF.GetAddrOfLocalVar(VD);
13370 LValue LVal;
13371 if (VD->getType()->isReferenceType())
13372 LVal = CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
13373 Source: AlignmentSource::Decl);
13374 else
13375 LVal = CGF.MakeAddrLValue(Addr, T: VD->getType().getNonReferenceType(),
13376 Source: AlignmentSource::Decl);
13377 emitLastprivateConditionalUpdate(CGF, IVLVal: It->IVLVal, UniqueDeclName: Pair.second, LVal,
13378 Loc: D.getBeginLoc());
13379 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
13380 CGF.EmitBlock(BB: DoneBB, /*IsFinal=*/IsFinished: true);
13381 // }
13382 }
13383}
13384
13385void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
13386 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
13387 SourceLocation Loc) {
13388 if (CGF.getLangOpts().OpenMP < 50)
13389 return;
13390 auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(Key: VD);
13391 assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() &&
13392 "Unknown lastprivate conditional variable.");
13393 StringRef UniqueName = It->second;
13394 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: UniqueName);
13395 // The variable was not updated in the region - exit.
13396 if (!GV)
13397 return;
13398 LValue LPLVal = CGF.MakeRawAddrLValue(
13399 V: GV, T: PrivLVal.getType().getNonReferenceType(), Alignment: PrivLVal.getAlignment());
13400 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: LPLVal, Loc);
13401 CGF.EmitStoreOfScalar(value: Res, lvalue: PrivLVal);
13402}
13403
13404llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
13405 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13406 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13407 const RegionCodeGenTy &CodeGen) {
13408 llvm_unreachable("Not supported in SIMD-only mode");
13409}
13410
13411llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
13412 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13413 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13414 const RegionCodeGenTy &CodeGen) {
13415 llvm_unreachable("Not supported in SIMD-only mode");
13416}
13417
13418llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
13419 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
13420 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
13421 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
13422 bool Tied, unsigned &NumberOfParts) {
13423 llvm_unreachable("Not supported in SIMD-only mode");
13424}
13425
13426void CGOpenMPSIMDRuntime::emitParallelCall(
13427 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
13428 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
13429 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
13430 OpenMPSeverityClauseKind Severity, const Expr *Message) {
13431 llvm_unreachable("Not supported in SIMD-only mode");
13432}
13433
13434void CGOpenMPSIMDRuntime::emitCriticalRegion(
13435 CodeGenFunction &CGF, StringRef CriticalName,
13436 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
13437 const Expr *Hint) {
13438 llvm_unreachable("Not supported in SIMD-only mode");
13439}
13440
13441void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
13442 const RegionCodeGenTy &MasterOpGen,
13443 SourceLocation Loc) {
13444 llvm_unreachable("Not supported in SIMD-only mode");
13445}
13446
13447void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,
13448 const RegionCodeGenTy &MasterOpGen,
13449 SourceLocation Loc,
13450 const Expr *Filter) {
13451 llvm_unreachable("Not supported in SIMD-only mode");
13452}
13453
13454void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
13455 SourceLocation Loc) {
13456 llvm_unreachable("Not supported in SIMD-only mode");
13457}
13458
13459void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
13460 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
13461 SourceLocation Loc) {
13462 llvm_unreachable("Not supported in SIMD-only mode");
13463}
13464
13465void CGOpenMPSIMDRuntime::emitSingleRegion(
13466 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
13467 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
13468 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
13469 ArrayRef<const Expr *> AssignmentOps) {
13470 llvm_unreachable("Not supported in SIMD-only mode");
13471}
13472
13473void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
13474 const RegionCodeGenTy &OrderedOpGen,
13475 SourceLocation Loc,
13476 bool IsThreads) {
13477 llvm_unreachable("Not supported in SIMD-only mode");
13478}
13479
13480void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
13481 SourceLocation Loc,
13482 OpenMPDirectiveKind Kind,
13483 bool EmitChecks,
13484 bool ForceSimpleCall) {
13485 llvm_unreachable("Not supported in SIMD-only mode");
13486}
13487
13488void CGOpenMPSIMDRuntime::emitForDispatchInit(
13489 CodeGenFunction &CGF, SourceLocation Loc,
13490 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
13491 bool Ordered, const DispatchRTInput &DispatchValues) {
13492 llvm_unreachable("Not supported in SIMD-only mode");
13493}
13494
13495void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
13496 SourceLocation Loc) {
13497 llvm_unreachable("Not supported in SIMD-only mode");
13498}
13499
13500void CGOpenMPSIMDRuntime::emitForStaticInit(
13501 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
13502 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
13503 llvm_unreachable("Not supported in SIMD-only mode");
13504}
13505
13506void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
13507 CodeGenFunction &CGF, SourceLocation Loc,
13508 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
13509 llvm_unreachable("Not supported in SIMD-only mode");
13510}
13511
13512void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
13513 SourceLocation Loc,
13514 unsigned IVSize,
13515 bool IVSigned) {
13516 llvm_unreachable("Not supported in SIMD-only mode");
13517}
13518
13519void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
13520 SourceLocation Loc,
13521 OpenMPDirectiveKind DKind) {
13522 llvm_unreachable("Not supported in SIMD-only mode");
13523}
13524
13525llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
13526 SourceLocation Loc,
13527 unsigned IVSize, bool IVSigned,
13528 Address IL, Address LB,
13529 Address UB, Address ST) {
13530 llvm_unreachable("Not supported in SIMD-only mode");
13531}
13532
13533void CGOpenMPSIMDRuntime::emitNumThreadsClause(
13534 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
13535 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
13536 SourceLocation SeverityLoc, const Expr *Message,
13537 SourceLocation MessageLoc) {
13538 llvm_unreachable("Not supported in SIMD-only mode");
13539}
13540
13541void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
13542 ProcBindKind ProcBind,
13543 SourceLocation Loc) {
13544 llvm_unreachable("Not supported in SIMD-only mode");
13545}
13546
13547Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
13548 const VarDecl *VD,
13549 Address VDAddr,
13550 SourceLocation Loc) {
13551 llvm_unreachable("Not supported in SIMD-only mode");
13552}
13553
13554llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
13555 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
13556 CodeGenFunction *CGF) {
13557 llvm_unreachable("Not supported in SIMD-only mode");
13558}
13559
13560Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
13561 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
13562 llvm_unreachable("Not supported in SIMD-only mode");
13563}
13564
13565void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
13566 ArrayRef<const Expr *> Vars,
13567 SourceLocation Loc,
13568 llvm::AtomicOrdering AO) {
13569 llvm_unreachable("Not supported in SIMD-only mode");
13570}
13571
13572void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
13573 const OMPExecutableDirective &D,
13574 llvm::Function *TaskFunction,
13575 QualType SharedsTy, Address Shareds,
13576 const Expr *IfCond,
13577 const OMPTaskDataTy &Data) {
13578 llvm_unreachable("Not supported in SIMD-only mode");
13579}
13580
13581void CGOpenMPSIMDRuntime::emitTaskLoopCall(
13582 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
13583 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
13584 const Expr *IfCond, const OMPTaskDataTy &Data) {
13585 llvm_unreachable("Not supported in SIMD-only mode");
13586}
13587
13588void CGOpenMPSIMDRuntime::emitReduction(
13589 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
13590 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
13591 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
13592 assert(Options.SimpleReduction && "Only simple reduction is expected.");
13593 CGOpenMPRuntime::emitReduction(CGF, Loc, OrgPrivates: Privates, OrgLHSExprs: LHSExprs, OrgRHSExprs: RHSExprs,
13594 OrgReductionOps: ReductionOps, Options);
13595}
13596
13597llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
13598 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
13599 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
13600 llvm_unreachable("Not supported in SIMD-only mode");
13601}
13602
13603void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
13604 SourceLocation Loc,
13605 bool IsWorksharingReduction) {
13606 llvm_unreachable("Not supported in SIMD-only mode");
13607}
13608
13609void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
13610 SourceLocation Loc,
13611 ReductionCodeGen &RCG,
13612 unsigned N) {
13613 llvm_unreachable("Not supported in SIMD-only mode");
13614}
13615
13616Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
13617 SourceLocation Loc,
13618 llvm::Value *ReductionsPtr,
13619 LValue SharedLVal) {
13620 llvm_unreachable("Not supported in SIMD-only mode");
13621}
13622
13623void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
13624 SourceLocation Loc,
13625 const OMPTaskDataTy &Data) {
13626 llvm_unreachable("Not supported in SIMD-only mode");
13627}
13628
13629void CGOpenMPSIMDRuntime::emitCancellationPointCall(
13630 CodeGenFunction &CGF, SourceLocation Loc,
13631 OpenMPDirectiveKind CancelRegion) {
13632 llvm_unreachable("Not supported in SIMD-only mode");
13633}
13634
13635void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
13636 SourceLocation Loc, const Expr *IfCond,
13637 OpenMPDirectiveKind CancelRegion) {
13638 llvm_unreachable("Not supported in SIMD-only mode");
13639}
13640
13641void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
13642 const OMPExecutableDirective &D, StringRef ParentName,
13643 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
13644 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
13645 llvm_unreachable("Not supported in SIMD-only mode");
13646}
13647
13648void CGOpenMPSIMDRuntime::emitTargetCall(
13649 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13650 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
13651 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
13652 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
13653 const OMPLoopDirective &D)>
13654 SizeEmitter) {
13655 llvm_unreachable("Not supported in SIMD-only mode");
13656}
13657
13658bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
13659 llvm_unreachable("Not supported in SIMD-only mode");
13660}
13661
13662bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
13663 llvm_unreachable("Not supported in SIMD-only mode");
13664}
13665
13666bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
13667 return false;
13668}
13669
13670void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
13671 const OMPExecutableDirective &D,
13672 SourceLocation Loc,
13673 llvm::Function *OutlinedFn,
13674 ArrayRef<llvm::Value *> CapturedVars) {
13675 llvm_unreachable("Not supported in SIMD-only mode");
13676}
13677
13678void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
13679 const Expr *NumTeams,
13680 const Expr *ThreadLimit,
13681 SourceLocation Loc) {
13682 llvm_unreachable("Not supported in SIMD-only mode");
13683}
13684
13685void CGOpenMPSIMDRuntime::emitTargetDataCalls(
13686 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13687 const Expr *Device, const RegionCodeGenTy &CodeGen,
13688 CGOpenMPRuntime::TargetDataInfo &Info) {
13689 llvm_unreachable("Not supported in SIMD-only mode");
13690}
13691
13692void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
13693 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13694 const Expr *Device) {
13695 llvm_unreachable("Not supported in SIMD-only mode");
13696}
13697
13698void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
13699 const OMPLoopDirective &D,
13700 ArrayRef<Expr *> NumIterations) {
13701 llvm_unreachable("Not supported in SIMD-only mode");
13702}
13703
13704void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13705 const OMPDependClause *C) {
13706 llvm_unreachable("Not supported in SIMD-only mode");
13707}
13708
13709void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13710 const OMPDoacrossClause *C) {
13711 llvm_unreachable("Not supported in SIMD-only mode");
13712}
13713
13714const VarDecl *
13715CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
13716 const VarDecl *NativeParam) const {
13717 llvm_unreachable("Not supported in SIMD-only mode");
13718}
13719
13720Address
13721CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
13722 const VarDecl *NativeParam,
13723 const VarDecl *TargetParam) const {
13724 llvm_unreachable("Not supported in SIMD-only mode");
13725}
13726