1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "ABIInfoImpl.h"
15#include "CGCXXABI.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGRecordLayout.h"
19#include "CodeGenFunction.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/Attr.h"
23#include "clang/AST/Decl.h"
24#include "clang/AST/OpenMPClause.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/SourceManager.h"
30#include "clang/CodeGen/ConstantInitBuilder.h"
31#include "clang/CodeGenUtils/RecordLayoutUtils.h"
32#include "llvm/ADT/ArrayRef.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/Bitcode/BitcodeReader.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/InstrTypes.h"
43#include "llvm/IR/Value.h"
44#include "llvm/Support/AtomicOrdering.h"
45#include "llvm/Support/VirtualFileSystem.h"
46#include "llvm/Support/raw_ostream.h"
47#include <cassert>
48#include <cstdint>
49#include <numeric>
50#include <optional>
51
52using namespace clang;
53using namespace CodeGen;
54using namespace llvm::omp;
55
56namespace {
57/// Base class for handling code generation inside OpenMP regions.
58class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
59public:
60 /// Kinds of OpenMP regions used in codegen.
61 enum CGOpenMPRegionKind {
62 /// Region with outlined function for standalone 'parallel'
63 /// directive.
64 ParallelOutlinedRegion,
65 /// Region with outlined function for standalone 'task' directive.
66 TaskOutlinedRegion,
67 /// Region for constructs that do not require function outlining,
68 /// like 'for', 'sections', 'atomic' etc. directives.
69 InlinedRegion,
70 /// Region with outlined function for standalone 'target' directive.
71 TargetRegion,
72 };
73
74 CGOpenMPRegionInfo(const CapturedStmt &CS,
75 const CGOpenMPRegionKind RegionKind,
76 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
77 bool HasCancel)
78 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
79 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
80
81 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
82 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
83 bool HasCancel)
84 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
85 Kind(Kind), HasCancel(HasCancel) {}
86
87 /// Get a variable or parameter for storing global thread id
88 /// inside OpenMP construct.
89 virtual const VarDecl *getThreadIDVariable() const = 0;
90
91 /// Emit the captured statement body.
92 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
93
94 /// Get an LValue for the current ThreadID variable.
95 /// \return LValue for thread id variable. This LValue always has type int32*.
96 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
97
98 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
99
100 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
101
102 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
103
104 bool hasCancel() const { return HasCancel; }
105
106 static bool classof(const CGCapturedStmtInfo *Info) {
107 return Info->getKind() == CR_OpenMP;
108 }
109
110 ~CGOpenMPRegionInfo() override = default;
111
112protected:
113 CGOpenMPRegionKind RegionKind;
114 RegionCodeGenTy CodeGen;
115 OpenMPDirectiveKind Kind;
116 bool HasCancel;
117};
118
119/// API for captured statement code generation in OpenMP constructs.
120class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
121public:
122 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
123 const RegionCodeGenTy &CodeGen,
124 OpenMPDirectiveKind Kind, bool HasCancel,
125 StringRef HelperName)
126 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
127 HasCancel),
128 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
129 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
130 }
131
132 /// Get a variable or parameter for storing global thread id
133 /// inside OpenMP construct.
134 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
135
136 /// Get the name of the capture helper.
137 StringRef getHelperName() const override { return HelperName; }
138
139 static bool classof(const CGCapturedStmtInfo *Info) {
140 return CGOpenMPRegionInfo::classof(Info) &&
141 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
142 ParallelOutlinedRegion;
143 }
144
145private:
146 /// A variable or parameter storing global thread id for OpenMP
147 /// constructs.
148 const VarDecl *ThreadIDVar;
149 StringRef HelperName;
150};
151
152/// API for captured statement code generation in OpenMP constructs.
153class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
154public:
155 class UntiedTaskActionTy final : public PrePostActionTy {
156 bool Untied;
157 const VarDecl *PartIDVar;
158 const RegionCodeGenTy UntiedCodeGen;
159 llvm::SwitchInst *UntiedSwitch = nullptr;
160
161 public:
162 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
163 const RegionCodeGenTy &UntiedCodeGen)
164 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
165 void Enter(CodeGenFunction &CGF) override {
166 if (Untied) {
167 // Emit task switching point.
168 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
169 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
170 PtrTy: PartIDVar->getType()->castAs<PointerType>());
171 llvm::Value *Res =
172 CGF.EmitLoadOfScalar(lvalue: PartIdLVal, Loc: PartIDVar->getLocation());
173 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: ".untied.done.");
174 UntiedSwitch = CGF.Builder.CreateSwitch(V: Res, Dest: DoneBB);
175 CGF.EmitBlock(BB: DoneBB);
176 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
177 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
178 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: 0),
179 Dest: CGF.Builder.GetInsertBlock());
180 emitUntiedSwitch(CGF);
181 }
182 }
183 void emitUntiedSwitch(CodeGenFunction &CGF) const {
184 if (Untied) {
185 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
186 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
187 PtrTy: PartIDVar->getType()->castAs<PointerType>());
188 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
189 lvalue: PartIdLVal);
190 UntiedCodeGen(CGF);
191 CodeGenFunction::JumpDest CurPoint =
192 CGF.getJumpDestInCurrentScope(Name: ".untied.next.");
193 CGF.EmitBranch(Block: CGF.ReturnBlock.getBlock());
194 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
195 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
196 Dest: CGF.Builder.GetInsertBlock());
197 CGF.EmitBranchThroughCleanup(Dest: CurPoint);
198 CGF.EmitBlock(BB: CurPoint.getBlock());
199 }
200 }
201 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
202 };
203 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
204 const VarDecl *ThreadIDVar,
205 const RegionCodeGenTy &CodeGen,
206 OpenMPDirectiveKind Kind, bool HasCancel,
207 const UntiedTaskActionTy &Action)
208 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
209 ThreadIDVar(ThreadIDVar), Action(Action) {
210 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
211 }
212
213 /// Get a variable or parameter for storing global thread id
214 /// inside OpenMP construct.
215 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
216
217 /// Get an LValue for the current ThreadID variable.
218 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
219
220 /// Get the name of the capture helper.
221 StringRef getHelperName() const override { return ".omp_outlined."; }
222
223 void emitUntiedSwitch(CodeGenFunction &CGF) override {
224 Action.emitUntiedSwitch(CGF);
225 }
226
227 static bool classof(const CGCapturedStmtInfo *Info) {
228 return CGOpenMPRegionInfo::classof(Info) &&
229 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
230 TaskOutlinedRegion;
231 }
232
233private:
234 /// A variable or parameter storing global thread id for OpenMP
235 /// constructs.
236 const VarDecl *ThreadIDVar;
237 /// Action for emitting code for untied tasks.
238 const UntiedTaskActionTy &Action;
239};
240
241/// API for inlined captured statement code generation in OpenMP
242/// constructs.
243class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
244public:
245 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
246 const RegionCodeGenTy &CodeGen,
247 OpenMPDirectiveKind Kind, bool HasCancel)
248 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
249 OldCSI(OldCSI),
250 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(Val: OldCSI)) {}
251
252 // Retrieve the value of the context parameter.
253 llvm::Value *getContextValue() const override {
254 if (OuterRegionInfo)
255 return OuterRegionInfo->getContextValue();
256 llvm_unreachable("No context value for inlined OpenMP region");
257 }
258
259 void setContextValue(llvm::Value *V) override {
260 if (OuterRegionInfo) {
261 OuterRegionInfo->setContextValue(V);
262 return;
263 }
264 llvm_unreachable("No context value for inlined OpenMP region");
265 }
266
267 /// Lookup the captured field decl for a variable.
268 const FieldDecl *lookup(const VarDecl *VD) const override {
269 if (OuterRegionInfo)
270 return OuterRegionInfo->lookup(VD);
271 // If there is no outer outlined region,no need to lookup in a list of
272 // captured variables, we can use the original one.
273 return nullptr;
274 }
275
276 FieldDecl *getThisFieldDecl() const override {
277 if (OuterRegionInfo)
278 return OuterRegionInfo->getThisFieldDecl();
279 return nullptr;
280 }
281
282 /// Get a variable or parameter for storing global thread id
283 /// inside OpenMP construct.
284 const VarDecl *getThreadIDVariable() const override {
285 if (OuterRegionInfo)
286 return OuterRegionInfo->getThreadIDVariable();
287 return nullptr;
288 }
289
290 /// Get an LValue for the current ThreadID variable.
291 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
292 if (OuterRegionInfo)
293 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
294 llvm_unreachable("No LValue for inlined OpenMP construct");
295 }
296
297 /// Get the name of the capture helper.
298 StringRef getHelperName() const override {
299 if (auto *OuterRegionInfo = getOldCSI())
300 return OuterRegionInfo->getHelperName();
301 llvm_unreachable("No helper name for inlined OpenMP construct");
302 }
303
304 void emitUntiedSwitch(CodeGenFunction &CGF) override {
305 if (OuterRegionInfo)
306 OuterRegionInfo->emitUntiedSwitch(CGF);
307 }
308
309 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
310
311 static bool classof(const CGCapturedStmtInfo *Info) {
312 return CGOpenMPRegionInfo::classof(Info) &&
313 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == InlinedRegion;
314 }
315
316 ~CGOpenMPInlinedRegionInfo() override = default;
317
318private:
319 /// CodeGen info about outer OpenMP region.
320 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
321 CGOpenMPRegionInfo *OuterRegionInfo;
322};
323
324/// API for captured statement code generation in OpenMP target
325/// constructs. For this captures, implicit parameters are used instead of the
326/// captured fields. The name of the target region has to be unique in a given
327/// application so it is provided by the client, because only the client has
328/// the information to generate that.
329class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
330public:
331 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
332 const RegionCodeGenTy &CodeGen, StringRef HelperName)
333 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
334 /*HasCancel=*/false),
335 HelperName(HelperName) {}
336
337 /// This is unused for target regions because each starts executing
338 /// with a single thread.
339 const VarDecl *getThreadIDVariable() const override { return nullptr; }
340
341 /// Get the name of the capture helper.
342 StringRef getHelperName() const override { return HelperName; }
343
344 static bool classof(const CGCapturedStmtInfo *Info) {
345 return CGOpenMPRegionInfo::classof(Info) &&
346 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == TargetRegion;
347 }
348
349private:
350 StringRef HelperName;
351};
352
353static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
354 llvm_unreachable("No codegen for expressions");
355}
356/// API for generation of expressions captured in a innermost OpenMP
357/// region.
358class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
359public:
360 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
361 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
362 OMPD_unknown,
363 /*HasCancel=*/false),
364 PrivScope(CGF) {
365 // Make sure the globals captured in the provided statement are local by
366 // using the privatization logic. We assume the same variable is not
367 // captured more than once.
368 for (const auto &C : CS.captures()) {
369 if (!C.capturesVariable() && !C.capturesVariableByCopy())
370 continue;
371
372 const VarDecl *VD = C.getCapturedVar();
373 if (VD->isLocalVarDeclOrParm())
374 continue;
375
376 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
377 /*RefersToEnclosingVariableOrCapture=*/false,
378 VD->getType().getNonReferenceType(), VK_LValue,
379 C.getLocation());
380 PrivScope.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
381 }
382 (void)PrivScope.Privatize();
383 }
384
385 /// Lookup the captured field decl for a variable.
386 const FieldDecl *lookup(const VarDecl *VD) const override {
387 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
388 return FD;
389 return nullptr;
390 }
391
392 /// Emit the captured statement body.
393 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
394 llvm_unreachable("No body for expressions");
395 }
396
397 /// Get a variable or parameter for storing global thread id
398 /// inside OpenMP construct.
399 const VarDecl *getThreadIDVariable() const override {
400 llvm_unreachable("No thread id for expressions");
401 }
402
403 /// Get the name of the capture helper.
404 StringRef getHelperName() const override {
405 llvm_unreachable("No helper name for expressions");
406 }
407
408 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
409
410private:
411 /// Private scope to capture global variables.
412 CodeGenFunction::OMPPrivateScope PrivScope;
413};
414
415/// RAII for emitting code of OpenMP constructs.
416class InlinedOpenMPRegionRAII {
417 CodeGenFunction &CGF;
418 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
419 FieldDecl *LambdaThisCaptureField = nullptr;
420 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
421 bool NoInheritance = false;
422
423public:
424 /// Constructs region for combined constructs.
425 /// \param CodeGen Code generation sequence for combined directives. Includes
426 /// a list of functions used for code generation of implicitly inlined
427 /// regions.
428 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
429 OpenMPDirectiveKind Kind, bool HasCancel,
430 bool NoInheritance = true)
431 : CGF(CGF), NoInheritance(NoInheritance) {
432 // Start emission for the construct.
433 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
434 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
435 if (NoInheritance) {
436 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
437 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
438 CGF.LambdaThisCaptureField = nullptr;
439 BlockInfo = CGF.BlockInfo;
440 CGF.BlockInfo = nullptr;
441 }
442 }
443
444 ~InlinedOpenMPRegionRAII() {
445 // Restore original CapturedStmtInfo only if we're done with code emission.
446 auto *OldCSI =
447 cast<CGOpenMPInlinedRegionInfo>(Val: CGF.CapturedStmtInfo)->getOldCSI();
448 delete CGF.CapturedStmtInfo;
449 CGF.CapturedStmtInfo = OldCSI;
450 if (NoInheritance) {
451 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
452 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
453 CGF.BlockInfo = BlockInfo;
454 }
455 }
456};
457
458/// Values for bit flags used in the ident_t to describe the fields.
459/// All enumeric elements are named and described in accordance with the code
460/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
461enum OpenMPLocationFlags : unsigned {
462 /// Use trampoline for internal microtask.
463 OMP_IDENT_IMD = 0x01,
464 /// Use c-style ident structure.
465 OMP_IDENT_KMPC = 0x02,
466 /// Atomic reduction option for kmpc_reduce.
467 OMP_ATOMIC_REDUCE = 0x10,
468 /// Explicit 'barrier' directive.
469 OMP_IDENT_BARRIER_EXPL = 0x20,
470 /// Implicit barrier in code.
471 OMP_IDENT_BARRIER_IMPL = 0x40,
472 /// Implicit barrier in 'for' directive.
473 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
474 /// Implicit barrier in 'sections' directive.
475 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
476 /// Implicit barrier in 'single' directive.
477 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
478 /// Call of __kmp_for_static_init for static loop.
479 OMP_IDENT_WORK_LOOP = 0x200,
480 /// Call of __kmp_for_static_init for sections.
481 OMP_IDENT_WORK_SECTIONS = 0x400,
482 /// Call of __kmp_for_static_init for distribute.
483 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
484 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
485};
486
487/// Describes ident structure that describes a source location.
488/// All descriptions are taken from
489/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
490/// Original structure:
491/// typedef struct ident {
492/// kmp_int32 reserved_1; /**< might be used in Fortran;
493/// see above */
494/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
495/// KMP_IDENT_KMPC identifies this union
496/// member */
497/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
498/// see above */
499///#if USE_ITT_BUILD
500/// /* but currently used for storing
501/// region-specific ITT */
502/// /* contextual information. */
503///#endif /* USE_ITT_BUILD */
504/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
505/// C++ */
506/// char const *psource; /**< String describing the source location.
507/// The string is composed of semi-colon separated
508// fields which describe the source file,
509/// the function and a pair of line numbers that
510/// delimit the construct.
511/// */
512/// } ident_t;
513enum IdentFieldIndex {
514 /// might be used in Fortran
515 IdentField_Reserved_1,
516 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
517 IdentField_Flags,
518 /// Not really used in Fortran any more
519 IdentField_Reserved_2,
520 /// Source[4] in Fortran, do not use for C++
521 IdentField_Reserved_3,
522 /// String describing the source location. The string is composed of
523 /// semi-colon separated fields which describe the source file, the function
524 /// and a pair of line numbers that delimit the construct.
525 IdentField_PSource
526};
527
528/// Schedule types for 'omp for' loops (these enumerators are taken from
529/// the enum sched_type in kmp.h).
530enum OpenMPSchedType {
531 /// Lower bound for default (unordered) versions.
532 OMP_sch_lower = 32,
533 OMP_sch_static_chunked = 33,
534 OMP_sch_static = 34,
535 OMP_sch_dynamic_chunked = 35,
536 OMP_sch_guided_chunked = 36,
537 OMP_sch_runtime = 37,
538 OMP_sch_auto = 38,
539 /// static with chunk adjustment (e.g., simd)
540 OMP_sch_static_balanced_chunked = 45,
541 /// Lower bound for 'ordered' versions.
542 OMP_ord_lower = 64,
543 OMP_ord_static_chunked = 65,
544 OMP_ord_static = 66,
545 OMP_ord_dynamic_chunked = 67,
546 OMP_ord_guided_chunked = 68,
547 OMP_ord_runtime = 69,
548 OMP_ord_auto = 70,
549 OMP_sch_default = OMP_sch_static,
550 /// dist_schedule types
551 OMP_dist_sch_static_chunked = 91,
552 OMP_dist_sch_static = 92,
553 /// Fused distribute+for static schedule (entityId = team*nthreads + tid,
554 /// num_entities = nteams*nthreads). One for_static_init call, no
555 /// surrounding distribute_static_init. Matches
556 /// kmp_sched_distr_static_chunk_sched_static_chunkone in the device RTL
557 /// (openmp/device/include/DeviceTypes.h).
558 OMP_dist_sch_static_chunked_sch_static_chunkone = 93,
559 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
560 /// Set if the monotonic schedule modifier was present.
561 OMP_sch_modifier_monotonic = (1 << 29),
562 /// Set if the nonmonotonic schedule modifier was present.
563 OMP_sch_modifier_nonmonotonic = (1 << 30),
564};
565
566/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
567/// region.
568class CleanupTy final : public EHScopeStack::Cleanup {
569 PrePostActionTy *Action;
570
571public:
572 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
573 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
574 if (!CGF.HaveInsertPoint())
575 return;
576 Action->Exit(CGF);
577 }
578};
579
580} // anonymous namespace
581
582void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
583 CodeGenFunction::RunCleanupsScope Scope(CGF);
584 if (PrePostAction) {
585 CGF.EHStack.pushCleanup<CleanupTy>(Kind: NormalAndEHCleanup, A: PrePostAction);
586 Callback(CodeGen, CGF, *PrePostAction);
587 } else {
588 PrePostActionTy Action;
589 Callback(CodeGen, CGF, Action);
590 }
591}
592
593/// Check if the combiner is a call to UDR combiner and if it is so return the
594/// UDR decl used for reduction.
595static const OMPDeclareReductionDecl *
596getReductionInit(const Expr *ReductionOp) {
597 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
598 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
599 if (const auto *DRE =
600 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
601 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl()))
602 return DRD;
603 return nullptr;
604}
605
606static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
607 const OMPDeclareReductionDecl *DRD,
608 const Expr *InitOp,
609 Address Private, Address Original,
610 QualType Ty) {
611 if (DRD->getInitializer()) {
612 std::pair<llvm::Function *, llvm::Function *> Reduction =
613 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
614 const auto *CE = cast<CallExpr>(Val: InitOp);
615 const auto *OVE = cast<OpaqueValueExpr>(Val: CE->getCallee());
616 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
617 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
618 const auto *LHSDRE =
619 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: LHS)->getSubExpr());
620 const auto *RHSDRE =
621 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: RHS)->getSubExpr());
622 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
623 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: LHSDRE->getDecl()), Addr: Private);
624 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: RHSDRE->getDecl()), Addr: Original);
625 (void)PrivateScope.Privatize();
626 RValue Func = RValue::get(V: Reduction.second);
627 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
628 CGF.EmitIgnoredExpr(E: InitOp);
629 } else {
630 llvm::Constant *Init = CGF.CGM.EmitNullConstant(T: Ty);
631 std::string Name = CGF.CGM.getOpenMPRuntime().getName(Parts: {"init"});
632 auto *GV = new llvm::GlobalVariable(
633 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
634 llvm::GlobalValue::PrivateLinkage, Init, Name);
635 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V: GV, T: Ty);
636 RValue InitRVal;
637 switch (CGF.getEvaluationKind(T: Ty)) {
638 case TEK_Scalar:
639 InitRVal = CGF.EmitLoadOfLValue(V: LV, Loc: DRD->getLocation());
640 break;
641 case TEK_Complex:
642 InitRVal =
643 RValue::getComplex(C: CGF.EmitLoadOfComplex(src: LV, loc: DRD->getLocation()));
644 break;
645 case TEK_Aggregate: {
646 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);
647 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
648 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
649 /*IsInitializer=*/false);
650 return;
651 }
652 }
653 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue);
654 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
655 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
656 /*IsInitializer=*/false);
657 }
658}
659
660/// Emit initialization of arrays of complex types.
661/// \param DestAddr Address of the array.
662/// \param Type Type of array.
663/// \param Init Initial expression of array.
664/// \param SrcAddr Address of the original array, required only if the declare
665/// reduction has an initializer.
666static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
667 QualType Type, bool EmitDeclareReductionInit,
668 const Expr *Init,
669 const OMPDeclareReductionDecl *DRD,
670 Address SrcAddr = Address::invalid()) {
671 // Perform element-by-element initialization.
672 QualType ElementTy;
673
674 // Only the initializer of a declare reduction reads the original item.
675 bool UseSrc = DRD && DRD->getInitializer();
676 assert((!UseSrc || SrcAddr.isValid()) &&
677 "Declare reduction initializer requires the original item address.");
678
679 // Drill down to the base element type on both arrays.
680 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
681 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
682 if (UseSrc)
683 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
684
685 llvm::Value *SrcBegin = nullptr;
686 if (UseSrc)
687 SrcBegin = SrcAddr.emitRawPointer(CGF);
688 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
689 // Cast from pointer to array type to pointer to single element.
690 llvm::Value *DestEnd =
691 CGF.Builder.CreateGEP(Ty: DestAddr.getElementType(), Ptr: DestBegin, IdxList: NumElements);
692 // The basic structure here is a while-do loop.
693 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arrayinit.body");
694 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arrayinit.done");
695 llvm::Value *IsEmpty =
696 CGF.Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arrayinit.isempty");
697 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
698
699 // Enter the loop body, making that address the current address.
700 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
701 CGF.EmitBlock(BB: BodyBB);
702
703 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
704
705 llvm::PHINode *SrcElementPHI = nullptr;
706 Address SrcElementCurrent = Address::invalid();
707 if (UseSrc) {
708 SrcElementPHI = CGF.Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2,
709 Name: "omp.arraycpy.srcElementPast");
710 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
711 SrcElementCurrent =
712 Address(SrcElementPHI, SrcAddr.getElementType(),
713 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
714 }
715 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
716 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
717 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
718 Address DestElementCurrent =
719 Address(DestElementPHI, DestAddr.getElementType(),
720 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
721
722 // Emit copy.
723 {
724 CodeGenFunction::RunCleanupsScope InitScope(CGF);
725 if (EmitDeclareReductionInit) {
726 emitInitWithReductionInitializer(CGF, DRD, InitOp: Init, Private: DestElementCurrent,
727 Original: SrcElementCurrent, Ty: ElementTy);
728 } else
729 CGF.EmitAnyExprToMem(E: Init, Location: DestElementCurrent, Quals: ElementTy.getQualifiers(),
730 /*IsInitializer=*/false);
731 }
732
733 if (UseSrc) {
734 // Shift the address forward by one element.
735 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
736 Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI, /*Idx0=*/1,
737 Name: "omp.arraycpy.dest.element");
738 SrcElementPHI->addIncoming(V: SrcElementNext, BB: CGF.Builder.GetInsertBlock());
739 }
740
741 // Shift the address forward by one element.
742 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
743 Ty: DestAddr.getElementType(), Ptr: DestElementPHI, /*Idx0=*/1,
744 Name: "omp.arraycpy.dest.element");
745 // Check whether we've reached the end.
746 llvm::Value *Done =
747 CGF.Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
748 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
749 DestElementPHI->addIncoming(V: DestElementNext, BB: CGF.Builder.GetInsertBlock());
750
751 // Done.
752 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
753}
754
755LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
756 return CGF.EmitOMPSharedLValue(E);
757}
758
759LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
760 const Expr *E) {
761 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E))
762 return CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false);
763 return LValue();
764}
765
766void ReductionCodeGen::emitAggregateInitialization(
767 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
768 const OMPDeclareReductionDecl *DRD) {
769 // Emit VarDecl with copy init for arrays.
770 // Get the address of the original variable captured in current
771 // captured region.
772 const auto *PrivateVD =
773 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
774 bool EmitDeclareReductionInit =
775 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
776 EmitOMPAggregateInit(CGF, DestAddr: PrivateAddr, Type: PrivateVD->getType(),
777 EmitDeclareReductionInit,
778 Init: EmitDeclareReductionInit ? ClausesData[N].ReductionOp
779 : PrivateVD->getInit(),
780 DRD, SrcAddr: SharedAddr);
781}
782
783ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
784 ArrayRef<const Expr *> Origs,
785 ArrayRef<const Expr *> Privates,
786 ArrayRef<const Expr *> ReductionOps) {
787 ClausesData.reserve(N: Shareds.size());
788 SharedAddresses.reserve(N: Shareds.size());
789 Sizes.reserve(N: Shareds.size());
790 BaseDecls.reserve(N: Shareds.size());
791 const auto *IOrig = Origs.begin();
792 const auto *IPriv = Privates.begin();
793 const auto *IRed = ReductionOps.begin();
794 for (const Expr *Ref : Shareds) {
795 ClausesData.emplace_back(Args&: Ref, Args: *IOrig, Args: *IPriv, Args: *IRed);
796 std::advance(i&: IOrig, n: 1);
797 std::advance(i&: IPriv, n: 1);
798 std::advance(i&: IRed, n: 1);
799 }
800}
801
802void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
803 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
804 "Number of generated lvalues must be exactly N.");
805 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Shared);
806 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Shared);
807 SharedAddresses.emplace_back(Args&: First, Args&: Second);
808 if (ClausesData[N].Shared == ClausesData[N].Ref) {
809 OrigAddresses.emplace_back(Args&: First, Args&: Second);
810 } else {
811 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Ref);
812 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Ref);
813 OrigAddresses.emplace_back(Args&: First, Args&: Second);
814 }
815}
816
817void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
818 QualType PrivateType = getPrivateType(N);
819 bool AsArraySection = isa<ArraySectionExpr>(Val: ClausesData[N].Ref);
820 if (!PrivateType->isVariablyModifiedType()) {
821 Sizes.emplace_back(
822 Args: CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType()),
823 Args: nullptr);
824 return;
825 }
826 llvm::Value *Size;
827 llvm::Value *SizeInChars;
828 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
829 auto *ElemSizeOf = llvm::ConstantInt::get(
830 Ty: CGF.SizeTy, V: CGF.CGM.getDataLayout().getTypeAllocSize(Ty: ElemType));
831 if (AsArraySection) {
832 SizeInChars =
833 CGF.Builder.CreatePtrDiff(LHS: OrigAddresses[N].second.getPointer(CGF),
834 RHS: OrigAddresses[N].first.getPointer(CGF));
835 SizeInChars = CGF.Builder.CreateNUWAdd(LHS: SizeInChars, RHS: ElemSizeOf);
836 } else {
837 SizeInChars =
838 CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType());
839 }
840 Size = ElemSizeOf->isOne()
841 ? SizeInChars
842 : CGF.Builder.CreateExactUDiv(LHS: SizeInChars, RHS: ElemSizeOf);
843 Sizes.emplace_back(Args&: SizeInChars, Args&: Size);
844 CodeGenFunction::OpaqueValueMapping OpaqueMap(
845 CGF,
846 cast<OpaqueValueExpr>(
847 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
848 RValue::get(V: Size));
849 CGF.EmitVariablyModifiedType(Ty: PrivateType);
850}
851
852void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
853 llvm::Value *Size) {
854 QualType PrivateType = getPrivateType(N);
855 if (!PrivateType->isVariablyModifiedType()) {
856 assert(!Size && !Sizes[N].second &&
857 "Size should be nullptr for non-variably modified reduction "
858 "items.");
859 return;
860 }
861 CodeGenFunction::OpaqueValueMapping OpaqueMap(
862 CGF,
863 cast<OpaqueValueExpr>(
864 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
865 RValue::get(V: Size));
866 CGF.EmitVariablyModifiedType(Ty: PrivateType);
867}
868
869void ReductionCodeGen::emitInitialization(
870 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
871 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
872 assert(SharedAddresses.size() > N && "No variable was generated");
873 const auto *PrivateVD =
874 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
875 const OMPDeclareReductionDecl *DRD =
876 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
877 if (CGF.getContext().getAsArrayType(T: PrivateVD->getType())) {
878 if (DRD && DRD->getInitializer())
879 (void)DefaultInit(CGF);
880 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
881 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
882 (void)DefaultInit(CGF);
883 QualType SharedType = SharedAddresses[N].first.getType();
884 emitInitWithReductionInitializer(CGF, DRD, InitOp: ClausesData[N].ReductionOp,
885 Private: PrivateAddr, Original: SharedAddr, Ty: SharedType);
886 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
887 !CGF.isTrivialInitializer(Init: PrivateVD->getInit())) {
888 CGF.EmitAnyExprToMem(E: PrivateVD->getInit(), Location: PrivateAddr,
889 Quals: PrivateVD->getType().getQualifiers(),
890 /*IsInitializer=*/false);
891 }
892}
893
894bool ReductionCodeGen::needCleanups(unsigned N) {
895 QualType PrivateType = getPrivateType(N);
896 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
897 return DTorKind != QualType::DK_none;
898}
899
900void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
901 Address PrivateAddr) {
902 QualType PrivateType = getPrivateType(N);
903 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
904 if (needCleanups(N)) {
905 PrivateAddr =
906 PrivateAddr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: PrivateType));
907 CGF.pushDestroy(dtorKind: DTorKind, addr: PrivateAddr, type: PrivateType);
908 }
909}
910
911static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
912 LValue BaseLV) {
913 BaseTy = BaseTy.getNonReferenceType();
914 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
915 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
916 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
917 BaseLV = CGF.EmitLoadOfPointerLValue(Ptr: BaseLV.getAddress(), PtrTy);
918 } else {
919 LValue RefLVal = CGF.MakeAddrLValue(Addr: BaseLV.getAddress(), T: BaseTy);
920 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
921 }
922 BaseTy = BaseTy->getPointeeType();
923 }
924 return CGF.MakeAddrLValue(
925 Addr: BaseLV.getAddress().withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy)),
926 T: BaseLV.getType(), BaseInfo: BaseLV.getBaseInfo(),
927 TBAAInfo: CGF.CGM.getTBAAInfoForSubobject(Base: BaseLV, AccessType: BaseLV.getType()));
928}
929
930static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
931 Address OriginalBaseAddress, llvm::Value *Addr) {
932 RawAddress Tmp = RawAddress::invalid();
933 Address TopTmp = Address::invalid();
934 Address MostTopTmp = Address::invalid();
935 BaseTy = BaseTy.getNonReferenceType();
936 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
937 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
938 Tmp = CGF.CreateMemTempWithoutCast(T: BaseTy);
939 if (TopTmp.isValid())
940 CGF.Builder.CreateStore(Val: Tmp.getPointer(), Addr: TopTmp);
941 else
942 MostTopTmp = Tmp;
943 TopTmp = Tmp;
944 BaseTy = BaseTy->getPointeeType();
945 }
946
947 if (Tmp.isValid()) {
948 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
949 V: Addr, DestTy: Tmp.getElementType());
950 CGF.Builder.CreateStore(Val: Addr, Addr: Tmp);
951 return MostTopTmp;
952 }
953
954 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
955 V: Addr, DestTy: OriginalBaseAddress.getType());
956 return OriginalBaseAddress.withPointer(NewPointer: Addr, IsKnownNonNull: NotKnownNonNull);
957}
958
959static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
960 const VarDecl *OrigVD = nullptr;
961 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Ref)) {
962 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
963 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Val: Base))
964 Base = TempOASE->getBase()->IgnoreParenImpCasts();
965 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
966 Base = TempASE->getBase()->IgnoreParenImpCasts();
967 DE = cast<DeclRefExpr>(Val: Base);
968 OrigVD = cast<VarDecl>(Val: DE->getDecl());
969 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Ref)) {
970 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
971 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
972 Base = TempASE->getBase()->IgnoreParenImpCasts();
973 DE = cast<DeclRefExpr>(Val: Base);
974 OrigVD = cast<VarDecl>(Val: DE->getDecl());
975 }
976 return OrigVD;
977}
978
979Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
980 Address PrivateAddr) {
981 const DeclRefExpr *DE;
982 if (const VarDecl *OrigVD = ::getBaseDecl(Ref: ClausesData[N].Ref, DE)) {
983 BaseDecls.emplace_back(Args&: OrigVD);
984 LValue OriginalBaseLValue = CGF.EmitLValue(E: DE);
985 LValue BaseLValue =
986 loadToBegin(CGF, BaseTy: OrigVD->getType(), ElTy: SharedAddresses[N].first.getType(),
987 BaseLV: OriginalBaseLValue);
988 Address SharedAddr = SharedAddresses[N].first.getAddress();
989 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
990 ElemTy: SharedAddr.getElementType(), LHS: BaseLValue.getPointer(CGF),
991 RHS: SharedAddr.emitRawPointer(CGF));
992 llvm::Value *PrivatePointer =
993 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
994 V: PrivateAddr.emitRawPointer(CGF), DestTy: SharedAddr.getType());
995 llvm::Value *Ptr = CGF.Builder.CreateGEP(
996 Ty: SharedAddr.getElementType(), Ptr: PrivatePointer, IdxList: Adjustment);
997 return castToBase(CGF, BaseTy: OrigVD->getType(),
998 ElTy: SharedAddresses[N].first.getType(),
999 OriginalBaseAddress: OriginalBaseLValue.getAddress(), Addr: Ptr);
1000 }
1001 BaseDecls.emplace_back(
1002 Args: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Ref)->getDecl()));
1003 return PrivateAddr;
1004}
1005
1006bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1007 const OMPDeclareReductionDecl *DRD =
1008 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
1009 return DRD && DRD->getInitializer();
1010}
1011
1012LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1013 return CGF.EmitLoadOfPointerLValue(
1014 Ptr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1015 PtrTy: getThreadIDVariable()->getType()->castAs<PointerType>());
1016}
1017
1018void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1019 if (!CGF.HaveInsertPoint())
1020 return;
1021 // 1.2.2 OpenMP Language Terminology
1022 // Structured block - An executable statement with a single entry at the
1023 // top and a single exit at the bottom.
1024 // The point of exit cannot be a branch out of the structured block.
1025 // longjmp() and throw() must not violate the entry/exit criteria.
1026 CGF.EHStack.pushTerminate();
1027 if (S)
1028 CGF.incrementProfileCounter(S);
1029 CodeGen(CGF);
1030 CGF.EHStack.popTerminate();
1031}
1032
1033LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1034 CodeGenFunction &CGF) {
1035 return CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1036 T: getThreadIDVariable()->getType(),
1037 Source: AlignmentSource::Decl);
1038}
1039
1040static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1041 QualType FieldTy) {
1042 auto *Field = FieldDecl::Create(
1043 C, DC, StartLoc: SourceLocation(), IdLoc: SourceLocation(), /*Id=*/nullptr, T: FieldTy,
1044 TInfo: C.getTrivialTypeSourceInfo(T: FieldTy, Loc: SourceLocation()),
1045 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1046 Field->setAccess(AS_public);
1047 DC->addDecl(D: Field);
1048 return Field;
1049}
1050
1051CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
1052 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1053 KmpCriticalNameTy = llvm::ArrayType::get(ElementType: CGM.Int32Ty, /*NumElements*/ 8);
1054 llvm::OpenMPIRBuilderConfig Config(
1055 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),
1056 CGM.getLangOpts().OpenMPOffloadMandatory,
1057 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,
1058 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);
1059 Config.setDefaultTargetAS(
1060 CGM.getContext().getTargetInfo().getTargetAddressSpace(AS: LangAS::Default));
1061 Config.setRuntimeCC(CGM.getRuntimeCC());
1062
1063 OMPBuilder.setConfig(Config);
1064 OMPBuilder.initialize();
1065 OMPBuilder.loadOffloadInfoMetadata(VFS&: *CGM.getFileSystem(),
1066 HostFilePath: CGM.getLangOpts().OpenMPIsTargetDevice
1067 ? CGM.getLangOpts().OMPHostIRFile
1068 : StringRef{});
1069
1070 // The user forces the compiler to behave as if omp requires
1071 // unified_shared_memory was given.
1072 if (CGM.getLangOpts().OpenMPForceUSM) {
1073 HasRequiresUnifiedSharedMemory = true;
1074 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
1075 }
1076}
1077
1078void CGOpenMPRuntime::clear() {
1079 InternalVars.clear();
1080 // Clean non-target variable declarations possibly used only in debug info.
1081 for (const auto &Data : EmittedNonTargetVariables) {
1082 if (!Data.getValue().pointsToAliveValue())
1083 continue;
1084 auto *GV = dyn_cast<llvm::GlobalVariable>(Val: Data.getValue());
1085 if (!GV)
1086 continue;
1087 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1088 continue;
1089 GV->eraseFromParent();
1090 }
1091}
1092
1093std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1094 return OMPBuilder.createPlatformSpecificName(Parts);
1095}
1096
1097static llvm::Function *
1098emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1099 const Expr *CombinerInitializer, const VarDecl *In,
1100 const VarDecl *Out, bool IsCombiner) {
1101 // void .omp_combiner.(Ty *in, Ty *out);
1102 ASTContext &C = CGM.getContext();
1103 QualType PtrTy = C.getPointerType(T: Ty).withRestrict();
1104 auto *OmpOutParm = ImplicitParamDecl::Create(
1105 C, /*DC=*/nullptr, IdLoc: Out->getLocation(),
1106 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1107 auto *OmpInParm = ImplicitParamDecl::Create(
1108 C, /*DC=*/nullptr, IdLoc: In->getLocation(),
1109 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1110 FunctionArgList Args{OmpOutParm, OmpInParm};
1111 const CGFunctionInfo &FnInfo =
1112 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
1113 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1114 std::string Name = CGM.getOpenMPRuntime().getName(
1115 Parts: {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1116 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
1117 N: Name, M: &CGM.getModule());
1118 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
1119 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1120 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
1121 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
1122 Fn->removeFnAttr(Kind: llvm::Attribute::NoInline);
1123 Fn->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
1124 Fn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1125 }
1126 CodeGenFunction CGF(CGM);
1127 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1128 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1129 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc: In->getLocation(),
1130 StartLoc: Out->getLocation());
1131 CodeGenFunction::OMPPrivateScope Scope(CGF);
1132 Address AddrIn = CGF.GetAddrOfLocalVar(VD: OmpInParm);
1133 Scope.addPrivate(
1134 LocalVD: In, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrIn, PtrTy: PtrTy->castAs<PointerType>())
1135 .getAddress());
1136 Address AddrOut = CGF.GetAddrOfLocalVar(VD: OmpOutParm);
1137 Scope.addPrivate(
1138 LocalVD: Out, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrOut, PtrTy: PtrTy->castAs<PointerType>())
1139 .getAddress());
1140 (void)Scope.Privatize();
1141 if (!IsCombiner && Out->hasInit() &&
1142 !CGF.isTrivialInitializer(Init: Out->getInit())) {
1143 CGF.EmitAnyExprToMem(E: Out->getInit(), Location: CGF.GetAddrOfLocalVar(VD: Out),
1144 Quals: Out->getType().getQualifiers(),
1145 /*IsInitializer=*/true);
1146 }
1147 if (CombinerInitializer)
1148 CGF.EmitIgnoredExpr(E: CombinerInitializer);
1149 Scope.ForceCleanup();
1150 CGF.FinishFunction();
1151 return Fn;
1152}
1153
1154void CGOpenMPRuntime::emitUserDefinedReduction(
1155 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1156 if (UDRMap.count(Val: D) > 0)
1157 return;
1158 llvm::Function *Combiner = emitCombinerOrInitializer(
1159 CGM, Ty: D->getType(), CombinerInitializer: D->getCombiner(),
1160 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerIn())->getDecl()),
1161 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerOut())->getDecl()),
1162 /*IsCombiner=*/true);
1163 llvm::Function *Initializer = nullptr;
1164 if (const Expr *Init = D->getInitializer()) {
1165 Initializer = emitCombinerOrInitializer(
1166 CGM, Ty: D->getType(),
1167 CombinerInitializer: D->getInitializerKind() == OMPDeclareReductionInitKind::Call ? Init
1168 : nullptr,
1169 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitOrig())->getDecl()),
1170 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitPriv())->getDecl()),
1171 /*IsCombiner=*/false);
1172 }
1173 UDRMap.try_emplace(Key: D, Args&: Combiner, Args&: Initializer);
1174 if (CGF)
1175 FunctionUDRMap[CGF->CurFn].push_back(Elt: D);
1176}
1177
1178std::pair<llvm::Function *, llvm::Function *>
1179CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1180 auto I = UDRMap.find(Val: D);
1181 if (I != UDRMap.end())
1182 return I->second;
1183 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1184 return UDRMap.lookup(Val: D);
1185}
1186
1187namespace {
1188// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1189// Builder if one is present.
1190struct PushAndPopStackRAII {
1191 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1192 bool HasCancel, llvm::omp::Directive Kind)
1193 : OMPBuilder(OMPBuilder) {
1194 if (!OMPBuilder)
1195 return;
1196
1197 // The following callback is the crucial part of clangs cleanup process.
1198 //
1199 // NOTE:
1200 // Once the OpenMPIRBuilder is used to create parallel regions (and
1201 // similar), the cancellation destination (Dest below) is determined via
1202 // IP. That means if we have variables to finalize we split the block at IP,
1203 // use the new block (=BB) as destination to build a JumpDest (via
1204 // getJumpDestInCurrentScope(BB)) which then is fed to
1205 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1206 // to push & pop an FinalizationInfo object.
1207 // The FiniCB will still be needed but at the point where the
1208 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1209 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1210 assert(IP == IP.getNodeParent()->end() &&
1211 "Clang CG should cause non-terminated block!");
1212 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1213 CGF.Builder.restoreIP(IP);
1214 CodeGenFunction::JumpDest Dest =
1215 CGF.getOMPCancelDestination(Kind: OMPD_parallel);
1216 CGF.EmitBranchThroughCleanup(Dest);
1217 return llvm::Error::success();
1218 };
1219
1220 // TODO: Remove this once we emit parallel regions through the
1221 // OpenMPIRBuilder as it can do this setup internally.
1222 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1223 OMPBuilder->pushFinalizationCB(FI: std::move(FI));
1224 }
1225 ~PushAndPopStackRAII() {
1226 if (OMPBuilder)
1227 OMPBuilder->popFinalizationCB();
1228 }
1229 llvm::OpenMPIRBuilder *OMPBuilder;
1230};
1231} // namespace
1232
1233static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1234 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1235 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1236 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1237 assert(ThreadIDVar->getType()->isPointerType() &&
1238 "thread id variable must be of type kmp_int32 *");
1239 CodeGenFunction CGF(CGM, true);
1240 bool HasCancel = false;
1241 if (const auto *OPD = dyn_cast<OMPParallelDirective>(Val: &D))
1242 HasCancel = OPD->hasCancel();
1243 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(Val: &D))
1244 HasCancel = OPD->hasCancel();
1245 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &D))
1246 HasCancel = OPSD->hasCancel();
1247 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(Val: &D))
1248 HasCancel = OPFD->hasCancel();
1249 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(Val: &D))
1250 HasCancel = OPFD->hasCancel();
1251 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(Val: &D))
1252 HasCancel = OPFD->hasCancel();
1253 else if (const auto *OPFD =
1254 dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &D))
1255 HasCancel = OPFD->hasCancel();
1256 else if (const auto *OPFD =
1257 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &D))
1258 HasCancel = OPFD->hasCancel();
1259
1260 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1261 // parallel region to make cancellation barriers work properly.
1262 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1263 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1264 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1265 HasCancel, OutlinedHelperName);
1266 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1267 return CGF.GenerateOpenMPCapturedStmtFunction(S: *CS, D);
1268}
1269
1270std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const {
1271 std::string Suffix = getName(Parts: {"omp_outlined"});
1272 return (Name + Suffix).str();
1273}
1274
1275std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {
1276 return getOutlinedHelperName(Name: CGF.CurFn->getName());
1277}
1278
1279std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const {
1280 std::string Suffix = getName(Parts: {"omp", "reduction", "reduction_func"});
1281 return (Name + Suffix).str();
1282}
1283
1284llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1285 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1286 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1287 const RegionCodeGenTy &CodeGen) {
1288 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_parallel);
1289 return emitParallelOrTeamsOutlinedFunction(
1290 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1291 CodeGen);
1292}
1293
1294llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1295 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1296 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1297 const RegionCodeGenTy &CodeGen) {
1298 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_teams);
1299 llvm::Function *OutlinedFn = emitParallelOrTeamsOutlinedFunction(
1300 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1301 CodeGen);
1302 // A teams body is called once per team and is not handed back to the runtime
1303 // as a callback, so unlike a parallel body it cannot be re-entered while a
1304 // call to it is live.
1305 OutlinedFn->setDoesNotRecurse();
1306 return OutlinedFn;
1307}
1308
1309llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1310 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1311 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1312 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1313 bool Tied, unsigned &NumberOfParts) {
1314 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1315 PrePostActionTy &) {
1316 llvm::Value *ThreadID = getThreadID(CGF, Loc: D.getBeginLoc());
1317 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
1318 llvm::Value *TaskArgs[] = {
1319 UpLoc, ThreadID,
1320 CGF.EmitLoadOfPointerLValue(Ptr: CGF.GetAddrOfLocalVar(VD: TaskTVar),
1321 PtrTy: TaskTVar->getType()->castAs<PointerType>())
1322 .getPointer(CGF)};
1323 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1324 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
1325 args: TaskArgs);
1326 };
1327 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1328 UntiedCodeGen);
1329 CodeGen.setAction(Action);
1330 assert(!ThreadIDVar->getType()->isPointerType() &&
1331 "thread id variable must be of type kmp_int32 for tasks");
1332 const OpenMPDirectiveKind Region =
1333 isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) ? OMPD_taskloop
1334 : OMPD_task;
1335 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: Region);
1336 bool HasCancel = false;
1337 if (const auto *TD = dyn_cast<OMPTaskDirective>(Val: &D))
1338 HasCancel = TD->hasCancel();
1339 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(Val: &D))
1340 HasCancel = TD->hasCancel();
1341 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(Val: &D))
1342 HasCancel = TD->hasCancel();
1343 else if (const auto *TD = dyn_cast<OMPMaskedTaskLoopDirective>(Val: &D))
1344 HasCancel = TD->hasCancel();
1345 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(Val: &D))
1346 HasCancel = TD->hasCancel();
1347 else if (const auto *TD = dyn_cast<OMPParallelMaskedTaskLoopDirective>(Val: &D))
1348 HasCancel = TD->hasCancel();
1349
1350 CodeGenFunction CGF(CGM, true);
1351 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1352 InnermostKind, HasCancel, Action);
1353 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1354 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(S: *CS);
1355 if (!Tied)
1356 NumberOfParts = Action.getNumberOfParts();
1357 return Res;
1358}
1359
1360void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1361 bool AtCurrentPoint) {
1362 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1363 assert(!Elem.ServiceInsertPt && "Insert point is set already.");
1364
1365 llvm::Value *Undef = llvm::UndefValue::get(T: CGF.Int32Ty);
1366 if (AtCurrentPoint) {
1367 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",
1368 CGF.Builder.GetInsertBlock());
1369 } else {
1370 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1371 Elem.ServiceInsertPt->insertAfter(InsertPos: CGF.AllocaInsertPt->getIterator());
1372 }
1373}
1374
1375void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1376 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1377 if (Elem.ServiceInsertPt) {
1378 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1379 Elem.ServiceInsertPt = nullptr;
1380 Ptr->eraseFromParent();
1381 }
1382}
1383
1384static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1385 SourceLocation Loc,
1386 SmallString<128> &Buffer) {
1387 llvm::raw_svector_ostream OS(Buffer);
1388 // Build debug location
1389 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1390 OS << ";";
1391 if (auto *DbgInfo = CGF.getDebugInfo())
1392 OS << DbgInfo->remapDIPath(PLoc.getFilename());
1393 else
1394 OS << PLoc.getFilename();
1395 OS << ";";
1396 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1397 OS << FD->getQualifiedNameAsString();
1398 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1399 return OS.str();
1400}
1401
1402llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1403 SourceLocation Loc,
1404 unsigned Flags, bool EmitLoc) {
1405 uint32_t SrcLocStrSize;
1406 llvm::Constant *SrcLocStr;
1407 if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() ==
1408 llvm::codegenoptions::NoDebugInfo) ||
1409 Loc.isInvalid()) {
1410 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1411 } else {
1412 std::string FunctionName;
1413 std::string FileName;
1414 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1415 FunctionName = FD->getQualifiedNameAsString();
1416 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1417 if (auto *DbgInfo = CGF.getDebugInfo())
1418 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
1419 else
1420 FileName = PLoc.getFilename();
1421 unsigned Line = PLoc.getLine();
1422 unsigned Column = PLoc.getColumn();
1423 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1424 Column, SrcLocStrSize);
1425 }
1426 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1427 return OMPBuilder.getOrCreateIdent(
1428 SrcLocStr, SrcLocStrSize, Flags: llvm::omp::IdentFlag(Flags), Reserve2Flags: Reserved2Flags);
1429}
1430
1431llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1432 SourceLocation Loc) {
1433 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1434 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1435 // the clang invariants used below might be broken.
1436 if (CGM.getLangOpts().OpenMPIRBuilder) {
1437 SmallString<128> Buffer;
1438 OMPBuilder.updateToLocation(Loc: CGF.Builder);
1439 uint32_t SrcLocStrSize;
1440 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1441 LocStr: getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1442 return OMPBuilder.getOrCreateThreadID(
1443 Ident: OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1444 }
1445
1446 llvm::Value *ThreadID = nullptr;
1447 // Check whether we've already cached a load of the thread id in this
1448 // function.
1449 auto I = OpenMPLocThreadIDMap.find(Val: CGF.CurFn);
1450 if (I != OpenMPLocThreadIDMap.end()) {
1451 ThreadID = I->second.ThreadID;
1452 if (ThreadID != nullptr)
1453 return ThreadID;
1454 }
1455 // If exceptions are enabled, do not use parameter to avoid possible crash.
1456 if (auto *OMPRegionInfo =
1457 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
1458 if (OMPRegionInfo->getThreadIDVariable()) {
1459 // Check if this an outlined function with thread id passed as argument.
1460 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1461 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1462 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1463 !CGF.getLangOpts().CXXExceptions ||
1464 CGF.Builder.GetInsertBlock() == TopBlock ||
1465 !isa<llvm::Instruction>(Val: LVal.getPointer(CGF)) ||
1466 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1467 TopBlock ||
1468 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1469 CGF.Builder.GetInsertBlock()) {
1470 ThreadID = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
1471 // If value loaded in entry block, cache it and use it everywhere in
1472 // function.
1473 if (CGF.Builder.GetInsertBlock() == TopBlock)
1474 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;
1475 return ThreadID;
1476 }
1477 }
1478 }
1479
1480 // This is not an outlined function region - need to call __kmpc_int32
1481 // kmpc_global_thread_num(ident_t *loc).
1482 // Generate thread id value and cache this value for use across the
1483 // function.
1484 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1485 if (!Elem.ServiceInsertPt)
1486 setLocThreadIdInsertPt(CGF);
1487 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1488 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);
1489 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
1490 llvm::CallInst *Call = CGF.Builder.CreateCall(
1491 Callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
1492 FnID: OMPRTL___kmpc_global_thread_num),
1493 Args: emitUpdateLocation(CGF, Loc));
1494 Call->setCallingConv(CGF.getRuntimeCC());
1495 Elem.ThreadID = Call;
1496 return Call;
1497}
1498
1499void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1500 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1501 if (OpenMPLocThreadIDMap.count(Val: CGF.CurFn)) {
1502 clearLocThreadIdInsertPt(CGF);
1503 OpenMPLocThreadIDMap.erase(Val: CGF.CurFn);
1504 }
1505 if (auto I = FunctionUDRMap.find(Val: CGF.CurFn); I != FunctionUDRMap.end()) {
1506 for (const auto *D : I->second)
1507 UDRMap.erase(Val: D);
1508 FunctionUDRMap.erase(I);
1509 }
1510 if (auto I = FunctionUDMMap.find(Val: CGF.CurFn); I != FunctionUDMMap.end()) {
1511 for (const auto *D : I->second)
1512 UDMMap.erase(Val: D);
1513 FunctionUDMMap.erase(I);
1514 }
1515 LastprivateConditionalToTypes.erase(Val: CGF.CurFn);
1516 FunctionToUntiedTaskStackMap.erase(Val: CGF.CurFn);
1517}
1518
1519llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1520 return OMPBuilder.IdentPtr;
1521}
1522
1523static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1524convertDeviceClause(const VarDecl *VD) {
1525 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1526 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1527 if (!DevTy)
1528 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1529
1530 switch ((int)*DevTy) { // Avoid -Wcovered-switch-default
1531 case OMPDeclareTargetDeclAttr::DT_Host:
1532 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1533 break;
1534 case OMPDeclareTargetDeclAttr::DT_NoHost:
1535 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1536 break;
1537 case OMPDeclareTargetDeclAttr::DT_Any:
1538 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1539 break;
1540 default:
1541 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1542 break;
1543 }
1544}
1545
1546static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1547convertCaptureClause(const VarDecl *VD) {
1548 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1549 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1550 if (!MapType)
1551 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1552 switch ((int)*MapType) { // Avoid -Wcovered-switch-default
1553 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1554 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1555 break;
1556 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1557 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1558 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1559 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1560 break;
1561 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local:
1562 // MT_Local variables don't need offload entry (device-local).
1563 llvm_unreachable("MT_Local should not reach convertCaptureClause");
1564 break;
1565 default:
1566 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1567 break;
1568 }
1569}
1570
1571static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
1572 CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder,
1573 SourceLocation BeginLoc, llvm::StringRef ParentName = "") {
1574
1575 auto FileInfoCallBack = [&]() {
1576 SourceManager &SM = CGM.getContext().getSourceManager();
1577 PresumedLoc PLoc = SM.getPresumedLoc(Loc: BeginLoc);
1578
1579 if (!CGM.getFileSystem()->exists(Path: PLoc.getFilename()))
1580 PLoc = SM.getPresumedLoc(Loc: BeginLoc, /*UseLineDirectives=*/false);
1581
1582 return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine());
1583 };
1584
1585 return OMPBuilder.getTargetEntryUniqueInfo(CallBack: FileInfoCallBack,
1586 VFS&: *CGM.getFileSystem(), ParentName);
1587}
1588
1589ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1590 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
1591
1592 auto LinkageForVariable = [&VD, this]() {
1593 return CGM.getLLVMLinkageVarDefinition(VD);
1594 };
1595
1596 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1597
1598 llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(
1599 T: CGM.getContext().getPointerType(T: VD->getType()));
1600 llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar(
1601 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
1602 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
1603 IsExternallyVisible: VD->isExternallyVisible(),
1604 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
1605 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
1606 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
1607 TargetTriple: CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, GlobalInitializer: AddrOfGlobal,
1608 VariableLinkage: LinkageForVariable);
1609
1610 if (!addr)
1611 return ConstantAddress::invalid();
1612 return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(D: VD));
1613}
1614
1615llvm::Constant *
1616CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1617 assert(!CGM.getLangOpts().OpenMPUseTLS ||
1618 !CGM.getContext().getTargetInfo().isTLSSupported());
1619 // Lookup the entry, lazily creating it if necessary.
1620 std::string Suffix = getName(Parts: {"cache", ""});
1621 return OMPBuilder.getOrCreateInternalVariable(
1622 Ty: CGM.Int8PtrPtrTy, Name: Twine(CGM.getMangledName(GD: VD)).concat(Suffix).str());
1623}
1624
1625Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1626 const VarDecl *VD,
1627 Address VDAddr,
1628 SourceLocation Loc) {
1629 if (CGM.getLangOpts().OpenMPUseTLS &&
1630 CGM.getContext().getTargetInfo().isTLSSupported())
1631 return VDAddr;
1632
1633 llvm::Type *VarTy = VDAddr.getElementType();
1634 llvm::Value *Args[] = {
1635 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1636 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy),
1637 CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy)),
1638 getOrCreateThreadPrivateCache(VD)};
1639 return Address(
1640 CGF.EmitRuntimeCall(
1641 callee: OMPBuilder.getOrCreateRuntimeFunction(
1642 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1643 args: Args),
1644 CGF.Int8Ty, VDAddr.getAlignment());
1645}
1646
1647void CGOpenMPRuntime::emitThreadPrivateVarInit(
1648 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1649 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1650 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1651 // library.
1652 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1653 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1654 M&: CGM.getModule(), FnID: OMPRTL___kmpc_global_thread_num),
1655 args: OMPLoc);
1656 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1657 // to register constructor/destructor for variable.
1658 llvm::Value *Args[] = {
1659 OMPLoc,
1660 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy),
1661 Ctor, CopyCtor, Dtor};
1662 CGF.EmitRuntimeCall(
1663 callee: OMPBuilder.getOrCreateRuntimeFunction(
1664 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_register),
1665 args: Args);
1666}
1667
1668llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1669 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1670 bool PerformInit, CodeGenFunction *CGF) {
1671 if (CGM.getLangOpts().OpenMPUseTLS &&
1672 CGM.getContext().getTargetInfo().isTLSSupported())
1673 return nullptr;
1674
1675 VD = VD->getDefinition(C&: CGM.getContext());
1676 if (VD && ThreadPrivateWithDefinition.insert(key: CGM.getMangledName(GD: VD)).second) {
1677 QualType ASTTy = VD->getType();
1678
1679 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1680 const Expr *Init = VD->getAnyInitializer();
1681 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1682 // Generate function that re-emits the declaration's initializer into the
1683 // threadprivate copy of the variable VD
1684 CodeGenFunction CtorCGF(CGM);
1685 auto *Dst = ImplicitParamDecl::Create(
1686 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1687 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1688
1689 FunctionArgList Args{Dst};
1690 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1691 resultType: CGM.getContext().VoidPtrTy, args: Args);
1692 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1693 std::string Name = getName(Parts: {"__kmpc_global_ctor_", ""});
1694 llvm::Function *Fn =
1695 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1696 CtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidPtrTy, Fn, FnInfo: FI,
1697 Args, Loc, StartLoc: Loc);
1698 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1699 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1700 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1701 Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(T: ASTTy),
1702 VDAddr.getAlignment());
1703 CtorCGF.EmitAnyExprToMem(E: Init, Location: Arg, Quals: Init->getType().getQualifiers(),
1704 /*IsInitializer=*/true);
1705 ArgVal = CtorCGF.EmitLoadOfScalar(
1706 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1707 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1708 CtorCGF.Builder.CreateStore(Val: ArgVal, Addr: CtorCGF.ReturnValue);
1709 CtorCGF.FinishFunction();
1710 Ctor = Fn;
1711 }
1712 if (VD->getType().isDestructedType() != QualType::DK_none) {
1713 // Generate function that emits destructor call for the threadprivate copy
1714 // of the variable VD
1715 CodeGenFunction DtorCGF(CGM);
1716 auto *Dst = ImplicitParamDecl::Create(
1717 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1718 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1719
1720 FunctionArgList Args{Dst};
1721 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1722 resultType: CGM.getContext().VoidTy, args: Args);
1723 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1724 std::string Name = getName(Parts: {"__kmpc_global_dtor_", ""});
1725 llvm::Function *Fn =
1726 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1727 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: DtorCGF);
1728 DtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn, FnInfo: FI, Args,
1729 Loc, StartLoc: Loc);
1730 // Create a scope with an artificial location for the body of this function.
1731 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: DtorCGF);
1732 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1733 Addr: DtorCGF.GetAddrOfLocalVar(VD: Dst),
1734 /*Volatile=*/false, Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1735 DtorCGF.emitDestroy(
1736 addr: Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), type: ASTTy,
1737 destroyer: DtorCGF.getDestroyer(destructionKind: ASTTy.isDestructedType()),
1738 useEHCleanupForArray: DtorCGF.needsEHCleanup(kind: ASTTy.isDestructedType()));
1739 DtorCGF.FinishFunction();
1740 Dtor = Fn;
1741 }
1742 // Do not emit init function if it is not required.
1743 if (!Ctor && !Dtor)
1744 return nullptr;
1745
1746 // Copying constructor for the threadprivate variable.
1747 // Must be NULL - reserved by runtime, but currently it requires that this
1748 // parameter is always NULL. Otherwise it fires assertion.
1749 CopyCtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1750 if (Ctor == nullptr) {
1751 Ctor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1752 }
1753 if (Dtor == nullptr) {
1754 Dtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1755 }
1756 if (!CGF) {
1757 auto *InitFunctionTy =
1758 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg*/ false);
1759 std::string Name = getName(Parts: {"__omp_threadprivate_init_", ""});
1760 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1761 ty: InitFunctionTy, name: Name, FI: CGM.getTypes().arrangeNullaryFunction());
1762 CodeGenFunction InitCGF(CGM);
1763 FunctionArgList ArgList;
1764 InitCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: InitFunction,
1765 FnInfo: CGM.getTypes().arrangeNullaryFunction(), Args: ArgList,
1766 Loc, StartLoc: Loc);
1767 emitThreadPrivateVarInit(CGF&: InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1768 InitCGF.FinishFunction();
1769 return InitFunction;
1770 }
1771 emitThreadPrivateVarInit(CGF&: *CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1772 }
1773 return nullptr;
1774}
1775
1776void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD,
1777 llvm::GlobalValue *GV) {
1778 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1779 OMPDeclareTargetDeclAttr::getActiveAttr(VD: FD);
1780
1781 // We only need to handle active 'indirect' declare target functions.
1782 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1783 return;
1784
1785 // Get a mangled name to store the new device global in.
1786 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1787 CGM, OMPBuilder, BeginLoc: FD->getCanonicalDecl()->getBeginLoc(), ParentName: FD->getName());
1788 SmallString<128> Name;
1789 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1790
1791 // We need to generate a new global to hold the address of the indirectly
1792 // called device function. Doing this allows us to keep the visibility and
1793 // linkage of the associated function unchanged while allowing the runtime to
1794 // access its value.
1795 llvm::GlobalValue *Addr = GV;
1796 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1797 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1798 C&: CGM.getLLVMContext(),
1799 AddressSpace: CGM.getModule().getDataLayout().getProgramAddressSpace());
1800 Addr = new llvm::GlobalVariable(
1801 CGM.getModule(), FnPtrTy,
1802 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1803 nullptr, llvm::GlobalValue::NotThreadLocal,
1804 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1805 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1806 }
1807
1808 // Register the indirect Vtable:
1809 // This is similar to OMPTargetGlobalVarEntryIndirect, except that the
1810 // size field refers to the size of memory pointed to, not the size of
1811 // the pointer symbol itself (which is implicitly the size of a pointer).
1812 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1813 VarName: Name, Addr, VarSize: CGM.GetTargetTypeStoreSize(Ty: CGM.VoidPtrTy).getQuantity(),
1814 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1815 Linkage: llvm::GlobalValue::WeakODRLinkage);
1816}
1817
1818void CGOpenMPRuntime::registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1819 const VarDecl *VD) {
1820 // TODO: add logic to avoid duplicate vtable registrations per
1821 // translation unit; though for external linkage, this should no
1822 // longer be an issue - or at least we can avoid the issue by
1823 // checking for an existing offloading entry. But, perhaps the
1824 // better approach is to defer emission of the vtables and offload
1825 // entries until later (by tracking a list of items that need to be
1826 // emitted).
1827
1828 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1829
1830 // Generate a new externally visible global to point to the
1831 // internally visible vtable. Doing this allows us to keep the
1832 // visibility and linkage of the associated vtable unchanged while
1833 // allowing the runtime to access its value. The externally
1834 // visible global var needs to be emitted with a unique mangled
1835 // name that won't conflict with similarly named (internal)
1836 // vtables in other translation units.
1837
1838 // Register vtable with source location of dynamic object in map
1839 // clause.
1840 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1841 CGM, OMPBuilder, BeginLoc: VD->getCanonicalDecl()->getBeginLoc(),
1842 ParentName: VTable->getName());
1843
1844 llvm::GlobalVariable *Addr = VTable;
1845 SmallString<128> AddrName;
1846 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name&: AddrName, EntryInfo);
1847 AddrName.append(RHS: "addr");
1848
1849 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1850 Addr = new llvm::GlobalVariable(
1851 CGM.getModule(), VTable->getType(),
1852 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, VTable,
1853 AddrName,
1854 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1855 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1856 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1857 }
1858 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1859 VarName: AddrName, Addr: VTable,
1860 VarSize: CGM.getDataLayout().getTypeAllocSize(Ty: VTable->getInitializer()->getType()),
1861 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable,
1862 Linkage: llvm::GlobalValue::WeakODRLinkage);
1863}
1864
1865void CGOpenMPRuntime::emitAndRegisterVTable(CodeGenModule &CGM,
1866 CXXRecordDecl *CXXRecord,
1867 const VarDecl *VD) {
1868 // Register C++ VTable to OpenMP Offload Entry if it's a new
1869 // CXXRecordDecl.
1870 if (CXXRecord && CXXRecord->hasDefinition() && CXXRecord->isDynamicClass() &&
1871 !CGM.getOpenMPRuntime().VTableDeclMap.contains(Val: CXXRecord)) {
1872 auto Res = CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(Key: CXXRecord, Args&: VD);
1873 if (Res.second) {
1874 CGM.EmitVTable(Class: CXXRecord);
1875 CodeGenVTables VTables = CGM.getVTables();
1876 llvm::GlobalVariable *VTablesAddr = VTables.GetAddrOfVTable(RD: CXXRecord);
1877 assert(VTablesAddr && "Expected non-null VTable address");
1878 // Must set VTables to weak since we're emitting them in multiple TUs now
1879 if (VTablesAddr->hasExternalLinkage())
1880 VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1881 CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTable: VTablesAddr, VD);
1882 // Emit VTable for all the fields containing dynamic CXXRecord
1883 for (const FieldDecl *Field : CXXRecord->fields()) {
1884 if (CXXRecordDecl *RecordDecl = Field->getType()->getAsCXXRecordDecl())
1885 emitAndRegisterVTable(CGM, CXXRecord: RecordDecl, VD);
1886 }
1887 // Emit VTable for all dynamic parent class
1888 for (CXXBaseSpecifier &Base : CXXRecord->bases()) {
1889 if (CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl())
1890 emitAndRegisterVTable(CGM, CXXRecord: BaseDecl, VD);
1891 }
1892 }
1893 }
1894}
1895
1896void CGOpenMPRuntime::registerVTable(const OMPExecutableDirective &D) {
1897 // Register VTable by scanning through the map clause of OpenMP target region.
1898 // Get CXXRecordDecl and VarDecl from Expr.
1899 auto GetVTableDecl = [](const Expr *E) {
1900 QualType VDTy = E->getType();
1901 CXXRecordDecl *CXXRecord = nullptr;
1902 if (const auto *RefType = VDTy->getAs<LValueReferenceType>())
1903 VDTy = RefType->getPointeeType();
1904 if (VDTy->isPointerType())
1905 CXXRecord = VDTy->getPointeeType()->getAsCXXRecordDecl();
1906 else
1907 CXXRecord = VDTy->getAsCXXRecordDecl();
1908
1909 const VarDecl *VD = nullptr;
1910 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1911 // Handle BindingDecls by redirecting to their DecompositionDecl.
1912 if (auto *BD = dyn_cast<BindingDecl>(Val: DRE->getDecl()))
1913 VD = cast<VarDecl>(Val: BD->getDecomposedDecl());
1914 else
1915 VD = cast<VarDecl>(Val: DRE->getDecl());
1916 } else if (auto *MRE = dyn_cast<MemberExpr>(Val: E)) {
1917 if (auto *BaseDRE = dyn_cast<DeclRefExpr>(Val: MRE->getBase())) {
1918 if (auto *BaseVD = dyn_cast<VarDecl>(Val: BaseDRE->getDecl()))
1919 VD = BaseVD;
1920 }
1921 }
1922 return std::pair<CXXRecordDecl *, const VarDecl *>(CXXRecord, VD);
1923 };
1924 // Collect VTable from OpenMP map clause.
1925 for (const auto *C : D.getClausesOfKind<OMPMapClause>()) {
1926 for (const auto *E : C->varlist()) {
1927 auto DeclPair = GetVTableDecl(E);
1928 // Ensure VD is not null
1929 if (DeclPair.second)
1930 emitAndRegisterVTable(CGM, CXXRecord: DeclPair.first, VD: DeclPair.second);
1931 }
1932 }
1933}
1934
1935Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1936 QualType VarType,
1937 StringRef Name) {
1938 std::string Suffix = getName(Parts: {"artificial", ""});
1939 llvm::Type *VarLVType = CGF.ConvertTypeForMem(T: VarType);
1940 llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable(
1941 Ty: VarLVType, Name: Twine(Name).concat(Suffix).str());
1942 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1943 CGM.getTarget().isTLSSupported()) {
1944 GAddr->setThreadLocal(/*Val=*/true);
1945 return Address(GAddr, GAddr->getValueType(),
1946 CGM.getContext().getTypeAlignInChars(T: VarType));
1947 }
1948 std::string CacheSuffix = getName(Parts: {"cache", ""});
1949 llvm::Value *Args[] = {
1950 emitUpdateLocation(CGF, Loc: SourceLocation()),
1951 getThreadID(CGF, Loc: SourceLocation()),
1952 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: GAddr, DestTy: CGM.VoidPtrTy),
1953 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: VarType), DestTy: CGM.SizeTy,
1954 /*isSigned=*/false),
1955 OMPBuilder.getOrCreateInternalVariable(
1956 Ty: CGM.VoidPtrPtrTy,
1957 Name: Twine(Name).concat(Suffix).concat(Suffix: CacheSuffix).str())};
1958 return Address(
1959 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1960 V: CGF.EmitRuntimeCall(
1961 callee: OMPBuilder.getOrCreateRuntimeFunction(
1962 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1963 args: Args),
1964 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
1965 VarLVType, CGM.getContext().getTypeAlignInChars(T: VarType));
1966}
1967
1968void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
1969 const RegionCodeGenTy &ThenGen,
1970 const RegionCodeGenTy &ElseGen) {
1971 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1972
1973 // If the condition constant folds and can be elided, try to avoid emitting
1974 // the condition and the dead arm of the if/else.
1975 bool CondConstant;
1976 if (CGF.ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant)) {
1977 if (CondConstant)
1978 ThenGen(CGF);
1979 else
1980 ElseGen(CGF);
1981 return;
1982 }
1983
1984 // Otherwise, the condition did not fold, or we couldn't elide it. Just
1985 // emit the conditional branch.
1986 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
1987 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock(name: "omp_if.else");
1988 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "omp_if.end");
1989 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, /*TrueCount=*/0);
1990
1991 // Emit the 'then' code.
1992 CGF.EmitBlock(BB: ThenBlock);
1993 ThenGen(CGF);
1994 CGF.EmitBranch(Block: ContBlock);
1995 // Emit the 'else' code if present.
1996 // There is no need to emit line number for unconditional branch.
1997 (void)ApplyDebugLocation::CreateEmpty(CGF);
1998 CGF.EmitBlock(BB: ElseBlock);
1999 ElseGen(CGF);
2000 // There is no need to emit line number for unconditional branch.
2001 (void)ApplyDebugLocation::CreateEmpty(CGF);
2002 CGF.EmitBranch(Block: ContBlock);
2003 // Emit the continuation block for code after the if.
2004 CGF.EmitBlock(BB: ContBlock, /*IsFinished=*/true);
2005}
2006
2007void CGOpenMPRuntime::emitParallelCall(
2008 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
2009 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
2010 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
2011 OpenMPSeverityClauseKind Severity, const Expr *Message) {
2012 if (!CGF.HaveInsertPoint())
2013 return;
2014
2015 // The caller waits for the parallel region to finish, so its captured stack
2016 // variables remain alive until the region completes. Global variables also
2017 // remain alive. Record this lifetime guarantee on the outlined function's
2018 // parameters, even if other code has access to the same storage.
2019 // The storage may still be modified or accessed through other pointers.
2020 // For a captured pointer variable, this guarantees the lifetime of the
2021 // variable itself, not the object it points to.
2022 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
2023 assert(OutlinedFn->arg_size() == CapturedVars.size() + 2 &&
2024 "Expected tid and bound tid before the captured arguments");
2025 for (auto [I, Captured] : llvm::enumerate(First&: CapturedVars)) {
2026 if (Captured->getType()->isPointerTy() &&
2027 llvm::isa<llvm::AllocaInst, llvm::GlobalVariable>(
2028 Val: llvm::getUnderlyingObject(V: Captured)))
2029 OutlinedFn->addParamAttr(ArgNo: I + 2, Kind: llvm::Attribute::NoFreeObj);
2030 }
2031 }
2032
2033 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2034 auto &M = CGM.getModule();
2035 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2036 this](CodeGenFunction &CGF, PrePostActionTy &) {
2037 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2038 llvm::Value *Args[] = {
2039 RTLoc,
2040 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
2041 OutlinedFn};
2042 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2043 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
2044 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2045
2046 llvm::FunctionCallee RTLFn =
2047 OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_fork_call);
2048 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
2049 };
2050 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2051 this](CodeGenFunction &CGF, PrePostActionTy &) {
2052 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2053 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2054 // Build calls:
2055 // __kmpc_serialized_parallel(&Loc, GTid);
2056 llvm::Value *Args[] = {RTLoc, ThreadID};
2057 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2058 M, FnID: OMPRTL___kmpc_serialized_parallel),
2059 args: Args);
2060
2061 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2062 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2063 RawAddress ZeroAddrBound =
2064 CGF.CreateDefaultAlignTempAlloca(Ty: CGF.Int32Ty,
2065 /*Name=*/".bound.zero.addr");
2066 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(/*C*/ 0), Addr: ZeroAddrBound);
2067 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2068 // ThreadId for serialized parallels is 0.
2069 OutlinedFnArgs.push_back(Elt: ThreadIDAddr.emitRawPointer(CGF));
2070 OutlinedFnArgs.push_back(Elt: ZeroAddrBound.getPointer());
2071 OutlinedFnArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2072
2073 // Ensure we do not inline the function. This is trivially true for the ones
2074 // passed to __kmpc_fork_call but the ones called in serialized regions
2075 // could be inlined. This is not a perfect but it is closer to the invariant
2076 // we want, namely, every data environment starts with a new function.
2077 // TODO: We should pass the if condition to the runtime function and do the
2078 // handling there. Much cleaner code.
2079 OutlinedFn->removeFnAttr(Kind: llvm::Attribute::AlwaysInline);
2080 OutlinedFn->addFnAttr(Kind: llvm::Attribute::NoInline);
2081 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, Args: OutlinedFnArgs);
2082
2083 // __kmpc_end_serialized_parallel(&Loc, GTid);
2084 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2085 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2086 M, FnID: OMPRTL___kmpc_end_serialized_parallel),
2087 args: EndArgs);
2088 };
2089 if (IfCond) {
2090 emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
2091 } else {
2092 RegionCodeGenTy ThenRCG(ThenGen);
2093 ThenRCG(CGF);
2094 }
2095}
2096
2097// If we're inside an (outlined) parallel region, use the region info's
2098// thread-ID variable (it is passed in a first argument of the outlined function
2099// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2100// regular serial code region, get thread ID by calling kmp_int32
2101// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2102// return the address of that temp.
2103Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2104 SourceLocation Loc) {
2105 if (auto *OMPRegionInfo =
2106 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2107 if (OMPRegionInfo->getThreadIDVariable())
2108 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2109
2110 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2111 QualType Int32Ty =
2112 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2113 Address ThreadIDTemp =
2114 CGF.CreateMemTempWithoutCast(T: Int32Ty, /*Name*/ ".threadid_temp.");
2115 CGF.EmitStoreOfScalar(value: ThreadID,
2116 lvalue: CGF.MakeAddrLValue(Addr: ThreadIDTemp, T: Int32Ty));
2117
2118 return ThreadIDTemp;
2119}
2120
2121llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2122 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2123 std::string Name = getName(Parts: {Prefix, "var"});
2124 llvm::GlobalVariable *GV =
2125 OMPBuilder.getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
2126 CGM.setDSOLocal(GV);
2127 return GV;
2128}
2129
2130namespace {
2131/// Common pre(post)-action for different OpenMP constructs.
2132class CommonActionTy final : public PrePostActionTy {
2133 llvm::FunctionCallee EnterCallee;
2134 ArrayRef<llvm::Value *> EnterArgs;
2135 llvm::FunctionCallee ExitCallee;
2136 ArrayRef<llvm::Value *> ExitArgs;
2137 bool Conditional;
2138 llvm::BasicBlock *ContBlock = nullptr;
2139
2140public:
2141 CommonActionTy(llvm::FunctionCallee EnterCallee,
2142 ArrayRef<llvm::Value *> EnterArgs,
2143 llvm::FunctionCallee ExitCallee,
2144 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2145 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2146 ExitArgs(ExitArgs), Conditional(Conditional) {}
2147 void Enter(CodeGenFunction &CGF) override {
2148 llvm::Value *EnterRes = CGF.EmitRuntimeCall(callee: EnterCallee, args: EnterArgs);
2149 if (Conditional) {
2150 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(Arg: EnterRes);
2151 auto *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
2152 ContBlock = CGF.createBasicBlock(name: "omp_if.end");
2153 // Generate the branch (If-stmt)
2154 CGF.Builder.CreateCondBr(Cond: CallBool, True: ThenBlock, False: ContBlock);
2155 CGF.EmitBlock(BB: ThenBlock);
2156 }
2157 }
2158 void Done(CodeGenFunction &CGF) {
2159 // Emit the rest of blocks/branches
2160 CGF.EmitBranch(Block: ContBlock);
2161 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
2162 }
2163 void Exit(CodeGenFunction &CGF) override {
2164 CGF.EmitRuntimeCall(callee: ExitCallee, args: ExitArgs);
2165 }
2166};
2167} // anonymous namespace
2168
2169void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2170 StringRef CriticalName,
2171 const RegionCodeGenTy &CriticalOpGen,
2172 SourceLocation Loc, const Expr *Hint) {
2173 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2174 // CriticalOpGen();
2175 // __kmpc_end_critical(ident_t *, gtid, Lock);
2176 // Prepare arguments and build a call to __kmpc_critical
2177 if (!CGF.HaveInsertPoint())
2178 return;
2179 llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction(
2180 M&: CGM.getModule(),
2181 FnID: Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);
2182 llvm::Value *LockVar = getCriticalRegionLock(CriticalName);
2183 unsigned LockVarArgIdx = 2;
2184 if (cast<llvm::GlobalVariable>(Val: LockVar)->getAddressSpace() !=
2185 RuntimeFcn.getFunctionType()
2186 ->getParamType(i: LockVarArgIdx)
2187 ->getPointerAddressSpace())
2188 LockVar = CGF.Builder.CreateAddrSpaceCast(
2189 V: LockVar, DestTy: RuntimeFcn.getFunctionType()->getParamType(i: LockVarArgIdx));
2190 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2191 LockVar};
2192 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args),
2193 std::end(arr&: Args));
2194 if (Hint) {
2195 EnterArgs.push_back(Elt: CGF.Builder.CreateIntCast(
2196 V: CGF.EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, /*isSigned=*/false));
2197 }
2198 CommonActionTy Action(RuntimeFcn, EnterArgs,
2199 OMPBuilder.getOrCreateRuntimeFunction(
2200 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_critical),
2201 Args);
2202 CriticalOpGen.setAction(Action);
2203 emitInlinedDirective(CGF, InnermostKind: OMPD_critical, CodeGen: CriticalOpGen);
2204}
2205
2206void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2207 const RegionCodeGenTy &MasterOpGen,
2208 SourceLocation Loc) {
2209 if (!CGF.HaveInsertPoint())
2210 return;
2211 // if(__kmpc_master(ident_t *, gtid)) {
2212 // MasterOpGen();
2213 // __kmpc_end_master(ident_t *, gtid);
2214 // }
2215 // Prepare arguments and build a call to __kmpc_master
2216 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2217 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2218 M&: CGM.getModule(), FnID: OMPRTL___kmpc_master),
2219 Args,
2220 OMPBuilder.getOrCreateRuntimeFunction(
2221 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_master),
2222 Args,
2223 /*Conditional=*/true);
2224 MasterOpGen.setAction(Action);
2225 emitInlinedDirective(CGF, InnermostKind: OMPD_master, CodeGen: MasterOpGen);
2226 Action.Done(CGF);
2227}
2228
2229void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,
2230 const RegionCodeGenTy &MaskedOpGen,
2231 SourceLocation Loc, const Expr *Filter) {
2232 if (!CGF.HaveInsertPoint())
2233 return;
2234 // if(__kmpc_masked(ident_t *, gtid, filter)) {
2235 // MaskedOpGen();
2236 // __kmpc_end_masked(iden_t *, gtid);
2237 // }
2238 // Prepare arguments and build a call to __kmpc_masked
2239 llvm::Value *FilterVal = Filter
2240 ? CGF.EmitScalarExpr(E: Filter, IgnoreResultAssign: CGF.Int32Ty)
2241 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
2242 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2243 FilterVal};
2244 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2245 getThreadID(CGF, Loc)};
2246 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2247 M&: CGM.getModule(), FnID: OMPRTL___kmpc_masked),
2248 Args,
2249 OMPBuilder.getOrCreateRuntimeFunction(
2250 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_masked),
2251 ArgsEnd,
2252 /*Conditional=*/true);
2253 MaskedOpGen.setAction(Action);
2254 emitInlinedDirective(CGF, InnermostKind: OMPD_masked, CodeGen: MaskedOpGen);
2255 Action.Done(CGF);
2256}
2257
2258void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2259 SourceLocation Loc) {
2260 if (!CGF.HaveInsertPoint())
2261 return;
2262 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2263 OMPBuilder.createTaskyield(Loc: CGF.Builder);
2264 } else {
2265 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2266 llvm::Value *Args[] = {
2267 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2268 llvm::ConstantInt::get(Ty: CGM.IntTy, /*V=*/0, /*isSigned=*/IsSigned: true)};
2269 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2270 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_taskyield),
2271 args: Args);
2272 }
2273
2274 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2275 Region->emitUntiedSwitch(CGF);
2276}
2277
2278void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2279 const RegionCodeGenTy &TaskgroupOpGen,
2280 SourceLocation Loc) {
2281 if (!CGF.HaveInsertPoint())
2282 return;
2283 // __kmpc_taskgroup(ident_t *, gtid);
2284 // TaskgroupOpGen();
2285 // __kmpc_end_taskgroup(ident_t *, gtid);
2286 // Prepare arguments and build a call to __kmpc_taskgroup
2287 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2288 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2289 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskgroup),
2290 Args,
2291 OMPBuilder.getOrCreateRuntimeFunction(
2292 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_taskgroup),
2293 Args);
2294 TaskgroupOpGen.setAction(Action);
2295 emitInlinedDirective(CGF, InnermostKind: OMPD_taskgroup, CodeGen: TaskgroupOpGen);
2296}
2297
2298/// Given an array of pointers to variables, project the address of a
2299/// given variable.
2300static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2301 unsigned Index, const VarDecl *Var) {
2302 // Pull out the pointer to the variable.
2303 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Addr: Array, Index);
2304 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: PtrAddr);
2305
2306 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: Var->getType());
2307 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(D: Var));
2308}
2309
2310static llvm::Value *emitCopyprivateCopyFunction(
2311 CodeGenModule &CGM, llvm::Type *ArgsElemType,
2312 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2313 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2314 SourceLocation Loc) {
2315 ASTContext &C = CGM.getContext();
2316 // void copy_func(void *LHSArg, void *RHSArg);
2317
2318 auto *LHSArg =
2319 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2320 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2321 auto *RHSArg =
2322 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2323 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2324 FunctionArgList Args{LHSArg, RHSArg};
2325 const auto &CGFI =
2326 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
2327 std::string Name =
2328 CGM.getOpenMPRuntime().getName(Parts: {"omp", "copyprivate", "copy_func"});
2329 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
2330 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
2331 M: &CGM.getModule());
2332 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
2333 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
2334 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
2335 Fn->setDoesNotRecurse();
2336 CodeGenFunction CGF(CGM);
2337 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
2338 // Dest = (void*[n])(LHSArg);
2339 // Src = (void*[n])(RHSArg);
2340 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2341 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
2342 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2343 ArgsElemType, CGF.getPointerAlign());
2344 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2345 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
2346 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2347 ArgsElemType, CGF.getPointerAlign());
2348 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2349 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2350 // ...
2351 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2352 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2353 const auto *DestVar =
2354 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: DestExprs[I])->getDecl());
2355 Address DestAddr = emitAddrOfVarFromArray(CGF, Array: LHS, Index: I, Var: DestVar);
2356
2357 const auto *SrcVar =
2358 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: SrcExprs[I])->getDecl());
2359 Address SrcAddr = emitAddrOfVarFromArray(CGF, Array: RHS, Index: I, Var: SrcVar);
2360
2361 const auto *VD = cast<DeclRefExpr>(Val: CopyprivateVars[I])->getDecl();
2362 QualType Type = VD->getType();
2363 CGF.EmitOMPCopy(OriginalType: Type, DestAddr, SrcAddr, DestVD: DestVar, SrcVD: SrcVar, Copy: AssignmentOps[I]);
2364 }
2365 CGF.FinishFunction();
2366 return Fn;
2367}
2368
2369void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2370 const RegionCodeGenTy &SingleOpGen,
2371 SourceLocation Loc,
2372 ArrayRef<const Expr *> CopyprivateVars,
2373 ArrayRef<const Expr *> SrcExprs,
2374 ArrayRef<const Expr *> DstExprs,
2375 ArrayRef<const Expr *> AssignmentOps) {
2376 if (!CGF.HaveInsertPoint())
2377 return;
2378 assert(CopyprivateVars.size() == SrcExprs.size() &&
2379 CopyprivateVars.size() == DstExprs.size() &&
2380 CopyprivateVars.size() == AssignmentOps.size());
2381 ASTContext &C = CGM.getContext();
2382 // int32 did_it = 0;
2383 // if(__kmpc_single(ident_t *, gtid)) {
2384 // SingleOpGen();
2385 // __kmpc_end_single(ident_t *, gtid);
2386 // did_it = 1;
2387 // }
2388 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2389 // <copy_func>, did_it);
2390
2391 Address DidIt = Address::invalid();
2392 if (!CopyprivateVars.empty()) {
2393 // int32 did_it = 0;
2394 QualType KmpInt32Ty =
2395 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2396 DidIt = CGF.CreateMemTempWithoutCast(T: KmpInt32Ty, Name: ".omp.copyprivate.did_it");
2397 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 0), Addr: DidIt);
2398 }
2399 // Prepare arguments and build a call to __kmpc_single
2400 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2401 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2402 M&: CGM.getModule(), FnID: OMPRTL___kmpc_single),
2403 Args,
2404 OMPBuilder.getOrCreateRuntimeFunction(
2405 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_single),
2406 Args,
2407 /*Conditional=*/true);
2408 SingleOpGen.setAction(Action);
2409 emitInlinedDirective(CGF, InnermostKind: OMPD_single, CodeGen: SingleOpGen);
2410 if (DidIt.isValid()) {
2411 // did_it = 1;
2412 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 1), Addr: DidIt);
2413 }
2414 Action.Done(CGF);
2415 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2416 // <copy_func>, did_it);
2417 if (DidIt.isValid()) {
2418 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2419 QualType CopyprivateArrayTy = C.getConstantArrayType(
2420 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
2421 /*IndexTypeQuals=*/0);
2422 // Create a list of all private variables for copyprivate.
2423 Address CopyprivateList = CGF.CreateMemTempWithoutCast(
2424 T: CopyprivateArrayTy, Name: ".omp.copyprivate.cpr_list");
2425 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2426 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: CopyprivateList, Index: I);
2427 CGF.Builder.CreateStore(
2428 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2429 V: CGF.EmitLValue(E: CopyprivateVars[I]).getPointer(CGF),
2430 DestTy: CGF.VoidPtrTy),
2431 Addr: Elem);
2432 }
2433 // Build function that copies private values from single region to all other
2434 // threads in the corresponding parallel region.
2435 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2436 CGM, ArgsElemType: CGF.ConvertTypeForMem(T: CopyprivateArrayTy), CopyprivateVars,
2437 DestExprs: SrcExprs, SrcExprs: DstExprs, AssignmentOps, Loc);
2438 llvm::Value *BufSize = CGF.getTypeSize(Ty: CopyprivateArrayTy);
2439 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2440 Addr: CopyprivateList, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
2441 llvm::Value *DidItVal = CGF.Builder.CreateLoad(Addr: DidIt);
2442 llvm::Value *Args[] = {
2443 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2444 getThreadID(CGF, Loc), // i32 <gtid>
2445 BufSize, // size_t <buf_size>
2446 CL.emitRawPointer(CGF), // void *<copyprivate list>
2447 CpyFn, // void (*) (void *, void *) <copy_func>
2448 DidItVal // i32 did_it
2449 };
2450 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2451 M&: CGM.getModule(), FnID: OMPRTL___kmpc_copyprivate),
2452 args: Args);
2453 }
2454}
2455
2456void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2457 const RegionCodeGenTy &OrderedOpGen,
2458 SourceLocation Loc, bool IsThreads) {
2459 if (!CGF.HaveInsertPoint())
2460 return;
2461 // __kmpc_ordered(ident_t *, gtid);
2462 // OrderedOpGen();
2463 // __kmpc_end_ordered(ident_t *, gtid);
2464 // Prepare arguments and build a call to __kmpc_ordered
2465 if (IsThreads) {
2466 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2467 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2468 M&: CGM.getModule(), FnID: OMPRTL___kmpc_ordered),
2469 Args,
2470 OMPBuilder.getOrCreateRuntimeFunction(
2471 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_ordered),
2472 Args);
2473 OrderedOpGen.setAction(Action);
2474 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2475 return;
2476 }
2477 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2478}
2479
2480unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2481 unsigned Flags;
2482 if (Kind == OMPD_for)
2483 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2484 else if (Kind == OMPD_sections)
2485 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2486 else if (Kind == OMPD_single)
2487 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2488 else if (Kind == OMPD_barrier)
2489 Flags = OMP_IDENT_BARRIER_EXPL;
2490 else
2491 Flags = OMP_IDENT_BARRIER_IMPL;
2492 return Flags;
2493}
2494
2495void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2496 CodeGenFunction &CGF, const OMPLoopDirective &S,
2497 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2498 // Check if the loop directive is actually a doacross loop directive. In this
2499 // case choose static, 1 schedule.
2500 if (llvm::any_of(
2501 Range: S.getClausesOfKind<OMPOrderedClause>(),
2502 P: [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2503 ScheduleKind = OMPC_SCHEDULE_static;
2504 // Chunk size is 1 in this case.
2505 llvm::APInt ChunkSize(32, 1);
2506 ChunkExpr = IntegerLiteral::Create(
2507 C: CGF.getContext(), V: ChunkSize,
2508 type: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
2509 l: SourceLocation());
2510 }
2511}
2512
2513void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2514 OpenMPDirectiveKind Kind, bool EmitChecks,
2515 bool ForceSimpleCall) {
2516 // Check if we should use the OMPBuilder
2517 auto *OMPRegionInfo =
2518 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo);
2519 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2520 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2521 cantFail(ValOrErr: OMPBuilder.createBarrier(Loc: CGF.Builder, Kind, ForceSimpleCall,
2522 CheckCancelFlag: EmitChecks));
2523 CGF.Builder.restoreIP(IP: AfterIP);
2524 return;
2525 }
2526
2527 if (!CGF.HaveInsertPoint())
2528 return;
2529 // Build call __kmpc_cancel_barrier(loc, thread_id);
2530 // Build call __kmpc_barrier(loc, thread_id);
2531 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2532 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2533 // thread_id);
2534 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2535 getThreadID(CGF, Loc)};
2536 if (OMPRegionInfo) {
2537 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2538 llvm::Value *Result = CGF.EmitRuntimeCall(
2539 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
2540 FnID: OMPRTL___kmpc_cancel_barrier),
2541 args: Args);
2542 if (EmitChecks) {
2543 // if (__kmpc_cancel_barrier()) {
2544 // exit from construct;
2545 // }
2546 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
2547 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
2548 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
2549 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
2550 CGF.EmitBlock(BB: ExitBB);
2551 // exit from construct;
2552 CodeGenFunction::JumpDest CancelDestination =
2553 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
2554 CGF.EmitBranchThroughCleanup(Dest: CancelDestination);
2555 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
2556 }
2557 return;
2558 }
2559 }
2560 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2561 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
2562 args: Args);
2563}
2564
2565void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,
2566 Expr *ME, bool IsFatal) {
2567 llvm::Value *MVL = ME ? CGF.EmitScalarExpr(E: ME)
2568 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2569 // Build call void __kmpc_error(ident_t *loc, int severity, const char
2570 // *message)
2571 llvm::Value *Args[] = {
2572 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/EmitLoc: true),
2573 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: IsFatal ? 2 : 1),
2574 CGF.Builder.CreatePointerCast(V: MVL, DestTy: CGM.Int8PtrTy)};
2575 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2576 M&: CGM.getModule(), FnID: OMPRTL___kmpc_error),
2577 args: Args);
2578}
2579
2580/// Map the OpenMP loop schedule to the runtime enumeration.
2581static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2582 bool Chunked, bool Ordered) {
2583 switch (ScheduleKind) {
2584 case OMPC_SCHEDULE_static:
2585 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2586 : (Ordered ? OMP_ord_static : OMP_sch_static);
2587 case OMPC_SCHEDULE_dynamic:
2588 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2589 case OMPC_SCHEDULE_guided:
2590 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2591 case OMPC_SCHEDULE_runtime:
2592 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2593 case OMPC_SCHEDULE_auto:
2594 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2595 case OMPC_SCHEDULE_unknown:
2596 assert(!Chunked && "chunk was specified but schedule kind not known");
2597 return Ordered ? OMP_ord_static : OMP_sch_static;
2598 }
2599 llvm_unreachable("Unexpected runtime schedule");
2600}
2601
2602/// Map the OpenMP distribute schedule to the runtime enumeration.
2603static OpenMPSchedType
2604getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2605 // only static is allowed for dist_schedule
2606 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2607}
2608
2609bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2610 bool Chunked) const {
2611 OpenMPSchedType Schedule =
2612 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2613 return Schedule == OMP_sch_static;
2614}
2615
2616bool CGOpenMPRuntime::isStaticNonchunked(
2617 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2618 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2619 return Schedule == OMP_dist_sch_static;
2620}
2621
2622bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2623 bool Chunked) const {
2624 OpenMPSchedType Schedule =
2625 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2626 return Schedule == OMP_sch_static_chunked;
2627}
2628
2629bool CGOpenMPRuntime::isStaticChunked(
2630 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2631 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2632 return Schedule == OMP_dist_sch_static_chunked;
2633}
2634
2635bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2636 OpenMPSchedType Schedule =
2637 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2638 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
2639 return Schedule != OMP_sch_static;
2640}
2641
2642static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2643 OpenMPScheduleClauseModifier M1,
2644 OpenMPScheduleClauseModifier M2) {
2645 int Modifier = 0;
2646 switch (M1) {
2647 case OMPC_SCHEDULE_MODIFIER_monotonic:
2648 Modifier = OMP_sch_modifier_monotonic;
2649 break;
2650 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2651 Modifier = OMP_sch_modifier_nonmonotonic;
2652 break;
2653 case OMPC_SCHEDULE_MODIFIER_simd:
2654 if (Schedule == OMP_sch_static_chunked)
2655 Schedule = OMP_sch_static_balanced_chunked;
2656 break;
2657 case OMPC_SCHEDULE_MODIFIER_last:
2658 case OMPC_SCHEDULE_MODIFIER_unknown:
2659 break;
2660 }
2661 switch (M2) {
2662 case OMPC_SCHEDULE_MODIFIER_monotonic:
2663 Modifier = OMP_sch_modifier_monotonic;
2664 break;
2665 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2666 Modifier = OMP_sch_modifier_nonmonotonic;
2667 break;
2668 case OMPC_SCHEDULE_MODIFIER_simd:
2669 if (Schedule == OMP_sch_static_chunked)
2670 Schedule = OMP_sch_static_balanced_chunked;
2671 break;
2672 case OMPC_SCHEDULE_MODIFIER_last:
2673 case OMPC_SCHEDULE_MODIFIER_unknown:
2674 break;
2675 }
2676 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2677 // If the static schedule kind is specified or if the ordered clause is
2678 // specified, and if the nonmonotonic modifier is not specified, the effect is
2679 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2680 // modifier is specified, the effect is as if the nonmonotonic modifier is
2681 // specified.
2682 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2683 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2684 Schedule == OMP_sch_static_balanced_chunked ||
2685 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2686 Schedule == OMP_dist_sch_static_chunked ||
2687 Schedule == OMP_dist_sch_static ||
2688 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone))
2689 Modifier = OMP_sch_modifier_nonmonotonic;
2690 }
2691 return Schedule | Modifier;
2692}
2693
2694void CGOpenMPRuntime::emitForDispatchInit(
2695 CodeGenFunction &CGF, SourceLocation Loc,
2696 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2697 bool Ordered, const DispatchRTInput &DispatchValues) {
2698 if (!CGF.HaveInsertPoint())
2699 return;
2700 OpenMPSchedType Schedule = getRuntimeSchedule(
2701 ScheduleKind: ScheduleKind.Schedule, Chunked: DispatchValues.Chunk != nullptr, Ordered);
2702 assert(Ordered ||
2703 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2704 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2705 Schedule != OMP_sch_static_balanced_chunked));
2706 // Call __kmpc_dispatch_init(
2707 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2708 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2709 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2710
2711 // If the Chunk was not specified in the clause - use default value 1.
2712 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2713 : CGF.Builder.getIntN(N: IVSize, C: 1);
2714 llvm::Value *Args[] = {
2715 emitUpdateLocation(CGF, Loc),
2716 getThreadID(CGF, Loc),
2717 CGF.Builder.getInt32(C: addMonoNonMonoModifier(
2718 CGM, Schedule, M1: ScheduleKind.M1, M2: ScheduleKind.M2)), // Schedule type
2719 DispatchValues.LB, // Lower
2720 DispatchValues.UB, // Upper
2721 CGF.Builder.getIntN(N: IVSize, C: 1), // Stride
2722 Chunk // Chunk
2723 };
2724 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2725 args: Args);
2726}
2727
2728void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
2729 SourceLocation Loc) {
2730 if (!CGF.HaveInsertPoint())
2731 return;
2732 // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid);
2733 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2734 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchDeinitFunction(), args: Args);
2735}
2736
2737static void emitForStaticInitCall(
2738 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2739 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2740 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2741 const CGOpenMPRuntime::StaticRTInput &Values) {
2742 if (!CGF.HaveInsertPoint())
2743 return;
2744
2745 assert(!Values.Ordered);
2746 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2747 Schedule == OMP_sch_static_balanced_chunked ||
2748 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2749 Schedule == OMP_dist_sch_static ||
2750 Schedule == OMP_dist_sch_static_chunked ||
2751 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone);
2752
2753 // Call __kmpc_for_static_init(
2754 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2755 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2756 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2757 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2758 llvm::Value *Chunk = Values.Chunk;
2759 if (Chunk == nullptr) {
2760 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2761 Schedule == OMP_dist_sch_static) &&
2762 "expected static non-chunked schedule");
2763 // If the Chunk was not specified in the clause - use default value 1.
2764 Chunk = CGF.Builder.getIntN(N: Values.IVSize, C: 1);
2765 } else {
2766 assert((Schedule == OMP_sch_static_chunked ||
2767 Schedule == OMP_sch_static_balanced_chunked ||
2768 Schedule == OMP_ord_static_chunked ||
2769 Schedule == OMP_dist_sch_static_chunked ||
2770 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) &&
2771 "expected static chunked schedule");
2772 }
2773 llvm::Value *Args[] = {
2774 UpdateLocation,
2775 ThreadId,
2776 CGF.Builder.getInt32(C: addMonoNonMonoModifier(CGM&: CGF.CGM, Schedule, M1,
2777 M2)), // Schedule type
2778 Values.IL.emitRawPointer(CGF), // &isLastIter
2779 Values.LB.emitRawPointer(CGF), // &LB
2780 Values.UB.emitRawPointer(CGF), // &UB
2781 Values.ST.emitRawPointer(CGF), // &Stride
2782 CGF.Builder.getIntN(N: Values.IVSize, C: 1), // Incr
2783 Chunk // Chunk
2784 };
2785 CGF.EmitRuntimeCall(callee: ForStaticInitFunction, args: Args);
2786}
2787
2788void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2789 SourceLocation Loc,
2790 OpenMPDirectiveKind DKind,
2791 const OpenMPScheduleTy &ScheduleKind,
2792 const StaticRTInput &Values) {
2793 OpenMPSchedType ScheduleNum =
2794 ScheduleKind.UseFusedDistChunkSchedule
2795 ? OMP_dist_sch_static_chunked_sch_static_chunkone
2796 : getRuntimeSchedule(ScheduleKind: ScheduleKind.Schedule, Chunked: Values.Chunk != nullptr,
2797 Ordered: Values.Ordered);
2798 assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&
2799 "Expected loop-based or sections-based directive.");
2800 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2801 Flags: isOpenMPLoopDirective(DKind)
2802 ? OMP_IDENT_WORK_LOOP
2803 : OMP_IDENT_WORK_SECTIONS);
2804 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2805 llvm::FunctionCallee StaticInitFunction =
2806 OMPBuilder.createForStaticInitFunction(IVSize: Values.IVSize, IVSigned: Values.IVSigned,
2807 IsGPUDistribute: false);
2808 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2809 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2810 Schedule: ScheduleNum, M1: ScheduleKind.M1, M2: ScheduleKind.M2, Values);
2811}
2812
2813void CGOpenMPRuntime::emitDistributeStaticInit(
2814 CodeGenFunction &CGF, SourceLocation Loc,
2815 OpenMPDistScheduleClauseKind SchedKind,
2816 const CGOpenMPRuntime::StaticRTInput &Values) {
2817 OpenMPSchedType ScheduleNum =
2818 getRuntimeSchedule(ScheduleKind: SchedKind, Chunked: Values.Chunk != nullptr);
2819 llvm::Value *UpdatedLocation =
2820 emitUpdateLocation(CGF, Loc, Flags: OMP_IDENT_WORK_DISTRIBUTE);
2821 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2822 llvm::FunctionCallee StaticInitFunction;
2823 bool isGPUDistribute =
2824 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU();
2825 StaticInitFunction = OMPBuilder.createForStaticInitFunction(
2826 IVSize: Values.IVSize, IVSigned: Values.IVSigned, IsGPUDistribute: isGPUDistribute);
2827
2828 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2829 Schedule: ScheduleNum, M1: OMPC_SCHEDULE_MODIFIER_unknown,
2830 M2: OMPC_SCHEDULE_MODIFIER_unknown, Values);
2831}
2832
2833void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2834 SourceLocation Loc,
2835 OpenMPDirectiveKind DKind) {
2836 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2837 DKind == OMPD_sections) &&
2838 "Expected distribute, for, or sections directive kind");
2839 if (!CGF.HaveInsertPoint())
2840 return;
2841 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2842 llvm::Value *Args[] = {
2843 emitUpdateLocation(CGF, Loc,
2844 Flags: isOpenMPDistributeDirective(DKind) ||
2845 (DKind == OMPD_target_teams_loop)
2846 ? OMP_IDENT_WORK_DISTRIBUTE
2847 : isOpenMPLoopDirective(DKind)
2848 ? OMP_IDENT_WORK_LOOP
2849 : OMP_IDENT_WORK_SECTIONS),
2850 getThreadID(CGF, Loc)};
2851 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2852 if (isOpenMPDistributeDirective(DKind) &&
2853 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU())
2854 CGF.EmitRuntimeCall(
2855 callee: OMPBuilder.getOrCreateRuntimeFunction(
2856 M&: CGM.getModule(), FnID: OMPRTL___kmpc_distribute_static_fini),
2857 args: Args);
2858 else
2859 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2860 M&: CGM.getModule(), FnID: OMPRTL___kmpc_for_static_fini),
2861 args: Args);
2862}
2863
2864void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2865 SourceLocation Loc,
2866 unsigned IVSize,
2867 bool IVSigned) {
2868 if (!CGF.HaveInsertPoint())
2869 return;
2870 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2871 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2872 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2873 args: Args);
2874}
2875
2876llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2877 SourceLocation Loc, unsigned IVSize,
2878 bool IVSigned, Address IL,
2879 Address LB, Address UB,
2880 Address ST) {
2881 // Call __kmpc_dispatch_next(
2882 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2883 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2884 // kmp_int[32|64] *p_stride);
2885 llvm::Value *Args[] = {
2886 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2887 IL.emitRawPointer(CGF), // &isLastIter
2888 LB.emitRawPointer(CGF), // &Lower
2889 UB.emitRawPointer(CGF), // &Upper
2890 ST.emitRawPointer(CGF) // &Stride
2891 };
2892 llvm::Value *Call = CGF.EmitRuntimeCall(
2893 callee: OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), args: Args);
2894 return CGF.EmitScalarConversion(
2895 Src: Call, SrcTy: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/1),
2896 DstTy: CGF.getContext().BoolTy, Loc);
2897}
2898
2899llvm::Value *CGOpenMPRuntime::emitMessageClause(CodeGenFunction &CGF,
2900 const Expr *Message,
2901 SourceLocation Loc) {
2902 if (!Message)
2903 return llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2904 return CGF.EmitScalarExpr(E: Message);
2905}
2906
2907llvm::Value *
2908CGOpenMPRuntime::emitSeverityClause(OpenMPSeverityClauseKind Severity,
2909 SourceLocation Loc) {
2910 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is
2911 // as if sev-level is fatal."
2912 return llvm::ConstantInt::get(Ty: CGM.Int32Ty,
2913 V: Severity == OMPC_SEVERITY_warning ? 1 : 2);
2914}
2915
2916void CGOpenMPRuntime::emitNumThreadsClause(
2917 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
2918 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
2919 SourceLocation SeverityLoc, const Expr *Message,
2920 SourceLocation MessageLoc) {
2921 if (!CGF.HaveInsertPoint())
2922 return;
2923 llvm::SmallVector<llvm::Value *, 4> Args(
2924 {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2925 CGF.Builder.CreateIntCast(V: NumThreads, DestTy: CGF.Int32Ty, /*isSigned*/ true)});
2926 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2927 // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity,
2928 // messsage) if strict modifier is used.
2929 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2930 if (Modifier == OMPC_NUMTHREADS_strict) {
2931 FnID = OMPRTL___kmpc_push_num_threads_strict;
2932 Args.push_back(Elt: emitSeverityClause(Severity, Loc: SeverityLoc));
2933 Args.push_back(Elt: emitMessageClause(CGF, Message, Loc: MessageLoc));
2934 }
2935 CGF.EmitRuntimeCall(
2936 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args);
2937}
2938
2939void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2940 ProcBindKind ProcBind,
2941 SourceLocation Loc) {
2942 if (!CGF.HaveInsertPoint())
2943 return;
2944 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");
2945 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2946 llvm::Value *Args[] = {
2947 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2948 llvm::ConstantInt::get(Ty: CGM.IntTy, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
2949 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2950 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_proc_bind),
2951 args: Args);
2952}
2953
2954void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2955 SourceLocation Loc, llvm::AtomicOrdering AO) {
2956 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2957 OMPBuilder.createFlush(Loc: CGF.Builder);
2958 } else {
2959 if (!CGF.HaveInsertPoint())
2960 return;
2961 // Build call void __kmpc_flush(ident_t *loc)
2962 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2963 M&: CGM.getModule(), FnID: OMPRTL___kmpc_flush),
2964 args: emitUpdateLocation(CGF, Loc));
2965 }
2966}
2967
2968namespace {
2969/// Indexes of fields for type kmp_task_t.
2970enum KmpTaskTFields {
2971 /// List of shared variables.
2972 KmpTaskTShareds,
2973 /// Task routine.
2974 KmpTaskTRoutine,
2975 /// Partition id for the untied tasks.
2976 KmpTaskTPartId,
2977 /// Function with call of destructors for private variables.
2978 Data1,
2979 /// Task priority.
2980 Data2,
2981 /// (Taskloops only) Lower bound.
2982 KmpTaskTLowerBound,
2983 /// (Taskloops only) Upper bound.
2984 KmpTaskTUpperBound,
2985 /// (Taskloops only) Stride.
2986 KmpTaskTStride,
2987 /// (Taskloops only) Is last iteration flag.
2988 KmpTaskTLastIter,
2989 /// (Taskloops only) Reduction data.
2990 KmpTaskTReductions,
2991};
2992} // anonymous namespace
2993
2994void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
2995 // If we are in simd mode or there are no entries, we don't need to do
2996 // anything.
2997 if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty())
2998 return;
2999
3000 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
3001 [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
3002 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
3003 SourceLocation Loc;
3004 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
3005 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
3006 E = CGM.getContext().getSourceManager().fileinfo_end();
3007 I != E; ++I) {
3008 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
3009 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
3010 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
3011 SourceFile: I->getFirst(), Line: EntryInfo.Line, Col: 1);
3012 break;
3013 }
3014 }
3015 }
3016 switch (Kind) {
3017 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
3018 CGM.getDiags().Report(Loc,
3019 DiagID: diag::err_target_region_offloading_entry_incorrect)
3020 << EntryInfo.ParentName;
3021 } break;
3022 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
3023 CGM.getDiags().Report(
3024 Loc, DiagID: diag::err_target_var_offloading_entry_incorrect_with_parent)
3025 << EntryInfo.ParentName;
3026 } break;
3027 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
3028 CGM.getDiags().Report(DiagID: diag::err_target_var_offloading_entry_incorrect);
3029 } break;
3030 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: {
3031 unsigned DiagID = CGM.getDiags().getCustomDiagID(
3032 L: DiagnosticsEngine::Error, FormatString: "Offloading entry for indirect declare "
3033 "target variable is incorrect: the "
3034 "address is invalid.");
3035 CGM.getDiags().Report(DiagID);
3036 } break;
3037 }
3038 };
3039
3040 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
3041}
3042
3043void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3044 if (!KmpRoutineEntryPtrTy) {
3045 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3046 ASTContext &C = CGM.getContext();
3047 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3048 FunctionProtoType::ExtProtoInfo EPI;
3049 KmpRoutineEntryPtrQTy = C.getPointerType(
3050 T: C.getFunctionType(ResultTy: KmpInt32Ty, Args: KmpRoutineEntryTyArgs, EPI));
3051 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(T: KmpRoutineEntryPtrQTy);
3052 }
3053}
3054
3055namespace {
3056struct PrivateHelpersTy {
3057 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3058 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3059 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3060 PrivateElemInit(PrivateElemInit) {}
3061 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3062 const Expr *OriginalRef = nullptr;
3063 const VarDecl *Original = nullptr;
3064 const VarDecl *PrivateCopy = nullptr;
3065 const VarDecl *PrivateElemInit = nullptr;
3066 bool isLocalPrivate() const {
3067 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3068 }
3069};
3070typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3071} // anonymous namespace
3072
3073/// For BindingDecls, returns the DecomposedDecl as the original VarDecl.
3074/// For regular VarDecls, returns the VarDecl itself.
3075static const VarDecl *getOriginalVarDecl(const ValueDecl *Decl) {
3076 if (const auto *BD = dyn_cast<BindingDecl>(Val: Decl))
3077 return cast<VarDecl>(Val: BD->getDecomposedDecl());
3078 return cast<VarDecl>(Val: Decl);
3079}
3080
3081static bool isAllocatableDecl(const VarDecl *VD) {
3082 const VarDecl *CVD = VD->getCanonicalDecl();
3083 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3084 return false;
3085 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3086 // Use the default allocation.
3087 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3088 !AA->getAllocator());
3089}
3090
3091static RecordDecl *
3092createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3093 if (!Privates.empty()) {
3094 ASTContext &C = CGM.getContext();
3095 // Build struct .kmp_privates_t. {
3096 // /* private vars */
3097 // };
3098 RecordDecl *RD = C.buildImplicitRecord(Name: ".kmp_privates.t");
3099 RD->startDefinition();
3100 for (const auto &Pair : Privates) {
3101 const VarDecl *VD = Pair.second.Original;
3102 const VarDecl *PrivateCopy = Pair.second.PrivateCopy;
3103 // For BindingDecls, use PrivateCopy type (binding's actual type).
3104 // For regular variables, use Original type to preserve qualifiers.
3105 // Check OriginalRef to detect BindingDecls since Original may be the
3106 // DecompositionDecl.
3107 bool IsBinding =
3108 Pair.second.OriginalRef &&
3109 isa<BindingDecl>(
3110 Val: cast<DeclRefExpr>(Val: Pair.second.OriginalRef)->getDecl());
3111 QualType Type = IsBinding ? PrivateCopy->getType().getNonReferenceType()
3112 : VD->getType().getNonReferenceType();
3113
3114 // If the private variable is a local variable with lvalue ref type,
3115 // allocate the pointer instead of the pointee type.
3116 if (Pair.second.isLocalPrivate()) {
3117 if (VD->getType()->isLValueReferenceType())
3118 Type = C.getPointerType(T: Type);
3119 if (isAllocatableDecl(VD))
3120 Type = C.getPointerType(T: Type);
3121 }
3122 FieldDecl *FD = addFieldToRecordDecl(C, DC: RD, FieldTy: Type);
3123 if (VD->hasAttrs()) {
3124 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3125 E(VD->getAttrs().end());
3126 I != E; ++I)
3127 FD->addAttr(A: *I);
3128 }
3129 }
3130 RD->completeDefinition();
3131 return RD;
3132 }
3133 return nullptr;
3134}
3135
3136static RecordDecl *
3137createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3138 QualType KmpInt32Ty,
3139 QualType KmpRoutineEntryPointerQTy) {
3140 ASTContext &C = CGM.getContext();
3141 // Build struct kmp_task_t {
3142 // void * shareds;
3143 // kmp_routine_entry_t routine;
3144 // kmp_int32 part_id;
3145 // kmp_cmplrdata_t data1;
3146 // kmp_cmplrdata_t data2;
3147 // For taskloops additional fields:
3148 // kmp_uint64 lb;
3149 // kmp_uint64 ub;
3150 // kmp_int64 st;
3151 // kmp_int32 liter;
3152 // void * reductions;
3153 // };
3154 RecordDecl *UD = C.buildImplicitRecord(Name: "kmp_cmplrdata_t", TK: TagTypeKind::Union);
3155 UD->startDefinition();
3156 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpInt32Ty);
3157 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpRoutineEntryPointerQTy);
3158 UD->completeDefinition();
3159 CanQualType KmpCmplrdataTy = C.getCanonicalTagType(TD: UD);
3160 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t");
3161 RD->startDefinition();
3162 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3163 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpRoutineEntryPointerQTy);
3164 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3165 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3166 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3167 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3168 QualType KmpUInt64Ty =
3169 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3170 QualType KmpInt64Ty =
3171 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3172 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3173 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3174 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt64Ty);
3175 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3176 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3177 }
3178 RD->completeDefinition();
3179 return RD;
3180}
3181
3182static RecordDecl *
3183createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3184 ArrayRef<PrivateDataTy> Privates) {
3185 ASTContext &C = CGM.getContext();
3186 // Build struct kmp_task_t_with_privates {
3187 // kmp_task_t task_data;
3188 // .kmp_privates_t. privates;
3189 // };
3190 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t_with_privates");
3191 RD->startDefinition();
3192 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpTaskTQTy);
3193 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3194 addFieldToRecordDecl(C, DC: RD, FieldTy: C.getCanonicalTagType(TD: PrivateRD));
3195 RD->completeDefinition();
3196 return RD;
3197}
3198
3199/// Emit a proxy function which accepts kmp_task_t as the second
3200/// argument.
3201/// \code
3202/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3203/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3204/// For taskloops:
3205/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3206/// tt->reductions, tt->shareds);
3207/// return 0;
3208/// }
3209/// \endcode
3210static llvm::Function *
3211emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3212 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3213 QualType KmpTaskTWithPrivatesPtrQTy,
3214 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3215 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3216 llvm::Value *TaskPrivatesMap) {
3217 ASTContext &C = CGM.getContext();
3218 auto *GtidArg =
3219 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3220 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3221 auto *TaskTypeArg = ImplicitParamDecl::Create(
3222 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3223 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3224 FunctionArgList Args{GtidArg, TaskTypeArg};
3225 const auto &TaskEntryFnInfo =
3226 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3227 llvm::FunctionType *TaskEntryTy =
3228 CGM.getTypes().GetFunctionType(Info: TaskEntryFnInfo);
3229 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_entry", ""});
3230 auto *TaskEntry = llvm::Function::Create(
3231 Ty: TaskEntryTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3232 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskEntry, FI: TaskEntryFnInfo);
3233 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3234 TaskEntry->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3235 TaskEntry->setDoesNotRecurse();
3236 CodeGenFunction CGF(CGM);
3237 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: TaskEntry, FnInfo: TaskEntryFnInfo, Args,
3238 Loc, StartLoc: Loc);
3239
3240 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3241 // tt,
3242 // For taskloops:
3243 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3244 // tt->task_data.shareds);
3245 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3246 Addr: CGF.GetAddrOfLocalVar(VD: GtidArg), /*Volatile=*/false, Ty: KmpInt32Ty, Loc);
3247 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3248 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3249 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3250 const auto *KmpTaskTWithPrivatesQTyRD =
3251 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3252 LValue Base =
3253 CGF.EmitLValueForField(Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3254 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3255 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
3256 LValue PartIdLVal = CGF.EmitLValueForField(Base, Field: *PartIdFI);
3257 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3258
3259 auto SharedsFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds);
3260 LValue SharedsLVal = CGF.EmitLValueForField(Base, Field: *SharedsFI);
3261 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3262 V: CGF.EmitLoadOfScalar(lvalue: SharedsLVal, Loc),
3263 DestTy: CGF.ConvertTypeForMem(T: SharedsPtrTy));
3264
3265 auto PrivatesFI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin(), n: 1);
3266 llvm::Value *PrivatesParam;
3267 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3268 LValue PrivatesLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PrivatesFI);
3269 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3270 V: PrivatesLVal.getPointer(CGF), DestTy: CGF.VoidPtrTy);
3271 } else {
3272 PrivatesParam = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
3273 }
3274
3275 llvm::Value *CommonArgs[] = {
3276 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3277 CGF.Builder
3278 .CreatePointerBitCastOrAddrSpaceCast(Addr: TDBase.getAddress(),
3279 Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty)
3280 .emitRawPointer(CGF)};
3281 SmallVector<llvm::Value *, 16> CallArgs(std::begin(arr&: CommonArgs),
3282 std::end(arr&: CommonArgs));
3283 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3284 auto LBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound);
3285 LValue LBLVal = CGF.EmitLValueForField(Base, Field: *LBFI);
3286 llvm::Value *LBParam = CGF.EmitLoadOfScalar(lvalue: LBLVal, Loc);
3287 auto UBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound);
3288 LValue UBLVal = CGF.EmitLValueForField(Base, Field: *UBFI);
3289 llvm::Value *UBParam = CGF.EmitLoadOfScalar(lvalue: UBLVal, Loc);
3290 auto StFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride);
3291 LValue StLVal = CGF.EmitLValueForField(Base, Field: *StFI);
3292 llvm::Value *StParam = CGF.EmitLoadOfScalar(lvalue: StLVal, Loc);
3293 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3294 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3295 llvm::Value *LIParam = CGF.EmitLoadOfScalar(lvalue: LILVal, Loc);
3296 auto RFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions);
3297 LValue RLVal = CGF.EmitLValueForField(Base, Field: *RFI);
3298 llvm::Value *RParam = CGF.EmitLoadOfScalar(lvalue: RLVal, Loc);
3299 CallArgs.push_back(Elt: LBParam);
3300 CallArgs.push_back(Elt: UBParam);
3301 CallArgs.push_back(Elt: StParam);
3302 CallArgs.push_back(Elt: LIParam);
3303 CallArgs.push_back(Elt: RParam);
3304 }
3305 CallArgs.push_back(Elt: SharedsParam);
3306
3307 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskFunction,
3308 Args: CallArgs);
3309 CGF.EmitStoreThroughLValue(Src: RValue::get(V: CGF.Builder.getInt32(/*C=*/0)),
3310 Dst: CGF.MakeAddrLValue(Addr: CGF.ReturnValue, T: KmpInt32Ty));
3311 CGF.FinishFunction();
3312 return TaskEntry;
3313}
3314
3315static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3316 SourceLocation Loc,
3317 QualType KmpInt32Ty,
3318 QualType KmpTaskTWithPrivatesPtrQTy,
3319 QualType KmpTaskTWithPrivatesQTy) {
3320 ASTContext &C = CGM.getContext();
3321 auto *GtidArg =
3322 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3323 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3324 auto *TaskTypeArg = ImplicitParamDecl::Create(
3325 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3326 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3327 FunctionArgList Args{GtidArg, TaskTypeArg};
3328 const auto &DestructorFnInfo =
3329 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3330 llvm::FunctionType *DestructorFnTy =
3331 CGM.getTypes().GetFunctionType(Info: DestructorFnInfo);
3332 std::string Name =
3333 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_destructor", ""});
3334 auto *DestructorFn =
3335 llvm::Function::Create(Ty: DestructorFnTy, Linkage: llvm::GlobalValue::InternalLinkage,
3336 N: Name, M: &CGM.getModule());
3337 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: DestructorFn,
3338 FI: DestructorFnInfo);
3339 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3340 DestructorFn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3341 DestructorFn->setDoesNotRecurse();
3342 CodeGenFunction CGF(CGM);
3343 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: DestructorFn, FnInfo: DestructorFnInfo,
3344 Args, Loc, StartLoc: Loc);
3345
3346 LValue Base = CGF.EmitLoadOfPointerLValue(
3347 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3348 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3349 const auto *KmpTaskTWithPrivatesQTyRD =
3350 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3351 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3352 Base = CGF.EmitLValueForField(Base, Field: *FI);
3353 for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3354 if (QualType::DestructionKind DtorKind =
3355 Field->getType().isDestructedType()) {
3356 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3357 CGF.pushDestroy(dtorKind: DtorKind, addr: FieldLValue.getAddress(), type: Field->getType());
3358 }
3359 }
3360 CGF.FinishFunction();
3361 return DestructorFn;
3362}
3363
3364/// Emit a privates mapping function for correct handling of private and
3365/// firstprivate variables.
3366/// \code
3367/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3368/// **noalias priv1,..., <tyn> **noalias privn) {
3369/// *priv1 = &.privates.priv1;
3370/// ...;
3371/// *privn = &.privates.privn;
3372/// }
3373/// \endcode
3374static llvm::Value *
3375emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3376 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3377 ArrayRef<PrivateDataTy> Privates) {
3378 ASTContext &C = CGM.getContext();
3379 FunctionArgList Args;
3380 auto *TaskPrivatesArg = ImplicitParamDecl::Create(
3381 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3382 T: C.getPointerType(T: PrivatesQTy).withConst().withRestrict(),
3383 ParamKind: ImplicitParamKind::Other);
3384 Args.push_back(Elt: TaskPrivatesArg);
3385 llvm::SmallDenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3386 // Track BindingDecl positions separately since BindingDecl is not a VarDecl.
3387 llvm::SmallDenseMap<const BindingDecl *, unsigned> BindingDeclPos;
3388 unsigned Counter = 1;
3389 for (const Expr *E : Data.PrivateVars) {
3390 Args.push_back(Elt: ImplicitParamDecl::Create(
3391 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3392 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3393 .withConst()
3394 .withRestrict(),
3395 ParamKind: ImplicitParamKind::Other));
3396 const ValueDecl *VD = cast<DeclRefExpr>(Val: E)->getDecl();
3397 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
3398 BindingDeclPos[cast<BindingDecl>(Val: BD->getCanonicalDecl())] = Counter;
3399 else
3400 PrivateVarsPos[cast<VarDecl>(Val: VD)] = Counter;
3401 ++Counter;
3402 }
3403 for (const Expr *E : Data.FirstprivateVars) {
3404 Args.push_back(Elt: ImplicitParamDecl::Create(
3405 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3406 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3407 .withConst()
3408 .withRestrict(),
3409 ParamKind: ImplicitParamKind::Other));
3410 const ValueDecl *VD = cast<DeclRefExpr>(Val: E)->getDecl();
3411 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
3412 BindingDeclPos[cast<BindingDecl>(Val: BD->getCanonicalDecl())] = Counter;
3413 else
3414 PrivateVarsPos[cast<VarDecl>(Val: VD)] = Counter;
3415 ++Counter;
3416 }
3417 for (const Expr *E : Data.LastprivateVars) {
3418 Args.push_back(Elt: ImplicitParamDecl::Create(
3419 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3420 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3421 .withConst()
3422 .withRestrict(),
3423 ParamKind: ImplicitParamKind::Other));
3424 const ValueDecl *VD = cast<DeclRefExpr>(Val: E)->getDecl();
3425 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
3426 BindingDeclPos[cast<BindingDecl>(Val: BD->getCanonicalDecl())] = Counter;
3427 else
3428 PrivateVarsPos[cast<VarDecl>(Val: VD)] = Counter;
3429 ++Counter;
3430 }
3431 for (const VarDecl *VD : Data.PrivateLocals) {
3432 QualType Ty = VD->getType().getNonReferenceType();
3433 if (VD->getType()->isLValueReferenceType())
3434 Ty = C.getPointerType(T: Ty);
3435 if (isAllocatableDecl(VD))
3436 Ty = C.getPointerType(T: Ty);
3437 Args.push_back(Elt: ImplicitParamDecl::Create(
3438 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3439 T: C.getPointerType(T: C.getPointerType(T: Ty)).withConst().withRestrict(),
3440 ParamKind: ImplicitParamKind::Other));
3441 PrivateVarsPos[VD] = Counter;
3442 ++Counter;
3443 }
3444 const auto &TaskPrivatesMapFnInfo =
3445 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3446 llvm::FunctionType *TaskPrivatesMapTy =
3447 CGM.getTypes().GetFunctionType(Info: TaskPrivatesMapFnInfo);
3448 std::string Name =
3449 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_privates_map", ""});
3450 auto *TaskPrivatesMap = llvm::Function::Create(
3451 Ty: TaskPrivatesMapTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
3452 M: &CGM.getModule());
3453 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskPrivatesMap,
3454 FI: TaskPrivatesMapFnInfo);
3455 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3456 TaskPrivatesMap->addFnAttr(Kind: "sample-profile-suffix-elision-policy",
3457 Val: "selected");
3458 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
3459 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::NoInline);
3460 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
3461 TaskPrivatesMap->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
3462 }
3463 CodeGenFunction CGF(CGM);
3464 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskPrivatesMap,
3465 FnInfo: TaskPrivatesMapFnInfo, Args, Loc, StartLoc: Loc);
3466
3467 // *privi = &.privates.privi;
3468 LValue Base = CGF.EmitLoadOfPointerLValue(
3469 Ptr: CGF.GetAddrOfLocalVar(VD: TaskPrivatesArg),
3470 PtrTy: TaskPrivatesArg->getType()->castAs<PointerType>());
3471 const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl();
3472 Counter = 0;
3473 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3474 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3475 // Lookup by the original declaration (BindingDecl or VarDecl).
3476 const ValueDecl *LookupVD;
3477 if (Privates[Counter].second.OriginalRef) {
3478 LookupVD =
3479 cast<DeclRefExpr>(Val: Privates[Counter].second.OriginalRef)->getDecl();
3480 } else {
3481 LookupVD = Privates[Counter].second.Original;
3482 }
3483
3484 // For BindingDecls, the privates record now stores each binding's type
3485 // directly (not the full DecompositionDecl), so FieldLVal is already
3486 // correct.
3487
3488 unsigned Position;
3489 if (const auto *BD = dyn_cast<BindingDecl>(Val: LookupVD)) {
3490 Position =
3491 BindingDeclPos.lookup(Val: cast<BindingDecl>(Val: BD->getCanonicalDecl()));
3492 assert(Position && "binding not in privates mapping");
3493 } else {
3494 Position = PrivateVarsPos[cast<VarDecl>(Val: LookupVD)];
3495 }
3496 const VarDecl *VD = Args[Position];
3497 LValue RefLVal =
3498 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD), T: VD->getType());
3499 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3500 Ptr: RefLVal.getAddress(), PtrTy: RefLVal.getType()->castAs<PointerType>());
3501 CGF.EmitStoreOfScalar(value: FieldLVal.getPointer(CGF), lvalue: RefLoadLVal);
3502 ++Counter;
3503 }
3504 CGF.FinishFunction();
3505 return TaskPrivatesMap;
3506}
3507
3508/// Emit initialization for private variables in task-based directives.
3509static void emitPrivatesInit(CodeGenFunction &CGF,
3510 const OMPExecutableDirective &D,
3511 Address KmpTaskSharedsPtr, LValue TDBase,
3512 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3513 QualType SharedsTy, QualType SharedsPtrTy,
3514 const OMPTaskDataTy &Data,
3515 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3516 ASTContext &C = CGF.getContext();
3517 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3518 LValue PrivatesBase = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
3519 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())
3520 ? OMPD_taskloop
3521 : OMPD_task;
3522 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: Kind);
3523 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3524 LValue SrcBase;
3525 bool IsTargetTask =
3526 isOpenMPTargetDataManagementDirective(DKind: D.getDirectiveKind()) ||
3527 isOpenMPTargetExecutionDirective(DKind: D.getDirectiveKind());
3528 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3529 // PointersArray, SizesArray, and MappersArray. The original variables for
3530 // these arrays are not captured and we get their addresses explicitly.
3531 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3532 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3533 SrcBase = CGF.MakeAddrLValue(
3534 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3535 Addr: KmpTaskSharedsPtr, Ty: CGF.ConvertTypeForMem(T: SharedsPtrTy),
3536 ElementTy: CGF.ConvertTypeForMem(T: SharedsTy)),
3537 T: SharedsTy);
3538 }
3539 FI = FI->getType()->castAsRecordDecl()->field_begin();
3540 for (const PrivateDataTy &Pair : Privates) {
3541 // Do not initialize private locals.
3542 if (Pair.second.isLocalPrivate()) {
3543 ++FI;
3544 continue;
3545 }
3546 const VarDecl *VD = Pair.second.PrivateCopy;
3547 const Expr *Init = VD->getAnyInitializer();
3548 if (Init && (!ForDup || (isa<CXXConstructExpr>(Val: Init) &&
3549 !CGF.isTrivialInitializer(Init)))) {
3550 LValue PrivateLValue = CGF.EmitLValueForField(Base: PrivatesBase, Field: *FI);
3551 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3552 const VarDecl *OriginalVD = Pair.second.Original;
3553 // Check if the variable is the target-based BasePointersArray,
3554 // PointersArray, SizesArray, or MappersArray.
3555 LValue SharedRefLValue;
3556 QualType Type = PrivateLValue.getType();
3557 const FieldDecl *SharedField = CapturesInfo.lookup(VD: OriginalVD);
3558 if (IsTargetTask && !SharedField) {
3559 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3560 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
3561 cast<CapturedDecl>(OriginalVD->getDeclContext())
3562 ->getNumParams() == 0 &&
3563 isa<TranslationUnitDecl>(
3564 cast<CapturedDecl>(OriginalVD->getDeclContext())
3565 ->getDeclContext()) &&
3566 "Expected artificial target data variable.");
3567 SharedRefLValue =
3568 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: OriginalVD), T: Type);
3569 } else if (ForDup) {
3570 SharedRefLValue = CGF.EmitLValueForField(Base: SrcBase, Field: SharedField);
3571 // For BindingDecls, access the specific binding field within the
3572 // captured DecompositionDecl.
3573 if (Pair.second.OriginalRef) {
3574 if (const auto *DRE =
3575 dyn_cast<DeclRefExpr>(Val: Pair.second.OriginalRef)) {
3576 if (const auto *BD = dyn_cast<BindingDecl>(Val: DRE->getDecl())) {
3577 // Emit the binding subobject (member or array element) with
3578 // the decomposed decl temporarily mapped to the capture.
3579 const VarDecl *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3580 auto It = CGF.findLocalDecl(D: DD);
3581 bool WasMapped = It != CGF.localDeclMapEnd();
3582 Address Saved = WasMapped ? It->second : Address::invalid();
3583 if (WasMapped)
3584 It->second = SharedRefLValue.getAddress();
3585 else
3586 CGF.insertLocalDecl(D: DD, Addr: SharedRefLValue.getAddress());
3587 SharedRefLValue = CGF.EmitLValue(E: BD->getBinding());
3588 if (WasMapped) {
3589 auto RestoreIt = CGF.findLocalDecl(D: DD);
3590 RestoreIt->second = Saved;
3591 } else {
3592 CGF.eraseLocalDecl(D: DD);
3593 }
3594 }
3595 }
3596 }
3597 bool IsBinding =
3598 Pair.second.OriginalRef &&
3599 isa<BindingDecl>(
3600 Val: cast<DeclRefExpr>(Val: Pair.second.OriginalRef)->getDecl());
3601 SharedRefLValue = CGF.MakeAddrLValue(
3602 Addr: SharedRefLValue.getAddress().withAlignment(
3603 NewAlignment: IsBinding ? C.toCharUnitsFromBits(
3604 BitSize: C.getTypeAlign(T: SharedRefLValue.getType()))
3605 : C.getDeclAlign(D: OriginalVD)),
3606 T: SharedRefLValue.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3607 TBAAInfo: SharedRefLValue.getTBAAInfo());
3608 } else if (CGF.LambdaCaptureFields.count(
3609 Val: Pair.second.Original->getCanonicalDecl()) > 0 ||
3610 isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl)) {
3611 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3612 } else {
3613 // Processing for implicitly captured variables.
3614 InlinedOpenMPRegionRAII Region(
3615 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3616 /*HasCancel=*/false, /*NoInheritance=*/true);
3617 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3618 }
3619 if (Type->isArrayType()) {
3620 // Initialize firstprivate array.
3621 if (!isa<CXXConstructExpr>(Val: Init) || CGF.isTrivialInitializer(Init)) {
3622 // Perform simple memcpy.
3623 CGF.EmitAggregateAssign(Dest: PrivateLValue, Src: SharedRefLValue, EltTy: Type);
3624 } else {
3625 // Initialize firstprivate array using element-by-element
3626 // initialization.
3627 CGF.EmitOMPAggregateAssign(
3628 DestAddr: PrivateLValue.getAddress(), SrcAddr: SharedRefLValue.getAddress(), OriginalType: Type,
3629 CopyGen: [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3630 Address SrcElement) {
3631 // Clean up any temporaries needed by the initialization.
3632 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3633 InitScope.addPrivate(LocalVD: Elem, Addr: SrcElement);
3634 (void)InitScope.Privatize();
3635 // Emit initialization for single element.
3636 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3637 CGF, &CapturesInfo);
3638 CGF.EmitAnyExprToMem(E: Init, Location: DestElement,
3639 Quals: Init->getType().getQualifiers(),
3640 /*IsInitializer=*/false);
3641 });
3642 }
3643 } else {
3644 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3645 InitScope.addPrivate(LocalVD: Elem, Addr: SharedRefLValue.getAddress());
3646 (void)InitScope.Privatize();
3647 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3648 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue,
3649 /*capturedByInit=*/false);
3650 }
3651 } else {
3652 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue, /*capturedByInit=*/false);
3653 }
3654 } else if (const VarDecl *OriginalVD = Pair.second.Original) {
3655 // Handle array bindings without initializers (firstprivate only).
3656 // For private clause (PrivateElemInit is nullptr), skip initialization.
3657 // Check if OriginalRef is a BindingDecl with array type.
3658 const BindingDecl *BD = nullptr;
3659 if (Pair.second.OriginalRef) {
3660 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Pair.second.OriginalRef)) {
3661 BD = dyn_cast<BindingDecl>(Val: DRE->getDecl());
3662 }
3663 }
3664 if (BD && BD->getType()->isArrayType() && Pair.second.PrivateElemInit) {
3665 LValue PrivateLValue = CGF.EmitLValueForField(Base: PrivatesBase, Field: *FI);
3666 QualType Type = PrivateLValue.getType();
3667 const FieldDecl *SharedField = CapturesInfo.lookup(VD: OriginalVD);
3668
3669 // Lambda to temporarily map DecompositionDecl and emit the binding
3670 // expression.
3671 auto EmitBindingWithTempMap = [&CGF](const BindingDecl *BD,
3672 Address DDAddr) -> LValue {
3673 const VarDecl *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3674 auto It = CGF.findLocalDecl(D: DD);
3675 bool WasMapped = It != CGF.localDeclMapEnd();
3676 Address Saved = WasMapped ? It->second : Address::invalid();
3677 if (WasMapped)
3678 It->second = DDAddr;
3679 else
3680 CGF.insertLocalDecl(D: DD, Addr: DDAddr);
3681
3682 LValue Result = CGF.EmitLValue(E: BD->getBinding());
3683
3684 // Restore the mapping
3685 if (WasMapped) {
3686 auto RestoreIt = CGF.findLocalDecl(D: DD);
3687 RestoreIt->second = Saved;
3688 } else {
3689 CGF.eraseLocalDecl(D: DD);
3690 }
3691
3692 return Result;
3693 };
3694
3695 LValue SharedRefLValue;
3696 if (ForDup) {
3697 SharedRefLValue = CGF.EmitLValueForField(Base: SrcBase, Field: SharedField);
3698 SharedRefLValue =
3699 EmitBindingWithTempMap(BD, SharedRefLValue.getAddress());
3700 SharedRefLValue = CGF.MakeAddrLValue(
3701 Addr: SharedRefLValue.getAddress().withAlignment(
3702 NewAlignment: C.getDeclAlign(D: OriginalVD)),
3703 T: SharedRefLValue.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3704 TBAAInfo: SharedRefLValue.getTBAAInfo());
3705 } else {
3706 // For !ForDup (first task), emit binding from parent scope using
3707 // InlinedOpenMPRegionRAII to access the correct scope
3708 InlinedOpenMPRegionRAII Region(
3709 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3710 /*HasCancel=*/false, /*NoInheritance=*/true);
3711
3712 // Get the decomposed decl address from parent scope
3713 const VarDecl *DD = cast<VarDecl>(Val: BD->getDecomposedDecl());
3714 Address DDAddr = CGF.GetAddrOfLocalVar(VD: DD);
3715 SharedRefLValue = EmitBindingWithTempMap(BD, DDAddr);
3716 }
3717 // Perform simple memcpy for array binding
3718 CGF.EmitAggregateAssign(Dest: PrivateLValue, Src: SharedRefLValue, EltTy: Type);
3719 }
3720 }
3721 ++FI;
3722 }
3723}
3724
3725/// Check if duplication function is required for taskloops.
3726static bool checkInitIsRequired(CodeGenFunction &CGF,
3727 ArrayRef<PrivateDataTy> Privates) {
3728 bool InitRequired = false;
3729 for (const PrivateDataTy &Pair : Privates) {
3730 if (Pair.second.isLocalPrivate())
3731 continue;
3732 const VarDecl *VD = Pair.second.PrivateCopy;
3733 const Expr *Init = VD->getAnyInitializer();
3734 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Val: Init) &&
3735 !CGF.isTrivialInitializer(Init));
3736 if (InitRequired)
3737 break;
3738 }
3739 return InitRequired;
3740}
3741
3742
3743/// Emit task_dup function (for initialization of
3744/// private/firstprivate/lastprivate vars and last_iter flag)
3745/// \code
3746/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3747/// lastpriv) {
3748/// // setup lastprivate flag
3749/// task_dst->last = lastpriv;
3750/// // could be constructor calls here...
3751/// }
3752/// \endcode
3753static llvm::Value *
3754emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3755 const OMPExecutableDirective &D,
3756 QualType KmpTaskTWithPrivatesPtrQTy,
3757 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3758 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3759 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3760 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3761 ASTContext &C = CGM.getContext();
3762 auto *DstArg = ImplicitParamDecl::Create(
3763 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3764 ParamKind: ImplicitParamKind::Other);
3765 auto *SrcArg = ImplicitParamDecl::Create(
3766 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3767 ParamKind: ImplicitParamKind::Other);
3768 auto *LastprivArg =
3769 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: C.IntTy,
3770 ParamKind: ImplicitParamKind::Other);
3771 FunctionArgList Args{DstArg, SrcArg, LastprivArg};
3772 const auto &TaskDupFnInfo =
3773 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3774 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(Info: TaskDupFnInfo);
3775 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_dup", ""});
3776 auto *TaskDup = llvm::Function::Create(
3777 Ty: TaskDupTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3778 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskDup, FI: TaskDupFnInfo);
3779 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3780 TaskDup->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3781 TaskDup->setDoesNotRecurse();
3782 CodeGenFunction CGF(CGM);
3783 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskDup, FnInfo: TaskDupFnInfo, Args, Loc,
3784 StartLoc: Loc);
3785
3786 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3787 Ptr: CGF.GetAddrOfLocalVar(VD: DstArg),
3788 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3789 // task_dst->liter = lastpriv;
3790 if (WithLastIter) {
3791 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3792 LValue Base = CGF.EmitLValueForField(
3793 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3794 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3795 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3796 Addr: CGF.GetAddrOfLocalVar(VD: LastprivArg), /*Volatile=*/false, Ty: C.IntTy, Loc);
3797 CGF.EmitStoreOfScalar(value: Lastpriv, lvalue: LILVal);
3798 }
3799
3800 // Emit initial values for private copies (if any).
3801 assert(!Privates.empty());
3802 Address KmpTaskSharedsPtr = Address::invalid();
3803 if (!Data.FirstprivateVars.empty()) {
3804 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3805 Ptr: CGF.GetAddrOfLocalVar(VD: SrcArg),
3806 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3807 LValue Base = CGF.EmitLValueForField(
3808 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3809 KmpTaskSharedsPtr = Address(
3810 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValueForField(
3811 Base, Field: *std::next(x: KmpTaskTQTyRD->field_begin(),
3812 n: KmpTaskTShareds)),
3813 Loc),
3814 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
3815 }
3816 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3817 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3818 CGF.FinishFunction();
3819 return TaskDup;
3820}
3821
3822/// Checks if destructor function is required to be generated.
3823/// \return true if cleanups are required, false otherwise.
3824static bool
3825checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3826 ArrayRef<PrivateDataTy> Privates) {
3827 for (const PrivateDataTy &P : Privates) {
3828 if (P.second.isLocalPrivate())
3829 continue;
3830 QualType Ty = P.second.Original->getType().getNonReferenceType();
3831 if (Ty.isDestructedType())
3832 return true;
3833 }
3834 return false;
3835}
3836
3837namespace {
3838/// Loop generator for OpenMP iterator expression.
3839class OMPIteratorGeneratorScope final
3840 : public CodeGenFunction::OMPPrivateScope {
3841 CodeGenFunction &CGF;
3842 const OMPIteratorExpr *E = nullptr;
3843 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3844 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3845 OMPIteratorGeneratorScope() = delete;
3846 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
3847
3848public:
3849 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3850 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3851 if (!E)
3852 return;
3853 SmallVector<llvm::Value *, 4> Uppers;
3854 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3855 Uppers.push_back(Elt: CGF.EmitScalarExpr(E: E->getHelper(I).Upper));
3856 const auto *VD = cast<VarDecl>(Val: E->getIteratorDecl(I));
3857 addPrivate(LocalVD: VD, Addr: CGF.CreateMemTemp(T: VD->getType(), Name: VD->getName()));
3858 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3859 addPrivate(
3860 LocalVD: HelperData.CounterVD,
3861 Addr: CGF.CreateMemTemp(T: HelperData.CounterVD->getType(), Name: "counter.addr"));
3862 }
3863 Privatize();
3864
3865 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3866 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3867 LValue CLVal =
3868 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: HelperData.CounterVD),
3869 T: HelperData.CounterVD->getType());
3870 // Counter = 0;
3871 CGF.EmitStoreOfScalar(
3872 value: llvm::ConstantInt::get(Ty: CLVal.getAddress().getElementType(), V: 0),
3873 lvalue: CLVal);
3874 CodeGenFunction::JumpDest &ContDest =
3875 ContDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.cont"));
3876 CodeGenFunction::JumpDest &ExitDest =
3877 ExitDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.exit"));
3878 // N = <number-of_iterations>;
3879 llvm::Value *N = Uppers[I];
3880 // cont:
3881 // if (Counter < N) goto body; else goto exit;
3882 CGF.EmitBlock(BB: ContDest.getBlock());
3883 auto *CVal =
3884 CGF.EmitLoadOfScalar(lvalue: CLVal, Loc: HelperData.CounterVD->getLocation());
3885 llvm::Value *Cmp =
3886 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3887 ? CGF.Builder.CreateICmpSLT(LHS: CVal, RHS: N)
3888 : CGF.Builder.CreateICmpULT(LHS: CVal, RHS: N);
3889 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "iter.body");
3890 CGF.Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitDest.getBlock());
3891 // body:
3892 CGF.EmitBlock(BB: BodyBB);
3893 // Iteri = Begini + Counter * Stepi;
3894 CGF.EmitIgnoredExpr(E: HelperData.Update);
3895 }
3896 }
3897 ~OMPIteratorGeneratorScope() {
3898 if (!E)
3899 return;
3900 for (unsigned I = E->numOfIterators(); I > 0; --I) {
3901 // Counter = Counter + 1;
3902 const OMPIteratorHelperData &HelperData = E->getHelper(I: I - 1);
3903 CGF.EmitIgnoredExpr(E: HelperData.CounterUpdate);
3904 // goto cont;
3905 CGF.EmitBranchThroughCleanup(Dest: ContDests[I - 1]);
3906 // exit:
3907 CGF.EmitBlock(BB: ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3908 }
3909 }
3910};
3911} // namespace
3912
3913static std::pair<llvm::Value *, llvm::Value *>
3914getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
3915 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(Val: E);
3916 llvm::Value *Addr;
3917 if (OASE) {
3918 const Expr *Base = OASE->getBase();
3919 Addr = CGF.EmitScalarExpr(E: Base);
3920 } else {
3921 Addr = CGF.EmitLValue(E).getPointer(CGF);
3922 }
3923 llvm::Value *SizeVal;
3924 QualType Ty = E->getType();
3925 if (OASE) {
3926 SizeVal = CGF.getTypeSize(Ty: OASE->getBase()->getType()->getPointeeType());
3927 for (const Expr *SE : OASE->getDimensions()) {
3928 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
3929 Sz = CGF.EmitScalarConversion(
3930 Src: Sz, SrcTy: SE->getType(), DstTy: CGF.getContext().getSizeType(), Loc: SE->getExprLoc());
3931 SizeVal = CGF.Builder.CreateNUWMul(LHS: SizeVal, RHS: Sz);
3932 }
3933 } else if (const auto *ASE =
3934 dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenImpCasts())) {
3935 LValue UpAddrLVal = CGF.EmitArraySectionExpr(E: ASE, /*IsLowerBound=*/false);
3936 Address UpAddrAddress = UpAddrLVal.getAddress();
3937 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3938 Ty: UpAddrAddress.getElementType(), Ptr: UpAddrAddress.emitRawPointer(CGF),
3939 /*Idx0=*/1);
3940 SizeVal = CGF.Builder.CreatePtrDiff(LHS: UpAddr, RHS: Addr, Name: "", /*IsNUW=*/true);
3941 } else {
3942 SizeVal = CGF.getTypeSize(Ty);
3943 }
3944 return std::make_pair(x&: Addr, y&: SizeVal);
3945}
3946
3947/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
3948static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
3949 QualType FlagsTy = C.getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/false);
3950 if (KmpTaskAffinityInfoTy.isNull()) {
3951 RecordDecl *KmpAffinityInfoRD =
3952 C.buildImplicitRecord(Name: "kmp_task_affinity_info_t");
3953 KmpAffinityInfoRD->startDefinition();
3954 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getIntPtrType());
3955 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getSizeType());
3956 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: FlagsTy);
3957 KmpAffinityInfoRD->completeDefinition();
3958 KmpTaskAffinityInfoTy = C.getCanonicalTagType(TD: KmpAffinityInfoRD);
3959 }
3960}
3961
3962CGOpenMPRuntime::TaskResultTy
3963CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3964 const OMPExecutableDirective &D,
3965 llvm::Function *TaskFunction, QualType SharedsTy,
3966 Address Shareds, const OMPTaskDataTy &Data) {
3967 ASTContext &C = CGM.getContext();
3968 llvm::SmallVector<PrivateDataTy, 4> Privates;
3969 // Aggregate privates and sort them by the alignment.
3970 const auto *I = Data.PrivateCopies.begin();
3971 for (const Expr *E : Data.PrivateVars) {
3972 const auto *Decl = cast<DeclRefExpr>(Val: E)->getDecl();
3973 const auto *VD = getOriginalVarDecl(Decl);
3974 const auto *CopyVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
3975 Privates.emplace_back(Args: C.getDeclAlign(D: VD),
3976 Args: PrivateHelpersTy(E, VD, CopyVD,
3977 /*PrivateElemInit=*/nullptr));
3978 ++I;
3979 }
3980 I = Data.FirstprivateCopies.begin();
3981 const auto *IElemInitRef = Data.FirstprivateInits.begin();
3982 for (const Expr *E : Data.FirstprivateVars) {
3983 const auto *Decl = cast<DeclRefExpr>(Val: E)->getDecl();
3984 const auto *VD = getOriginalVarDecl(Decl);
3985 const auto *CopyVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
3986 const auto *InitVD =
3987 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IElemInitRef)->getDecl());
3988 Privates.emplace_back(Args: C.getDeclAlign(D: VD),
3989 Args: PrivateHelpersTy(E, VD, CopyVD, InitVD));
3990 ++I;
3991 ++IElemInitRef;
3992 }
3993 I = Data.LastprivateCopies.begin();
3994 for (const Expr *E : Data.LastprivateVars) {
3995 const auto *Decl = cast<DeclRefExpr>(Val: E)->getDecl();
3996 const auto *VD = getOriginalVarDecl(Decl);
3997 const auto *CopyVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl());
3998 Privates.emplace_back(Args: C.getDeclAlign(D: VD),
3999 Args: PrivateHelpersTy(E, VD, CopyVD,
4000 /*PrivateElemInit=*/nullptr));
4001 ++I;
4002 }
4003 for (const VarDecl *VD : Data.PrivateLocals) {
4004 if (isAllocatableDecl(VD))
4005 Privates.emplace_back(Args: CGM.getPointerAlign(), Args: PrivateHelpersTy(VD));
4006 else
4007 Privates.emplace_back(Args: C.getDeclAlign(D: VD), Args: PrivateHelpersTy(VD));
4008 }
4009 llvm::stable_sort(Range&: Privates,
4010 C: [](const PrivateDataTy &L, const PrivateDataTy &R) {
4011 return L.first > R.first;
4012 });
4013 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4014 // Build type kmp_routine_entry_t (if not built yet).
4015 emitKmpRoutineEntryT(KmpInt32Ty);
4016 // Build type kmp_task_t (if not built yet).
4017 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())) {
4018 if (SavedKmpTaskloopTQTy.isNull()) {
4019 SavedKmpTaskloopTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
4020 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
4021 }
4022 KmpTaskTQTy = SavedKmpTaskloopTQTy;
4023 } else {
4024 assert((D.getDirectiveKind() == OMPD_task ||
4025 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
4026 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
4027 "Expected taskloop, task or target directive");
4028 if (SavedKmpTaskTQTy.isNull()) {
4029 SavedKmpTaskTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
4030 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
4031 }
4032 KmpTaskTQTy = SavedKmpTaskTQTy;
4033 }
4034 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
4035 // Build particular struct kmp_task_t for the given task.
4036 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4037 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4038 CanQualType KmpTaskTWithPrivatesQTy =
4039 C.getCanonicalTagType(TD: KmpTaskTWithPrivatesQTyRD);
4040 QualType KmpTaskTWithPrivatesPtrQTy =
4041 C.getPointerType(T: KmpTaskTWithPrivatesQTy);
4042 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(AddrSpace: 0);
4043 llvm::Value *KmpTaskTWithPrivatesTySize =
4044 CGF.getTypeSize(Ty: KmpTaskTWithPrivatesQTy);
4045 QualType SharedsPtrTy = C.getPointerType(T: SharedsTy);
4046
4047 // Emit initial values for private copies (if any).
4048 llvm::Value *TaskPrivatesMap = nullptr;
4049 llvm::Type *TaskPrivatesMapTy =
4050 std::next(x: TaskFunction->arg_begin(), n: 3)->getType();
4051 if (!Privates.empty()) {
4052 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
4053 TaskPrivatesMap =
4054 emitTaskPrivateMappingFunction(CGM, Loc, Data, PrivatesQTy: FI->getType(), Privates);
4055 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4056 V: TaskPrivatesMap, DestTy: TaskPrivatesMapTy);
4057 } else {
4058 TaskPrivatesMap = llvm::ConstantPointerNull::get(
4059 T: cast<llvm::PointerType>(Val: TaskPrivatesMapTy));
4060 }
4061 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4062 // kmp_task_t *tt);
4063 llvm::Function *TaskEntry = emitProxyTaskFunction(
4064 CGM, Loc, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4065 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4066 TaskPrivatesMap);
4067
4068 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4069 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4070 // kmp_routine_entry_t *task_entry);
4071 // Task flags. Format is taken from
4072 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
4073 // description of kmp_tasking_flags struct.
4074 enum {
4075 TiedFlag = 0x1,
4076 FinalFlag = 0x2,
4077 DestructorsFlag = 0x8,
4078 PriorityFlag = 0x20,
4079 DetachableFlag = 0x40,
4080 FreeAgentFlag = 0x80,
4081 TransparentFlag = 0x100,
4082 };
4083 unsigned Flags = Data.Tied ? TiedFlag : 0;
4084 bool NeedsCleanup = false;
4085 if (!Privates.empty()) {
4086 NeedsCleanup =
4087 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
4088 if (NeedsCleanup)
4089 Flags = Flags | DestructorsFlag;
4090 }
4091 if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {
4092 OpenMPThreadsetKind Kind = Clause->getThreadsetKind();
4093 if (Kind == OMPC_THREADSET_omp_pool)
4094 Flags = Flags | FreeAgentFlag;
4095 }
4096 if (D.getSingleClause<OMPTransparentClause>())
4097 Flags |= TransparentFlag;
4098
4099 if (Data.Priority.getInt())
4100 Flags = Flags | PriorityFlag;
4101 if (D.hasClausesOfKind<OMPDetachClause>())
4102 Flags = Flags | DetachableFlag;
4103 llvm::Value *TaskFlags =
4104 Data.Final.getPointer()
4105 ? CGF.Builder.CreateSelect(C: Data.Final.getPointer(),
4106 True: CGF.Builder.getInt32(C: FinalFlag),
4107 False: CGF.Builder.getInt32(/*C=*/0))
4108 : CGF.Builder.getInt32(C: Data.Final.getInt() ? FinalFlag : 0);
4109 TaskFlags = CGF.Builder.CreateOr(LHS: TaskFlags, RHS: CGF.Builder.getInt32(C: Flags));
4110 llvm::Value *SharedsSize = CGM.getSize(numChars: C.getTypeSizeInChars(T: SharedsTy));
4111 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
4112 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
4113 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4114 V: TaskEntry, DestTy: KmpRoutineEntryPtrTy)};
4115 llvm::Value *NewTask;
4116 if (D.hasClausesOfKind<OMPNowaitClause>()) {
4117 // Check if we have any device clause associated with the directive.
4118 const Expr *Device = nullptr;
4119 if (auto *C = D.getSingleClause<OMPDeviceClause>())
4120 Device = C->getDevice();
4121 // Emit device ID if any otherwise use default value.
4122 llvm::Value *DeviceID;
4123 if (Device)
4124 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
4125 DestTy: CGF.Int64Ty, /*isSigned=*/true);
4126 else
4127 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
4128 AllocArgs.push_back(Elt: DeviceID);
4129 NewTask = CGF.EmitRuntimeCall(
4130 callee: OMPBuilder.getOrCreateRuntimeFunction(
4131 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_target_task_alloc),
4132 args: AllocArgs);
4133 } else {
4134 NewTask =
4135 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4136 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_alloc),
4137 args: AllocArgs);
4138 }
4139 // Emit detach clause initialization.
4140 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
4141 // task_descriptor);
4142 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
4143 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
4144 LValue EvtLVal = CGF.EmitLValue(E: Evt);
4145
4146 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
4147 // int gtid, kmp_task_t *task);
4148 llvm::Value *Loc = emitUpdateLocation(CGF, Loc: DC->getBeginLoc());
4149 llvm::Value *Tid = getThreadID(CGF, Loc: DC->getBeginLoc());
4150 Tid = CGF.Builder.CreateIntCast(V: Tid, DestTy: CGF.IntTy, /*isSigned=*/false);
4151 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
4152 callee: OMPBuilder.getOrCreateRuntimeFunction(
4153 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_allow_completion_event),
4154 args: {Loc, Tid, NewTask});
4155 EvtVal = CGF.EmitScalarConversion(Src: EvtVal, SrcTy: C.VoidPtrTy, DstTy: Evt->getType(),
4156 Loc: Evt->getExprLoc());
4157 CGF.EmitStoreOfScalar(value: EvtVal, lvalue: EvtLVal);
4158 }
4159 // Process affinity clauses.
4160 if (D.hasClausesOfKind<OMPAffinityClause>()) {
4161 // Process list of affinity data.
4162 ASTContext &C = CGM.getContext();
4163 Address AffinitiesArray = Address::invalid();
4164 // Calculate number of elements to form the array of affinity data.
4165 llvm::Value *NumOfElements = nullptr;
4166 unsigned NumAffinities = 0;
4167 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4168 if (const Expr *Modifier = C->getModifier()) {
4169 const auto *IE = cast<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts());
4170 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4171 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4172 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
4173 NumOfElements =
4174 NumOfElements ? CGF.Builder.CreateNUWMul(LHS: NumOfElements, RHS: Sz) : Sz;
4175 }
4176 } else {
4177 NumAffinities += C->varlist_size();
4178 }
4179 }
4180 getKmpAffinityType(C&: CGM.getContext(), KmpTaskAffinityInfoTy);
4181 // Fields ids in kmp_task_affinity_info record.
4182 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
4183
4184 QualType KmpTaskAffinityInfoArrayTy;
4185 if (NumOfElements) {
4186 NumOfElements = CGF.Builder.CreateNUWAdd(
4187 LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumAffinities), RHS: NumOfElements);
4188 auto *OVE = new (C) OpaqueValueExpr(
4189 Loc,
4190 C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.getSizeType()), /*Signed=*/0),
4191 VK_PRValue);
4192 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4193 RValue::get(V: NumOfElements));
4194 KmpTaskAffinityInfoArrayTy = C.getVariableArrayType(
4195 EltTy: KmpTaskAffinityInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4196 /*IndexTypeQuals=*/0);
4197 // Properly emit variable-sized array.
4198 auto *PD = ImplicitParamDecl::Create(C, T: KmpTaskAffinityInfoArrayTy,
4199 ParamKind: ImplicitParamKind::Other);
4200 CGF.EmitVarDecl(D: *PD);
4201 AffinitiesArray = CGF.GetAddrOfLocalVar(VD: PD);
4202 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4203 /*isSigned=*/false);
4204 } else {
4205 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4206 EltTy: KmpTaskAffinityInfoTy,
4207 ArySize: llvm::APInt(C.getTypeSize(T: C.getSizeType()), NumAffinities), SizeExpr: nullptr,
4208 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4209 AffinitiesArray = CGF.CreateMemTempWithoutCast(T: KmpTaskAffinityInfoArrayTy,
4210 Name: ".affs.arr.addr");
4211 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(Addr: AffinitiesArray, Index: 0);
4212 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumAffinities,
4213 /*isSigned=*/IsSigned: false);
4214 }
4215
4216 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4217 // Fill array by elements without iterators.
4218 unsigned Pos = 0;
4219 bool HasIterator = false;
4220 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4221 if (C->getModifier()) {
4222 HasIterator = true;
4223 continue;
4224 }
4225 for (const Expr *E : C->varlist()) {
4226 llvm::Value *Addr;
4227 llvm::Value *Size;
4228 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4229 LValue Base =
4230 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateConstGEP(Addr: AffinitiesArray, Index: Pos),
4231 T: KmpTaskAffinityInfoTy);
4232 // affs[i].base_addr = &<Affinities[i].second>;
4233 LValue BaseAddrLVal = CGF.EmitLValueForField(
4234 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4235 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4236 lvalue: BaseAddrLVal);
4237 // affs[i].len = sizeof(<Affinities[i].second>);
4238 LValue LenLVal = CGF.EmitLValueForField(
4239 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4240 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4241 ++Pos;
4242 }
4243 }
4244 LValue PosLVal;
4245 if (HasIterator) {
4246 PosLVal = CGF.MakeAddrLValue(
4247 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "affs.counter.addr"),
4248 T: C.getSizeType());
4249 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4250 }
4251 // Process elements with iterators.
4252 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4253 const Expr *Modifier = C->getModifier();
4254 if (!Modifier)
4255 continue;
4256 OMPIteratorGeneratorScope IteratorScope(
4257 CGF, cast_or_null<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts()));
4258 for (const Expr *E : C->varlist()) {
4259 llvm::Value *Addr;
4260 llvm::Value *Size;
4261 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4262 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4263 LValue Base =
4264 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateGEP(CGF, Addr: AffinitiesArray, Index: Idx),
4265 T: KmpTaskAffinityInfoTy);
4266 // affs[i].base_addr = &<Affinities[i].second>;
4267 LValue BaseAddrLVal = CGF.EmitLValueForField(
4268 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4269 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4270 lvalue: BaseAddrLVal);
4271 // affs[i].len = sizeof(<Affinities[i].second>);
4272 LValue LenLVal = CGF.EmitLValueForField(
4273 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4274 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4275 Idx = CGF.Builder.CreateNUWAdd(
4276 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4277 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4278 }
4279 }
4280 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4281 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4282 // naffins, kmp_task_affinity_info_t *affin_list);
4283 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4284 llvm::Value *GTid = getThreadID(CGF, Loc);
4285 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4286 V: AffinitiesArray.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy);
4287 // FIXME: Emit the function and ignore its result for now unless the
4288 // runtime function is properly implemented.
4289 (void)CGF.EmitRuntimeCall(
4290 callee: OMPBuilder.getOrCreateRuntimeFunction(
4291 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_reg_task_with_affinity),
4292 args: {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4293 }
4294 llvm::Value *NewTaskNewTaskTTy =
4295 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4296 V: NewTask, DestTy: KmpTaskTWithPrivatesPtrTy);
4297 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(V: NewTaskNewTaskTTy,
4298 T: KmpTaskTWithPrivatesQTy);
4299 LValue TDBase =
4300 CGF.EmitLValueForField(Base, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
4301 // Fill the data in the resulting kmp_task_t record.
4302 // Copy shareds if there are any.
4303 Address KmpTaskSharedsPtr = Address::invalid();
4304 if (!SharedsTy->castAsRecordDecl()->field_empty()) {
4305 KmpTaskSharedsPtr = Address(
4306 CGF.EmitLoadOfScalar(
4307 lvalue: CGF.EmitLValueForField(
4308 Base: TDBase,
4309 Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds)),
4310 Loc),
4311 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
4312 LValue Dest = CGF.MakeAddrLValue(Addr: KmpTaskSharedsPtr, T: SharedsTy);
4313 LValue Src = CGF.MakeAddrLValue(Addr: Shareds, T: SharedsTy);
4314 CGF.EmitAggregateCopy(Dest, Src, EltTy: SharedsTy, MayOverlap: AggValueSlot::DoesNotOverlap);
4315 }
4316 // Emit initial values for private copies (if any).
4317 TaskResultTy Result;
4318 if (!Privates.empty()) {
4319 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase: Base, KmpTaskTWithPrivatesQTyRD,
4320 SharedsTy, SharedsPtrTy, Data, Privates,
4321 /*ForDup=*/false);
4322 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) &&
4323 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4324 Result.TaskDupFn = emitTaskDupFunction(
4325 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4326 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4327 /*WithLastIter=*/!Data.LastprivateVars.empty());
4328 }
4329 }
4330 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4331 enum { Priority = 0, Destructors = 1 };
4332 // Provide pointer to function with destructors for privates.
4333 auto FI = std::next(x: KmpTaskTQTyRD->field_begin(), n: Data1);
4334 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
4335 assert(KmpCmplrdataUD->isUnion());
4336 if (NeedsCleanup) {
4337 llvm::Value *DestructorFn = emitDestructorsFunction(
4338 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4339 KmpTaskTWithPrivatesQTy);
4340 LValue Data1LV = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
4341 LValue DestructorsLV = CGF.EmitLValueForField(
4342 Base: Data1LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Destructors));
4343 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4344 V: DestructorFn, DestTy: KmpRoutineEntryPtrTy),
4345 lvalue: DestructorsLV);
4346 }
4347 // Set priority.
4348 if (Data.Priority.getInt()) {
4349 LValue Data2LV = CGF.EmitLValueForField(
4350 Base: TDBase, Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: Data2));
4351 LValue PriorityLV = CGF.EmitLValueForField(
4352 Base: Data2LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Priority));
4353 CGF.EmitStoreOfScalar(value: Data.Priority.getPointer(), lvalue: PriorityLV);
4354 }
4355 Result.NewTask = NewTask;
4356 Result.TaskEntry = TaskEntry;
4357 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4358 Result.TDBase = TDBase;
4359 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4360 return Result;
4361}
4362
4363/// Translates internal dependency kind into the runtime kind.
4364static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4365 RTLDependenceKindTy DepKind;
4366 switch (K) {
4367 case OMPC_DEPEND_in:
4368 DepKind = RTLDependenceKindTy::DepIn;
4369 break;
4370 // Out and InOut dependencies must use the same code.
4371 case OMPC_DEPEND_out:
4372 case OMPC_DEPEND_inout:
4373 DepKind = RTLDependenceKindTy::DepInOut;
4374 break;
4375 case OMPC_DEPEND_mutexinoutset:
4376 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4377 break;
4378 case OMPC_DEPEND_inoutset:
4379 DepKind = RTLDependenceKindTy::DepInOutSet;
4380 break;
4381 case OMPC_DEPEND_outallmemory:
4382 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4383 break;
4384 case OMPC_DEPEND_source:
4385 case OMPC_DEPEND_sink:
4386 case OMPC_DEPEND_depobj:
4387 case OMPC_DEPEND_inoutallmemory:
4388 case OMPC_DEPEND_unknown:
4389 llvm_unreachable("Unknown task dependence type");
4390 }
4391 return DepKind;
4392}
4393
4394/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4395static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4396 QualType &FlagsTy) {
4397 FlagsTy = C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.BoolTy), /*Signed=*/false);
4398 if (KmpDependInfoTy.isNull()) {
4399 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord(Name: "kmp_depend_info");
4400 KmpDependInfoRD->startDefinition();
4401 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getIntPtrType());
4402 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getSizeType());
4403 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: FlagsTy);
4404 KmpDependInfoRD->completeDefinition();
4405 KmpDependInfoTy = C.getCanonicalTagType(TD: KmpDependInfoRD);
4406 }
4407}
4408
4409std::pair<llvm::Value *, LValue>
4410CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4411 SourceLocation Loc) {
4412 ASTContext &C = CGM.getContext();
4413 QualType FlagsTy;
4414 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4415 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4416 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4417 LValue Base = CGF.EmitLoadOfPointerLValue(
4418 Ptr: DepobjLVal.getAddress().withElementType(
4419 ElemTy: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy)),
4420 PtrTy: KmpDependInfoPtrTy->castAs<PointerType>());
4421 Address DepObjAddr = CGF.Builder.CreateGEP(
4422 CGF, Addr: Base.getAddress(),
4423 Index: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4424 LValue NumDepsBase = CGF.MakeAddrLValue(
4425 Addr: DepObjAddr, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(), TBAAInfo: Base.getTBAAInfo());
4426 // NumDeps = deps[i].base_addr;
4427 LValue BaseAddrLVal = CGF.EmitLValueForField(
4428 Base: NumDepsBase,
4429 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4430 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4431 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(lvalue: BaseAddrLVal, Loc);
4432 return std::make_pair(x&: NumDeps, y&: Base);
4433}
4434
4435static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4436 llvm::PointerUnion<unsigned *, LValue *> Pos,
4437 const OMPTaskDataTy::DependData &Data,
4438 Address DependenciesArray) {
4439 CodeGenModule &CGM = CGF.CGM;
4440 ASTContext &C = CGM.getContext();
4441 QualType FlagsTy;
4442 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4443 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4444 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4445
4446 OMPIteratorGeneratorScope IteratorScope(
4447 CGF, cast_or_null<OMPIteratorExpr>(
4448 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4449 : nullptr));
4450 for (const Expr *E : Data.DepExprs) {
4451 llvm::Value *Addr;
4452 llvm::Value *Size;
4453
4454 // The expression will be a nullptr in the 'omp_all_memory' case.
4455 if (E) {
4456 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4457 Addr = CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy);
4458 } else {
4459 Addr = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4460 Size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
4461 }
4462 LValue Base;
4463 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4464 Base = CGF.MakeAddrLValue(
4465 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: *P), T: KmpDependInfoTy);
4466 } else {
4467 assert(E && "Expected a non-null expression");
4468 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4469 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4470 Base = CGF.MakeAddrLValue(
4471 Addr: CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Idx), T: KmpDependInfoTy);
4472 }
4473 // deps[i].base_addr = &<Dependencies[i].second>;
4474 LValue BaseAddrLVal = CGF.EmitLValueForField(
4475 Base,
4476 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4477 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4478 CGF.EmitStoreOfScalar(value: Addr, lvalue: BaseAddrLVal);
4479 // deps[i].len = sizeof(<Dependencies[i].second>);
4480 LValue LenLVal = CGF.EmitLValueForField(
4481 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4482 n: static_cast<unsigned int>(RTLDependInfoFields::Len)));
4483 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4484 // deps[i].flags = <Dependencies[i].first>;
4485 RTLDependenceKindTy DepKind = translateDependencyKind(K: Data.DepKind);
4486 LValue FlagsLVal = CGF.EmitLValueForField(
4487 Base,
4488 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4489 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4490 CGF.EmitStoreOfScalar(
4491 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4492 lvalue: FlagsLVal);
4493 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4494 ++(*P);
4495 } else {
4496 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4497 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4498 Idx = CGF.Builder.CreateNUWAdd(LHS: Idx,
4499 RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4500 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4501 }
4502 }
4503}
4504
4505SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes(
4506 CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4507 const OMPTaskDataTy::DependData &Data) {
4508 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4509 "Expected depobj dependency kind.");
4510 SmallVector<llvm::Value *, 4> Sizes;
4511 SmallVector<LValue, 4> SizeLVals;
4512 ASTContext &C = CGF.getContext();
4513 {
4514 OMPIteratorGeneratorScope IteratorScope(
4515 CGF, cast_or_null<OMPIteratorExpr>(
4516 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4517 : nullptr));
4518 for (const Expr *E : Data.DepExprs) {
4519 llvm::Value *NumDeps;
4520 LValue Base;
4521 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4522 std::tie(args&: NumDeps, args&: Base) =
4523 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4524 LValue NumLVal = CGF.MakeAddrLValue(
4525 Addr: CGF.CreateMemTempWithoutCast(T: C.getUIntPtrType(), Name: "depobj.size.addr"),
4526 T: C.getUIntPtrType());
4527 CGF.Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0),
4528 Addr: NumLVal.getAddress());
4529 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(lvalue: NumLVal, Loc: E->getExprLoc());
4530 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: PrevVal, RHS: NumDeps);
4531 CGF.EmitStoreOfScalar(value: Add, lvalue: NumLVal);
4532 SizeLVals.push_back(Elt: NumLVal);
4533 }
4534 }
4535 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4536 llvm::Value *Size =
4537 CGF.EmitLoadOfScalar(lvalue: SizeLVals[I], Loc: Data.DepExprs[I]->getExprLoc());
4538 Sizes.push_back(Elt: Size);
4539 }
4540 return Sizes;
4541}
4542
4543void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
4544 QualType &KmpDependInfoTy,
4545 LValue PosLVal,
4546 const OMPTaskDataTy::DependData &Data,
4547 Address DependenciesArray) {
4548 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4549 "Expected depobj dependency kind.");
4550 llvm::Value *ElSize = CGF.getTypeSize(Ty: KmpDependInfoTy);
4551 {
4552 OMPIteratorGeneratorScope IteratorScope(
4553 CGF, cast_or_null<OMPIteratorExpr>(
4554 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4555 : nullptr));
4556 for (const Expr *E : Data.DepExprs) {
4557 llvm::Value *NumDeps;
4558 LValue Base;
4559 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4560 std::tie(args&: NumDeps, args&: Base) =
4561 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4562
4563 // memcopy dependency data.
4564 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4565 LHS: ElSize,
4566 RHS: CGF.Builder.CreateIntCast(V: NumDeps, DestTy: CGF.SizeTy, /*isSigned=*/false));
4567 llvm::Value *Pos = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4568 Address DepAddr = CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Pos);
4569 CGF.Builder.CreateMemCpy(Dest: DepAddr, Src: Base.getAddress(), Size);
4570
4571 // Increase pos.
4572 // pos += size;
4573 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: Pos, RHS: NumDeps);
4574 CGF.EmitStoreOfScalar(value: Add, lvalue: PosLVal);
4575 }
4576 }
4577}
4578
4579std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4580 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4581 SourceLocation Loc) {
4582 if (llvm::all_of(Range&: Dependencies, P: [](const OMPTaskDataTy::DependData &D) {
4583 return D.DepExprs.empty();
4584 }))
4585 return std::make_pair(x: nullptr, y: Address::invalid());
4586 // Process list of dependencies.
4587 ASTContext &C = CGM.getContext();
4588 Address DependenciesArray = Address::invalid();
4589 llvm::Value *NumOfElements = nullptr;
4590 unsigned NumDependencies = std::accumulate(
4591 first: Dependencies.begin(), last: Dependencies.end(), init: 0,
4592 binary_op: [](unsigned V, const OMPTaskDataTy::DependData &D) {
4593 return D.DepKind == OMPC_DEPEND_depobj
4594 ? V
4595 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4596 });
4597 QualType FlagsTy;
4598 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4599 bool HasDepobjDeps = false;
4600 bool HasRegularWithIterators = false;
4601 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4602 llvm::Value *NumOfRegularWithIterators =
4603 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4604 // Calculate number of depobj dependencies and regular deps with the
4605 // iterators.
4606 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4607 if (D.DepKind == OMPC_DEPEND_depobj) {
4608 SmallVector<llvm::Value *, 4> Sizes =
4609 emitDepobjElementsSizes(CGF, KmpDependInfoTy, Data: D);
4610 for (llvm::Value *Size : Sizes) {
4611 NumOfDepobjElements =
4612 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: Size);
4613 }
4614 HasDepobjDeps = true;
4615 continue;
4616 }
4617 // Include number of iterations, if any.
4618
4619 if (const auto *IE = cast_or_null<OMPIteratorExpr>(Val: D.IteratorExpr)) {
4620 llvm::Value *ClauseIteratorSpace =
4621 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 1);
4622 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4623 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4624 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4625 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(LHS: Sz, RHS: ClauseIteratorSpace);
4626 }
4627 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4628 LHS: ClauseIteratorSpace,
4629 RHS: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: D.DepExprs.size()));
4630 NumOfRegularWithIterators =
4631 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumClauseDeps);
4632 HasRegularWithIterators = true;
4633 continue;
4634 }
4635 }
4636
4637 QualType KmpDependInfoArrayTy;
4638 if (HasDepobjDeps || HasRegularWithIterators) {
4639 NumOfElements = llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: NumDependencies,
4640 /*isSigned=*/IsSigned: false);
4641 if (HasDepobjDeps) {
4642 NumOfElements =
4643 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: NumOfElements);
4644 }
4645 if (HasRegularWithIterators) {
4646 NumOfElements =
4647 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumOfElements);
4648 }
4649 auto *OVE = new (C) OpaqueValueExpr(
4650 Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4651 VK_PRValue);
4652 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4653 RValue::get(V: NumOfElements));
4654 KmpDependInfoArrayTy =
4655 C.getVariableArrayType(EltTy: KmpDependInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4656 /*IndexTypeQuals=*/0);
4657 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4658 // Properly emit variable-sized array.
4659 auto *PD = ImplicitParamDecl::Create(C, T: KmpDependInfoArrayTy,
4660 ParamKind: ImplicitParamKind::Other);
4661 CGF.EmitVarDecl(D: *PD);
4662 DependenciesArray = CGF.GetAddrOfLocalVar(VD: PD);
4663 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4664 /*isSigned=*/false);
4665 } else {
4666 KmpDependInfoArrayTy = C.getConstantArrayType(
4667 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies), SizeExpr: nullptr,
4668 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4669 DependenciesArray =
4670 CGF.CreateMemTempWithoutCast(T: KmpDependInfoArrayTy, Name: ".dep.arr.addr");
4671 DependenciesArray = CGF.Builder.CreateConstArrayGEP(Addr: DependenciesArray, Index: 0);
4672 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumDependencies,
4673 /*isSigned=*/IsSigned: false);
4674 }
4675 unsigned Pos = 0;
4676 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4677 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4678 continue;
4679 emitDependData(CGF, KmpDependInfoTy, Pos: &Pos, Data: Dep, DependenciesArray);
4680 }
4681 // Copy regular dependencies with iterators.
4682 LValue PosLVal = CGF.MakeAddrLValue(
4683 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "dep.counter.addr"),
4684 T: C.getSizeType());
4685 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4686 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4687 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4688 continue;
4689 emitDependData(CGF, KmpDependInfoTy, Pos: &PosLVal, Data: Dep, DependenciesArray);
4690 }
4691 // Copy final depobj arrays without iterators.
4692 if (HasDepobjDeps) {
4693 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4694 if (Dep.DepKind != OMPC_DEPEND_depobj)
4695 continue;
4696 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Data: Dep, DependenciesArray);
4697 }
4698 }
4699 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4700 Addr: DependenciesArray, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
4701 return std::make_pair(x&: NumOfElements, y&: DependenciesArray);
4702}
4703
4704Address CGOpenMPRuntime::emitDepobjDependClause(
4705 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4706 SourceLocation Loc) {
4707 if (Dependencies.DepExprs.empty())
4708 return Address::invalid();
4709 // Process list of dependencies.
4710 ASTContext &C = CGM.getContext();
4711 Address DependenciesArray = Address::invalid();
4712 unsigned NumDependencies = Dependencies.DepExprs.size();
4713 QualType FlagsTy;
4714 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4715 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4716
4717 llvm::Value *Size;
4718 // Define type kmp_depend_info[<Dependencies.size()>];
4719 // For depobj reserve one extra element to store the number of elements.
4720 // It is required to handle depobj(x) update(in) construct.
4721 // kmp_depend_info[<Dependencies.size()>] deps;
4722 llvm::Value *NumDepsVal;
4723 CharUnits Align = C.getTypeAlignInChars(T: KmpDependInfoTy);
4724 if (const auto *IE =
4725 cast_or_null<OMPIteratorExpr>(Val: Dependencies.IteratorExpr)) {
4726 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
4727 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4728 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4729 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
4730 NumDepsVal = CGF.Builder.CreateNUWMul(LHS: NumDepsVal, RHS: Sz);
4731 }
4732 Size = CGF.Builder.CreateNUWAdd(LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1),
4733 RHS: NumDepsVal);
4734 CharUnits SizeInBytes =
4735 C.getTypeSizeInChars(T: KmpDependInfoTy).alignTo(Align);
4736 llvm::Value *RecSize = CGM.getSize(numChars: SizeInBytes);
4737 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: RecSize);
4738 NumDepsVal =
4739 CGF.Builder.CreateIntCast(V: NumDepsVal, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4740 } else {
4741 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4742 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4743 SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4744 CharUnits Sz = C.getTypeSizeInChars(T: KmpDependInfoArrayTy);
4745 Size = CGM.getSize(numChars: Sz.alignTo(Align));
4746 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: NumDependencies);
4747 }
4748 // Need to allocate on the dynamic memory.
4749 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4750 // Use default allocator.
4751 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4752 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4753
4754 llvm::Value *Addr =
4755 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4756 M&: CGM.getModule(), FnID: OMPRTL___kmpc_alloc),
4757 args: Args, name: ".dep.arr.addr");
4758 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(T: KmpDependInfoTy);
4759 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4760 V: Addr, DestTy: CGF.Builder.getPtrTy(AddrSpace: 0));
4761 DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
4762 // Write number of elements in the first element of array for depobj.
4763 LValue Base = CGF.MakeAddrLValue(Addr: DependenciesArray, T: KmpDependInfoTy);
4764 // deps[i].base_addr = NumDependencies;
4765 LValue BaseAddrLVal = CGF.EmitLValueForField(
4766 Base,
4767 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4768 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4769 CGF.EmitStoreOfScalar(value: NumDepsVal, lvalue: BaseAddrLVal);
4770 llvm::PointerUnion<unsigned *, LValue *> Pos;
4771 unsigned Idx = 1;
4772 LValue PosLVal;
4773 if (Dependencies.IteratorExpr) {
4774 PosLVal = CGF.MakeAddrLValue(
4775 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "iterator.counter.addr"),
4776 T: C.getSizeType());
4777 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Idx), lvalue: PosLVal,
4778 /*IsInit=*/isInit: true);
4779 Pos = &PosLVal;
4780 } else {
4781 Pos = &Idx;
4782 }
4783 emitDependData(CGF, KmpDependInfoTy, Pos, Data: Dependencies, DependenciesArray);
4784 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4785 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: 1), Ty: CGF.VoidPtrTy,
4786 ElementTy: CGF.Int8Ty);
4787 return DependenciesArray;
4788}
4789
4790void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4791 SourceLocation Loc) {
4792 ASTContext &C = CGM.getContext();
4793 QualType FlagsTy;
4794 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4795 LValue Base = CGF.EmitLoadOfPointerLValue(Ptr: DepobjLVal.getAddress(),
4796 PtrTy: C.VoidPtrTy.castAs<PointerType>());
4797 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4798 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4799 Addr: Base.getAddress(), Ty: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy),
4800 ElementTy: CGF.ConvertTypeForMem(T: KmpDependInfoTy));
4801 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4802 Ty: Addr.getElementType(), Ptr: Addr.emitRawPointer(CGF),
4803 IdxList: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4804 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: DepObjAddr,
4805 DestTy: CGF.VoidPtrTy);
4806 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4807 // Use default allocator.
4808 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4809 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4810
4811 // _kmpc_free(gtid, addr, nullptr);
4812 (void)CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4813 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free),
4814 args: Args);
4815}
4816
4817void CGOpenMPRuntime::emitUpdateDependObjectsClause(
4818 CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind,
4819 SourceLocation Loc) {
4820 ASTContext &C = CGM.getContext();
4821 QualType FlagsTy;
4822 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4823 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4824 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4825 llvm::Value *NumDeps;
4826 LValue Base;
4827 std::tie(args&: NumDeps, args&: Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4828
4829 Address Begin = Base.getAddress();
4830 // Cast from pointer to array type to pointer to single element.
4831 llvm::Value *End = CGF.Builder.CreateGEP(Ty: Begin.getElementType(),
4832 Ptr: Begin.emitRawPointer(CGF), IdxList: NumDeps);
4833 // The basic structure here is a while-do loop.
4834 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.body");
4835 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.done");
4836 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4837 CGF.EmitBlock(BB: BodyBB);
4838 llvm::PHINode *ElementPHI =
4839 CGF.Builder.CreatePHI(Ty: Begin.getType(), NumReservedValues: 2, Name: "omp.elementPast");
4840 ElementPHI->addIncoming(V: Begin.emitRawPointer(CGF), BB: EntryBB);
4841 Begin = Begin.withPointer(NewPointer: ElementPHI, IsKnownNonNull: KnownNonNull);
4842 Base = CGF.MakeAddrLValue(Addr: Begin, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(),
4843 TBAAInfo: Base.getTBAAInfo());
4844 // deps[i].flags = NewDepKind;
4845 RTLDependenceKindTy DepKind = translateDependencyKind(K: NewDepKind);
4846 LValue FlagsLVal = CGF.EmitLValueForField(
4847 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4848 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4849 CGF.EmitStoreOfScalar(
4850 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4851 lvalue: FlagsLVal);
4852
4853 // Shift the address forward by one element.
4854 llvm::Value *ElementNext =
4855 CGF.Builder.CreateConstGEP(Addr: Begin, /*Index=*/1, Name: "omp.elementNext")
4856 .emitRawPointer(CGF);
4857 ElementPHI->addIncoming(V: ElementNext, BB: CGF.Builder.GetInsertBlock());
4858 llvm::Value *IsEmpty =
4859 CGF.Builder.CreateICmpEQ(LHS: ElementNext, RHS: End, Name: "omp.isempty");
4860 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4861 // Done.
4862 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4863}
4864
4865void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4866 const OMPExecutableDirective &D,
4867 llvm::Function *TaskFunction,
4868 QualType SharedsTy, Address Shareds,
4869 const Expr *IfCond,
4870 const OMPTaskDataTy &Data) {
4871 if (!CGF.HaveInsertPoint())
4872 return;
4873
4874 TaskResultTy Result =
4875 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4876 llvm::Value *NewTask = Result.NewTask;
4877 llvm::Function *TaskEntry = Result.TaskEntry;
4878 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4879 LValue TDBase = Result.TDBase;
4880 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
4881 // Process list of dependences.
4882 Address DependenciesArray = Address::invalid();
4883 llvm::Value *NumOfElements;
4884 std::tie(args&: NumOfElements, args&: DependenciesArray) =
4885 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
4886
4887 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4888 // libcall.
4889 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
4890 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
4891 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
4892 // list is not empty
4893 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4894 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4895 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4896 llvm::Value *DepTaskArgs[7];
4897 if (!Data.Dependences.empty()) {
4898 DepTaskArgs[0] = UpLoc;
4899 DepTaskArgs[1] = ThreadID;
4900 DepTaskArgs[2] = NewTask;
4901 DepTaskArgs[3] = NumOfElements;
4902 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4903 DepTaskArgs[5] = CGF.Builder.getInt32(C: 0);
4904 DepTaskArgs[6] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4905 }
4906 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4907 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4908 if (!Data.Tied) {
4909 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
4910 LValue PartIdLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PartIdFI);
4911 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: 0), lvalue: PartIdLVal);
4912 }
4913 if (!Data.Dependences.empty()) {
4914 CGF.EmitRuntimeCall(
4915 callee: OMPBuilder.getOrCreateRuntimeFunction(
4916 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_with_deps),
4917 args: DepTaskArgs);
4918 } else {
4919 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4920 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
4921 args: TaskArgs);
4922 }
4923 // Check if parent region is untied and build return for untied task;
4924 if (auto *Region =
4925 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
4926 Region->emitUntiedSwitch(CGF);
4927 };
4928
4929 llvm::Value *DepWaitTaskArgs[7];
4930 if (!Data.Dependences.empty()) {
4931 DepWaitTaskArgs[0] = UpLoc;
4932 DepWaitTaskArgs[1] = ThreadID;
4933 DepWaitTaskArgs[2] = NumOfElements;
4934 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4935 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
4936 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4937 DepWaitTaskArgs[6] =
4938 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
4939 }
4940 auto &M = CGM.getModule();
4941 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4942 TaskEntry, &Data, &DepWaitTaskArgs,
4943 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4944 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4945 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
4946 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
4947 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
4948 // is specified.
4949 if (!Data.Dependences.empty())
4950 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4951 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
4952 args: DepWaitTaskArgs);
4953 // Call proxy_task_entry(gtid, new_task);
4954 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4955 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4956 Action.Enter(CGF);
4957 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4958 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskEntry,
4959 Args: OutlinedFnArgs);
4960 };
4961
4962 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
4963 // kmp_task_t *new_task);
4964 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
4965 // kmp_task_t *new_task);
4966 RegionCodeGenTy RCG(CodeGen);
4967 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
4968 M, FnID: OMPRTL___kmpc_omp_task_begin_if0),
4969 TaskArgs,
4970 OMPBuilder.getOrCreateRuntimeFunction(
4971 M, FnID: OMPRTL___kmpc_omp_task_complete_if0),
4972 TaskArgs);
4973 RCG.setAction(Action);
4974 RCG(CGF);
4975 };
4976
4977 if (IfCond) {
4978 emitIfClause(CGF, Cond: IfCond, ThenGen: ThenCodeGen, ElseGen: ElseCodeGen);
4979 } else {
4980 RegionCodeGenTy ThenRCG(ThenCodeGen);
4981 ThenRCG(CGF);
4982 }
4983}
4984
4985void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4986 const OMPLoopDirective &D,
4987 llvm::Function *TaskFunction,
4988 QualType SharedsTy, Address Shareds,
4989 const Expr *IfCond,
4990 const OMPTaskDataTy &Data) {
4991 if (!CGF.HaveInsertPoint())
4992 return;
4993 TaskResultTy Result =
4994 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4995 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4996 // libcall.
4997 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
4998 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
4999 // sched, kmp_uint64 grainsize, void *task_dup);
5000 llvm::Value *ThreadID = getThreadID(CGF, Loc);
5001 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5002 llvm::Value *IfVal;
5003 if (IfCond) {
5004 IfVal = CGF.Builder.CreateIntCast(V: CGF.EvaluateExprAsBool(E: IfCond), DestTy: CGF.IntTy,
5005 /*isSigned=*/true);
5006 } else {
5007 IfVal = llvm::ConstantInt::getSigned(Ty: CGF.IntTy, /*V=*/1);
5008 }
5009
5010 LValue LBLVal = CGF.EmitLValueForField(
5011 Base: Result.TDBase,
5012 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound));
5013 const auto *LBVar =
5014 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getLowerBoundVariable())->getDecl());
5015 CGF.EmitAnyExprToMem(E: LBVar->getInit(), Location: LBLVal.getAddress(), Quals: LBLVal.getQuals(),
5016 /*IsInitializer=*/true);
5017 LValue UBLVal = CGF.EmitLValueForField(
5018 Base: Result.TDBase,
5019 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound));
5020 const auto *UBVar =
5021 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getUpperBoundVariable())->getDecl());
5022 CGF.EmitAnyExprToMem(E: UBVar->getInit(), Location: UBLVal.getAddress(), Quals: UBLVal.getQuals(),
5023 /*IsInitializer=*/true);
5024 LValue StLVal = CGF.EmitLValueForField(
5025 Base: Result.TDBase,
5026 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride));
5027 const auto *StVar =
5028 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getStrideVariable())->getDecl());
5029 CGF.EmitAnyExprToMem(E: StVar->getInit(), Location: StLVal.getAddress(), Quals: StLVal.getQuals(),
5030 /*IsInitializer=*/true);
5031 // Store reductions address.
5032 LValue RedLVal = CGF.EmitLValueForField(
5033 Base: Result.TDBase,
5034 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions));
5035 if (Data.Reductions) {
5036 CGF.EmitStoreOfScalar(value: Data.Reductions, lvalue: RedLVal);
5037 } else {
5038 CGF.EmitNullInitialization(DestPtr: RedLVal.getAddress(),
5039 Ty: CGF.getContext().VoidPtrTy);
5040 }
5041 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5042 llvm::SmallVector<llvm::Value *, 12> TaskArgs{
5043 UpLoc,
5044 ThreadID,
5045 Result.NewTask,
5046 IfVal,
5047 LBLVal.getPointer(CGF),
5048 UBLVal.getPointer(CGF),
5049 CGF.EmitLoadOfScalar(lvalue: StLVal, Loc),
5050 llvm::ConstantInt::getSigned(
5051 Ty: CGF.IntTy, V: 1), // Always 1 because taskgroup emitted by the compiler
5052 llvm::ConstantInt::getSigned(
5053 Ty: CGF.IntTy, V: Data.Schedule.getPointer()
5054 ? Data.Schedule.getInt() ? NumTasks : Grainsize
5055 : NoSchedule),
5056 Data.Schedule.getPointer()
5057 ? CGF.Builder.CreateIntCast(V: Data.Schedule.getPointer(), DestTy: CGF.Int64Ty,
5058 /*isSigned=*/false)
5059 : llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/0)};
5060 if (Data.HasModifier)
5061 TaskArgs.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 1));
5062
5063 TaskArgs.push_back(Elt: Result.TaskDupFn
5064 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5065 V: Result.TaskDupFn, DestTy: CGF.VoidPtrTy)
5066 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy));
5067 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5068 M&: CGM.getModule(), FnID: Data.HasModifier
5069 ? OMPRTL___kmpc_taskloop_5
5070 : OMPRTL___kmpc_taskloop),
5071 args: TaskArgs);
5072}
5073
5074/// Emit reduction operation for each element of array (required for
5075/// array sections) LHS op = RHS.
5076/// \param Type Type of array.
5077/// \param LHSVar Variable on the left side of the reduction operation
5078/// (references element of array in original variable).
5079/// \param RHSVar Variable on the right side of the reduction operation
5080/// (references element of array in original variable).
5081/// \param RedOpGen Generator of reduction operation with use of LHSVar and
5082/// RHSVar.
5083static void EmitOMPAggregateReduction(
5084 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5085 const VarDecl *RHSVar,
5086 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5087 const Expr *, const Expr *)> &RedOpGen,
5088 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5089 const Expr *UpExpr = nullptr) {
5090 // Perform element-by-element initialization.
5091 QualType ElementTy;
5092 Address LHSAddr = CGF.GetAddrOfLocalVar(VD: LHSVar);
5093 Address RHSAddr = CGF.GetAddrOfLocalVar(VD: RHSVar);
5094
5095 // Drill down to the base element type on both arrays.
5096 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5097 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: LHSAddr);
5098
5099 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
5100 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
5101 // Cast from pointer to array type to pointer to single element.
5102 llvm::Value *LHSEnd =
5103 CGF.Builder.CreateGEP(Ty: LHSAddr.getElementType(), Ptr: LHSBegin, IdxList: NumElements);
5104 // The basic structure here is a while-do loop.
5105 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arraycpy.body");
5106 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arraycpy.done");
5107 llvm::Value *IsEmpty =
5108 CGF.Builder.CreateICmpEQ(LHS: LHSBegin, RHS: LHSEnd, Name: "omp.arraycpy.isempty");
5109 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
5110
5111 // Enter the loop body, making that address the current address.
5112 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5113 CGF.EmitBlock(BB: BodyBB);
5114
5115 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
5116
5117 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5118 Ty: RHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
5119 RHSElementPHI->addIncoming(V: RHSBegin, BB: EntryBB);
5120 Address RHSElementCurrent(
5121 RHSElementPHI, RHSAddr.getElementType(),
5122 RHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
5123
5124 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5125 Ty: LHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
5126 LHSElementPHI->addIncoming(V: LHSBegin, BB: EntryBB);
5127 Address LHSElementCurrent(
5128 LHSElementPHI, LHSAddr.getElementType(),
5129 LHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
5130
5131 // Emit copy.
5132 CodeGenFunction::OMPPrivateScope Scope(CGF);
5133 Scope.addPrivate(LocalVD: LHSVar, Addr: LHSElementCurrent);
5134 Scope.addPrivate(LocalVD: RHSVar, Addr: RHSElementCurrent);
5135 Scope.Privatize();
5136 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5137 Scope.ForceCleanup();
5138
5139 // Shift the address forward by one element.
5140 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5141 Ty: LHSAddr.getElementType(), Ptr: LHSElementPHI, /*Idx0=*/1,
5142 Name: "omp.arraycpy.dest.element");
5143 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5144 Ty: RHSAddr.getElementType(), Ptr: RHSElementPHI, /*Idx0=*/1,
5145 Name: "omp.arraycpy.src.element");
5146 // Check whether we've reached the end.
5147 llvm::Value *Done =
5148 CGF.Builder.CreateICmpEQ(LHS: LHSElementNext, RHS: LHSEnd, Name: "omp.arraycpy.done");
5149 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
5150 LHSElementPHI->addIncoming(V: LHSElementNext, BB: CGF.Builder.GetInsertBlock());
5151 RHSElementPHI->addIncoming(V: RHSElementNext, BB: CGF.Builder.GetInsertBlock());
5152
5153 // Done.
5154 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
5155}
5156
5157/// Emit reduction combiner. If the combiner is a simple expression emit it as
5158/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5159/// UDR combiner function.
5160static void emitReductionCombiner(CodeGenFunction &CGF,
5161 const Expr *ReductionOp) {
5162 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
5163 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
5164 if (const auto *DRE =
5165 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
5166 if (const auto *DRD =
5167 dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl())) {
5168 std::pair<llvm::Function *, llvm::Function *> Reduction =
5169 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
5170 RValue Func = RValue::get(V: Reduction.first);
5171 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5172 CGF.EmitIgnoredExpr(E: ReductionOp);
5173 return;
5174 }
5175 CGF.EmitIgnoredExpr(E: ReductionOp);
5176}
5177
5178llvm::Function *CGOpenMPRuntime::emitReductionFunction(
5179 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
5180 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
5181 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
5182 ASTContext &C = CGM.getContext();
5183
5184 // void reduction_func(void *LHSArg, void *RHSArg);
5185 auto *LHSArg =
5186 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5187 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5188 auto *RHSArg =
5189 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5190 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5191 FunctionArgList Args{LHSArg, RHSArg};
5192 const auto &CGFI =
5193 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5194 std::string Name = getReductionFuncName(Name: ReducerName);
5195 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
5196 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
5197 M: &CGM.getModule());
5198 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
5199 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5200 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5201 Fn->setDoesNotRecurse();
5202 CodeGenFunction CGF(CGM);
5203 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
5204
5205 // Dst = (void*[n])(LHSArg);
5206 // Src = (void*[n])(RHSArg);
5207 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5208 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
5209 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5210 ArgsElemType, CGF.getPointerAlign());
5211 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5212 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
5213 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5214 ArgsElemType, CGF.getPointerAlign());
5215
5216 // ...
5217 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5218 // ...
5219 CodeGenFunction::OMPPrivateScope Scope(CGF);
5220 const auto *IPriv = Privates.begin();
5221 unsigned Idx = 0;
5222 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5223 const auto *RHSVar =
5224 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSExprs[I])->getDecl());
5225 Scope.addPrivate(LocalVD: RHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: RHS, Index: Idx, Var: RHSVar));
5226 const auto *LHSVar =
5227 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSExprs[I])->getDecl());
5228 Scope.addPrivate(LocalVD: LHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: LHS, Index: Idx, Var: LHSVar));
5229 QualType PrivTy = (*IPriv)->getType();
5230 if (PrivTy->isVariablyModifiedType()) {
5231 // Get array size and emit VLA type.
5232 ++Idx;
5233 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: LHS, Index: Idx);
5234 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Elem);
5235 const VariableArrayType *VLA =
5236 CGF.getContext().getAsVariableArrayType(T: PrivTy);
5237 const auto *OVE = cast<OpaqueValueExpr>(Val: VLA->getSizeExpr());
5238 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5239 CGF, OVE, RValue::get(V: CGF.Builder.CreatePtrToInt(V: Ptr, DestTy: CGF.SizeTy)));
5240 CGF.EmitVariablyModifiedType(Ty: PrivTy);
5241 }
5242 }
5243 Scope.Privatize();
5244 IPriv = Privates.begin();
5245 const auto *ILHS = LHSExprs.begin();
5246 const auto *IRHS = RHSExprs.begin();
5247 for (const Expr *E : ReductionOps) {
5248 if ((*IPriv)->getType()->isArrayType()) {
5249 // Emit reduction for array section.
5250 const auto *LHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5251 const auto *RHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5252 EmitOMPAggregateReduction(
5253 CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5254 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5255 emitReductionCombiner(CGF, ReductionOp: E);
5256 });
5257 } else {
5258 // Emit reduction for array subscript or single variable.
5259 emitReductionCombiner(CGF, ReductionOp: E);
5260 }
5261 ++IPriv;
5262 ++ILHS;
5263 ++IRHS;
5264 }
5265 Scope.ForceCleanup();
5266 CGF.FinishFunction();
5267 return Fn;
5268}
5269
5270void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5271 const Expr *ReductionOp,
5272 const Expr *PrivateRef,
5273 const DeclRefExpr *LHS,
5274 const DeclRefExpr *RHS) {
5275 if (PrivateRef->getType()->isArrayType()) {
5276 // Emit reduction for array section.
5277 const auto *LHSVar = cast<VarDecl>(Val: LHS->getDecl());
5278 const auto *RHSVar = cast<VarDecl>(Val: RHS->getDecl());
5279 EmitOMPAggregateReduction(
5280 CGF, Type: PrivateRef->getType(), LHSVar, RHSVar,
5281 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5282 emitReductionCombiner(CGF, ReductionOp);
5283 });
5284 } else {
5285 // Emit reduction for array subscript or single variable.
5286 emitReductionCombiner(CGF, ReductionOp);
5287 }
5288}
5289
5290static std::string generateUniqueName(CodeGenModule &CGM,
5291 llvm::StringRef Prefix, const Expr *Ref);
5292
5293void CGOpenMPRuntime::emitPrivateReduction(
5294 CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,
5295 const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {
5296
5297 // Create a shared global variable (__shared_reduction_var) to accumulate the
5298 // final result.
5299 //
5300 // Call __kmpc_barrier to synchronize threads before initialization.
5301 //
5302 // The master thread (thread_id == 0) initializes __shared_reduction_var
5303 // with the identity value or initializer.
5304 //
5305 // Call __kmpc_barrier to synchronize before combining.
5306 // For each i:
5307 // - Thread enters critical section.
5308 // - Reads its private value from LHSExprs[i].
5309 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
5310 // Privates[i]).
5311 // - Exits critical section.
5312 //
5313 // Call __kmpc_barrier after combining.
5314 //
5315 // Each thread copies __shared_reduction_var[i] back to RHSExprs[i].
5316 //
5317 // Final __kmpc_barrier to synchronize after broadcasting
5318 QualType PrivateType = Privates->getType();
5319 llvm::Type *LLVMType = CGF.ConvertTypeForMem(T: PrivateType);
5320
5321 const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOp: ReductionOps);
5322 std::string ReductionVarNameStr;
5323 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates->IgnoreParenCasts()))
5324 ReductionVarNameStr =
5325 generateUniqueName(CGM, Prefix: DRE->getDecl()->getNameAsString(), Ref: Privates);
5326 else
5327 ReductionVarNameStr = "unnamed_priv_var";
5328
5329 // Create an internal shared variable
5330 std::string SharedName =
5331 CGM.getOpenMPRuntime().getName(Parts: {"internal_pivate_", ReductionVarNameStr});
5332 llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(
5333 Ty: LLVMType, Name: ".omp.reduction." + SharedName);
5334
5335 SharedVar->setAlignment(
5336 llvm::MaybeAlign(CGF.getContext().getTypeAlign(T: PrivateType) / 8));
5337
5338 Address SharedResult =
5339 CGF.MakeNaturalAlignRawAddrLValue(V: SharedVar, T: PrivateType).getAddress();
5340
5341 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5342 llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5343 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
5344
5345 llvm::BasicBlock *InitBB = CGF.createBasicBlock(name: "init");
5346 llvm::BasicBlock *InitEndBB = CGF.createBasicBlock(name: "init.end");
5347
5348 llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(
5349 LHS: ThreadId, RHS: llvm::ConstantInt::get(Ty: ThreadId->getType(), V: 0));
5350 CGF.Builder.CreateCondBr(Cond: IsWorker, True: InitBB, False: InitEndBB);
5351
5352 CGF.EmitBlock(BB: InitBB);
5353
5354 auto EmitSharedInit = [&]() {
5355 if (UDR) { // Check if it's a User-Defined Reduction
5356 if (const Expr *UDRInitExpr = UDR->getInitializer()) {
5357 std::pair<llvm::Function *, llvm::Function *> FnPair =
5358 getUserDefinedReduction(D: UDR);
5359 llvm::Function *InitializerFn = FnPair.second;
5360 if (InitializerFn) {
5361 if (const auto *CE =
5362 dyn_cast<CallExpr>(Val: UDRInitExpr->IgnoreParenImpCasts())) {
5363 const auto *OutDRE = cast<DeclRefExpr>(
5364 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5365 ->getSubExpr());
5366 const VarDecl *OutVD = cast<VarDecl>(Val: OutDRE->getDecl());
5367
5368 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5369 LocalScope.addPrivate(LocalVD: OutVD, Addr: SharedResult);
5370
5371 (void)LocalScope.Privatize();
5372 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(
5373 Val: CE->getCallee()->IgnoreParenImpCasts())) {
5374 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5375 CGF, OVE, RValue::get(V: InitializerFn));
5376 CGF.EmitIgnoredExpr(E: CE);
5377 } else {
5378 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5379 Quals: PrivateType.getQualifiers(),
5380 /*IsInitializer=*/true);
5381 }
5382 } else {
5383 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5384 Quals: PrivateType.getQualifiers(),
5385 /*IsInitializer=*/true);
5386 }
5387 } else {
5388 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5389 Quals: PrivateType.getQualifiers(),
5390 /*IsInitializer=*/true);
5391 }
5392 } else {
5393 // EmitNullInitialization handles default construction for C++ classes
5394 // and zeroing for scalars, which is a reasonable default.
5395 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5396 }
5397 return; // UDR initialization handled
5398 }
5399 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates)) {
5400 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
5401 if (const Expr *InitExpr = VD->getInit()) {
5402 CGF.EmitAnyExprToMem(E: InitExpr, Location: SharedResult,
5403 Quals: PrivateType.getQualifiers(), IsInitializer: true);
5404 return;
5405 }
5406 }
5407 }
5408 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5409 };
5410 EmitSharedInit();
5411 CGF.Builder.CreateBr(Dest: InitEndBB);
5412 CGF.EmitBlock(BB: InitEndBB);
5413
5414 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5415 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5416 args: BarrierArgs);
5417
5418 const Expr *ReductionOp = ReductionOps;
5419 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
5420 LValue SharedLV = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5421 LValue LHSLV = CGF.EmitLValue(E: Privates);
5422
5423 auto EmitCriticalReduction = [&](auto ReductionGen) {
5424 std::string CriticalName = getName(Parts: {"reduction_critical"});
5425 emitCriticalRegion(CGF, CriticalName, CriticalOpGen: ReductionGen, Loc);
5426 };
5427
5428 if (CurrentUDR) {
5429 // Handle user-defined reduction.
5430 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5431 Action.Enter(CGF);
5432 std::pair<llvm::Function *, llvm::Function *> FnPair =
5433 getUserDefinedReduction(D: CurrentUDR);
5434 if (FnPair.first) {
5435 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp)) {
5436 const auto *OutDRE = cast<DeclRefExpr>(
5437 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5438 ->getSubExpr());
5439 const auto *InDRE = cast<DeclRefExpr>(
5440 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 1)->IgnoreParenImpCasts())
5441 ->getSubExpr());
5442 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5443 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: OutDRE->getDecl()),
5444 Addr: SharedLV.getAddress());
5445 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: InDRE->getDecl()),
5446 Addr: LHSLV.getAddress());
5447 (void)LocalScope.Privatize();
5448 emitReductionCombiner(CGF, ReductionOp);
5449 }
5450 }
5451 };
5452 EmitCriticalReduction(ReductionGen);
5453 } else {
5454 // Handle built-in reduction operations.
5455#ifndef NDEBUG
5456 const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();
5457 if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5458 ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
5459
5460 const Expr *AssignRHS = nullptr;
5461 if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5462 if (BinOp->getOpcode() == BO_Assign)
5463 AssignRHS = BinOp->getRHS();
5464 } else if (const auto *OpCall =
5465 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5466 if (OpCall->getOperator() == OO_Equal)
5467 AssignRHS = OpCall->getArg(1);
5468 }
5469
5470 assert(AssignRHS &&
5471 "Private Variable Reduction : Invalid ReductionOp expression");
5472#endif
5473
5474 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5475 Action.Enter(CGF);
5476 const auto *OmpOutDRE =
5477 dyn_cast<DeclRefExpr>(Val: LHSExprs->IgnoreParenImpCasts());
5478 const auto *OmpInDRE =
5479 dyn_cast<DeclRefExpr>(Val: RHSExprs->IgnoreParenImpCasts());
5480 assert(
5481 OmpOutDRE && OmpInDRE &&
5482 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5483 const VarDecl *OmpOutVD = cast<VarDecl>(Val: OmpOutDRE->getDecl());
5484 const VarDecl *OmpInVD = cast<VarDecl>(Val: OmpInDRE->getDecl());
5485 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5486 LocalScope.addPrivate(LocalVD: OmpOutVD, Addr: SharedLV.getAddress());
5487 LocalScope.addPrivate(LocalVD: OmpInVD, Addr: LHSLV.getAddress());
5488 (void)LocalScope.Privatize();
5489 // Emit the actual reduction operation
5490 CGF.EmitIgnoredExpr(E: ReductionOp);
5491 };
5492 EmitCriticalReduction(ReductionGen);
5493 }
5494
5495 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5496 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5497 args: BarrierArgs);
5498
5499 // Broadcast final result
5500 bool IsAggregate = PrivateType->isAggregateType();
5501 LValue SharedLV1 = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5502 llvm::Value *FinalResultVal = nullptr;
5503 Address FinalResultAddr = Address::invalid();
5504
5505 if (IsAggregate)
5506 FinalResultAddr = SharedResult;
5507 else
5508 FinalResultVal = CGF.EmitLoadOfScalar(lvalue: SharedLV1, Loc);
5509
5510 LValue TargetLHSLV = CGF.EmitLValue(E: RHSExprs);
5511 if (IsAggregate) {
5512 CGF.EmitAggregateCopy(Dest: TargetLHSLV,
5513 Src: CGF.MakeAddrLValue(Addr: FinalResultAddr, T: PrivateType),
5514 EltTy: PrivateType, MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: false);
5515 } else {
5516 CGF.EmitStoreOfScalar(value: FinalResultVal, lvalue: TargetLHSLV);
5517 }
5518 // Final synchronization barrier
5519 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5520 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5521 args: BarrierArgs);
5522
5523 // Combiner with original list item
5524 auto OriginalListCombiner = [&](CodeGenFunction &CGF,
5525 PrePostActionTy &Action) {
5526 Action.Enter(CGF);
5527 emitSingleReductionCombiner(CGF, ReductionOp: ReductionOps, PrivateRef: Privates,
5528 LHS: cast<DeclRefExpr>(Val: LHSExprs),
5529 RHS: cast<DeclRefExpr>(Val: RHSExprs));
5530 };
5531 EmitCriticalReduction(OriginalListCombiner);
5532}
5533
5534void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5535 ArrayRef<const Expr *> OrgPrivates,
5536 ArrayRef<const Expr *> OrgLHSExprs,
5537 ArrayRef<const Expr *> OrgRHSExprs,
5538 ArrayRef<const Expr *> OrgReductionOps,
5539 ReductionOptionsTy Options) {
5540 if (!CGF.HaveInsertPoint())
5541 return;
5542
5543 bool WithNowait = Options.WithNowait;
5544 bool SimpleReduction = Options.SimpleReduction;
5545
5546 // Next code should be emitted for reduction:
5547 //
5548 // static kmp_critical_name lock = { 0 };
5549 //
5550 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5551 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5552 // ...
5553 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5554 // *(Type<n>-1*)rhs[<n>-1]);
5555 // }
5556 //
5557 // ...
5558 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5559 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5560 // RedList, reduce_func, &<lock>)) {
5561 // case 1:
5562 // ...
5563 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5564 // ...
5565 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5566 // break;
5567 // case 2:
5568 // ...
5569 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5570 // ...
5571 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5572 // break;
5573 // default:;
5574 // }
5575 //
5576 // if SimpleReduction is true, only the next code is generated:
5577 // ...
5578 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5579 // ...
5580
5581 ASTContext &C = CGM.getContext();
5582
5583 if (SimpleReduction) {
5584 CodeGenFunction::RunCleanupsScope Scope(CGF);
5585 const auto *IPriv = OrgPrivates.begin();
5586 const auto *ILHS = OrgLHSExprs.begin();
5587 const auto *IRHS = OrgRHSExprs.begin();
5588 for (const Expr *E : OrgReductionOps) {
5589 emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5590 RHS: cast<DeclRefExpr>(Val: *IRHS));
5591 ++IPriv;
5592 ++ILHS;
5593 ++IRHS;
5594 }
5595 return;
5596 }
5597
5598 // Filter out shared reduction variables based on IsPrivateVarReduction flag.
5599 // Only keep entries where the corresponding variable is not private.
5600 SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
5601 FilteredRHSExprs, FilteredReductionOps;
5602 for (unsigned I : llvm::seq<unsigned>(
5603 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5604 if (!Options.IsPrivateVarReduction[I]) {
5605 FilteredPrivates.emplace_back(Args: OrgPrivates[I]);
5606 FilteredLHSExprs.emplace_back(Args: OrgLHSExprs[I]);
5607 FilteredRHSExprs.emplace_back(Args: OrgRHSExprs[I]);
5608 FilteredReductionOps.emplace_back(Args: OrgReductionOps[I]);
5609 }
5610 }
5611 // Wrap filtered vectors in ArrayRef for downstream shared reduction
5612 // processing.
5613 ArrayRef<const Expr *> Privates = FilteredPrivates;
5614 ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
5615 ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
5616 ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
5617
5618 // 1. Build a list of reduction variables.
5619 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5620 auto Size = RHSExprs.size();
5621 for (const Expr *E : Privates) {
5622 if (E->getType()->isVariablyModifiedType())
5623 // Reserve place for array size.
5624 ++Size;
5625 }
5626 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5627 QualType ReductionArrayTy = C.getConstantArrayType(
5628 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5629 /*IndexTypeQuals=*/0);
5630 RawAddress ReductionList =
5631 CGF.CreateMemTemp(T: ReductionArrayTy, Name: ".omp.reduction.red_list");
5632 const auto *IPriv = Privates.begin();
5633 unsigned Idx = 0;
5634 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5635 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5636 CGF.Builder.CreateStore(
5637 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5638 V: CGF.EmitLValue(E: RHSExprs[I]).getPointer(CGF), DestTy: CGF.VoidPtrTy),
5639 Addr: Elem);
5640 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5641 // Store array size.
5642 ++Idx;
5643 Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5644 llvm::Value *Size = CGF.Builder.CreateIntCast(
5645 V: CGF.getVLASize(
5646 vla: CGF.getContext().getAsVariableArrayType(T: (*IPriv)->getType()))
5647 .NumElts,
5648 DestTy: CGF.SizeTy, /*isSigned=*/false);
5649 CGF.Builder.CreateStore(Val: CGF.Builder.CreateIntToPtr(V: Size, DestTy: CGF.VoidPtrTy),
5650 Addr: Elem);
5651 }
5652 }
5653
5654 // 2. Emit reduce_func().
5655 llvm::Function *ReductionFn = emitReductionFunction(
5656 ReducerName: CGF.CurFn->getName(), Loc, ArgsElemType: CGF.ConvertTypeForMem(T: ReductionArrayTy),
5657 Privates, LHSExprs, RHSExprs, ReductionOps);
5658
5659 // 3. Create static kmp_critical_name lock = { 0 };
5660 std::string Name = getName(Parts: {"reduction"});
5661 llvm::Value *Lock = getCriticalRegionLock(CriticalName: Name);
5662
5663 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5664 // RedList, reduce_func, &<lock>);
5665 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5666 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5667 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(Ty: ReductionArrayTy);
5668 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5669 V: ReductionList.getPointer(), DestTy: CGF.VoidPtrTy);
5670 llvm::Value *Args[] = {
5671 IdentTLoc, // ident_t *<loc>
5672 ThreadId, // i32 <gtid>
5673 CGF.Builder.getInt32(C: RHSExprs.size()), // i32 <n>
5674 ReductionArrayTySize, // size_type sizeof(RedList)
5675 RL, // void *RedList
5676 ReductionFn, // void (*) (void *, void *) <reduce_func>
5677 Lock // kmp_critical_name *&<lock>
5678 };
5679 llvm::Value *Res = CGF.EmitRuntimeCall(
5680 callee: OMPBuilder.getOrCreateRuntimeFunction(
5681 M&: CGM.getModule(),
5682 FnID: WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5683 args: Args);
5684
5685 // 5. Build switch(res)
5686 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(name: ".omp.reduction.default");
5687 llvm::SwitchInst *SwInst =
5688 CGF.Builder.CreateSwitch(V: Res, Dest: DefaultBB, /*NumCases=*/2);
5689
5690 // 6. Build case 1:
5691 // ...
5692 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5693 // ...
5694 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5695 // break;
5696 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(name: ".omp.reduction.case1");
5697 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 1), Dest: Case1BB);
5698 CGF.EmitBlock(BB: Case1BB);
5699
5700 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5701 llvm::Value *EndArgs[] = {
5702 IdentTLoc, // ident_t *<loc>
5703 ThreadId, // i32 <gtid>
5704 Lock // kmp_critical_name *&<lock>
5705 };
5706 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5707 CodeGenFunction &CGF, PrePostActionTy &Action) {
5708 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5709 const auto *IPriv = Privates.begin();
5710 const auto *ILHS = LHSExprs.begin();
5711 const auto *IRHS = RHSExprs.begin();
5712 for (const Expr *E : ReductionOps) {
5713 RT.emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5714 RHS: cast<DeclRefExpr>(Val: *IRHS));
5715 ++IPriv;
5716 ++ILHS;
5717 ++IRHS;
5718 }
5719 };
5720 RegionCodeGenTy RCG(CodeGen);
5721 CommonActionTy Action(
5722 nullptr, {},
5723 OMPBuilder.getOrCreateRuntimeFunction(
5724 M&: CGM.getModule(), FnID: WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5725 : OMPRTL___kmpc_end_reduce),
5726 EndArgs);
5727 RCG.setAction(Action);
5728 RCG(CGF);
5729
5730 CGF.EmitBranch(Block: DefaultBB);
5731
5732 // 7. Build case 2:
5733 // ...
5734 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5735 // ...
5736 // break;
5737 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(name: ".omp.reduction.case2");
5738 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 2), Dest: Case2BB);
5739 CGF.EmitBlock(BB: Case2BB);
5740
5741 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5742 CodeGenFunction &CGF, PrePostActionTy &Action) {
5743 const auto *ILHS = LHSExprs.begin();
5744 const auto *IRHS = RHSExprs.begin();
5745 const auto *IPriv = Privates.begin();
5746 for (const Expr *E : ReductionOps) {
5747 const Expr *XExpr = nullptr;
5748 const Expr *EExpr = nullptr;
5749 const Expr *UpExpr = nullptr;
5750 BinaryOperatorKind BO = BO_Comma;
5751 if (const auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
5752 if (BO->getOpcode() == BO_Assign) {
5753 XExpr = BO->getLHS();
5754 UpExpr = BO->getRHS();
5755 }
5756 }
5757 // Try to emit update expression as a simple atomic.
5758 const Expr *RHSExpr = UpExpr;
5759 if (RHSExpr) {
5760 // Analyze RHS part of the whole expression.
5761 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5762 Val: RHSExpr->IgnoreParenImpCasts())) {
5763 // If this is a conditional operator, analyze its condition for
5764 // min/max reduction operator.
5765 RHSExpr = ACO->getCond();
5766 }
5767 if (const auto *BORHS =
5768 dyn_cast<BinaryOperator>(Val: RHSExpr->IgnoreParenImpCasts())) {
5769 EExpr = BORHS->getRHS();
5770 BO = BORHS->getOpcode();
5771 }
5772 }
5773 if (XExpr) {
5774 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5775 auto &&AtomicRedGen = [BO, VD,
5776 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5777 const Expr *EExpr, const Expr *UpExpr) {
5778 LValue X = CGF.EmitLValue(E: XExpr);
5779 RValue E;
5780 if (EExpr)
5781 E = CGF.EmitAnyExpr(E: EExpr);
5782 CGF.EmitOMPAtomicSimpleUpdateExpr(
5783 X, E, BO, /*IsXLHSInRHSPart=*/true,
5784 AO: llvm::AtomicOrdering::Monotonic, Loc,
5785 CommonGen: [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5786 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5787 Address LHSTemp = CGF.CreateMemTemp(T: VD->getType());
5788 CGF.emitOMPSimpleStore(
5789 LVal: CGF.MakeAddrLValue(Addr: LHSTemp, T: VD->getType()), RVal: XRValue,
5790 RValTy: VD->getType().getNonReferenceType(), Loc);
5791 PrivateScope.addPrivate(LocalVD: VD, Addr: LHSTemp);
5792 (void)PrivateScope.Privatize();
5793 return CGF.EmitAnyExpr(E: UpExpr);
5794 });
5795 };
5796 if ((*IPriv)->getType()->isArrayType()) {
5797 // Emit atomic reduction for array section.
5798 const auto *RHSVar =
5799 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5800 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar: VD, RHSVar,
5801 RedOpGen: AtomicRedGen, XExpr, EExpr, UpExpr);
5802 } else {
5803 // Emit atomic reduction for array subscript or single variable.
5804 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5805 }
5806 } else {
5807 // Emit as a critical region.
5808 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5809 const Expr *, const Expr *) {
5810 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5811 std::string Name = RT.getName(Parts: {"atomic_reduction"});
5812 RT.emitCriticalRegion(
5813 CGF, CriticalName: Name,
5814 CriticalOpGen: [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5815 Action.Enter(CGF);
5816 emitReductionCombiner(CGF, ReductionOp: E);
5817 },
5818 Loc);
5819 };
5820 if ((*IPriv)->getType()->isArrayType()) {
5821 const auto *LHSVar =
5822 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5823 const auto *RHSVar =
5824 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5825 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5826 RedOpGen: CritRedGen);
5827 } else {
5828 CritRedGen(CGF, nullptr, nullptr, nullptr);
5829 }
5830 }
5831 ++ILHS;
5832 ++IRHS;
5833 ++IPriv;
5834 }
5835 };
5836 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5837 if (!WithNowait) {
5838 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5839 llvm::Value *EndArgs[] = {
5840 IdentTLoc, // ident_t *<loc>
5841 ThreadId, // i32 <gtid>
5842 Lock // kmp_critical_name *&<lock>
5843 };
5844 CommonActionTy Action(nullptr, {},
5845 OMPBuilder.getOrCreateRuntimeFunction(
5846 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_reduce),
5847 EndArgs);
5848 AtomicRCG.setAction(Action);
5849 AtomicRCG(CGF);
5850 } else {
5851 AtomicRCG(CGF);
5852 }
5853
5854 CGF.EmitBranch(Block: DefaultBB);
5855 CGF.EmitBlock(BB: DefaultBB, /*IsFinished=*/true);
5856 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5857 "PrivateVarReduction: Privates size mismatch");
5858 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5859 "PrivateVarReduction: ReductionOps size mismatch");
5860 for (unsigned I : llvm::seq<unsigned>(
5861 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5862 if (Options.IsPrivateVarReduction[I])
5863 emitPrivateReduction(CGF, Loc, Privates: OrgPrivates[I], LHSExprs: OrgLHSExprs[I],
5864 RHSExprs: OrgRHSExprs[I], ReductionOps: OrgReductionOps[I]);
5865 }
5866}
5867
5868/// Generates unique name for artificial threadprivate variables.
5869/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5870static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5871 const Expr *Ref) {
5872 SmallString<256> Buffer;
5873 llvm::raw_svector_ostream Out(Buffer);
5874 const clang::DeclRefExpr *DE;
5875 const VarDecl *D = ::getBaseDecl(Ref, DE);
5876 if (!D) {
5877 auto *DRE = cast<DeclRefExpr>(Val: Ref);
5878 if (const auto *BD = dyn_cast<BindingDecl>(Val: DRE->getDecl())) {
5879 // For BindingDecls, use the decomposed declaration as the base.
5880 D = cast<VarDecl>(Val: BD->getDecomposedDecl());
5881 } else {
5882 D = cast<VarDecl>(Val: DRE->getDecl());
5883 }
5884 }
5885 D = D->getCanonicalDecl();
5886 std::string Name = CGM.getOpenMPRuntime().getName(
5887 Parts: {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(GD: D)});
5888 Out << Prefix << Name << "_"
5889 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5890 return std::string(Out.str());
5891}
5892
5893/// Emits reduction initializer function:
5894/// \code
5895/// void @.red_init(void* %arg, void* %orig) {
5896/// %0 = bitcast void* %arg to <type>*
5897/// store <type> <init>, <type>* %0
5898/// ret void
5899/// }
5900/// \endcode
5901static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5902 SourceLocation Loc,
5903 ReductionCodeGen &RCG, unsigned N) {
5904 ASTContext &C = CGM.getContext();
5905 QualType VoidPtrTy = C.VoidPtrTy;
5906 VoidPtrTy.addRestrict();
5907 FunctionArgList Args;
5908 auto *Param =
5909 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5910 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5911 auto *ParamOrig =
5912 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5913 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5914 Args.emplace_back(Args&: Param);
5915 Args.emplace_back(Args&: ParamOrig);
5916 const auto &FnInfo =
5917 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5918 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5919 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_init", ""});
5920 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5921 N: Name, M: &CGM.getModule());
5922 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5923 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5924 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5925 Fn->setDoesNotRecurse();
5926 CodeGenFunction CGF(CGM);
5927 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5928 QualType PrivateType = RCG.getPrivateType(N);
5929 Address PrivateAddr = CGF.EmitLoadOfPointer(
5930 Ptr: CGF.GetAddrOfLocalVar(VD: Param).withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5931 PtrTy: C.getPointerType(T: PrivateType)->castAs<PointerType>());
5932 llvm::Value *Size = nullptr;
5933 // If the size of the reduction item is non-constant, load it from global
5934 // threadprivate variable.
5935 if (RCG.getSizes(N).second) {
5936 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5937 CGF, VarType: CGM.getContext().getSizeType(),
5938 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5939 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5940 Ty: CGM.getContext().getSizeType(), Loc);
5941 }
5942 RCG.emitAggregateType(CGF, N, Size);
5943 Address OrigAddr = Address::invalid();
5944 // If initializer uses initializer from declare reduction construct, emit a
5945 // pointer to the address of the original reduction item (reuired by reduction
5946 // initializer)
5947 if (RCG.usesReductionInitializer(N)) {
5948 Address SharedAddr = CGF.GetAddrOfLocalVar(VD: ParamOrig);
5949 OrigAddr = CGF.EmitLoadOfPointer(
5950 Ptr: SharedAddr,
5951 PtrTy: CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5952 }
5953 // Emit the initializer:
5954 // %0 = bitcast void* %arg to <type>*
5955 // store <type> <init>, <type>* %0
5956 RCG.emitInitialization(CGF, N, PrivateAddr, SharedAddr: OrigAddr,
5957 DefaultInit: [](CodeGenFunction &) { return false; });
5958 CGF.FinishFunction();
5959 return Fn;
5960}
5961
5962/// Emits reduction combiner function:
5963/// \code
5964/// void @.red_comb(void* %arg0, void* %arg1) {
5965/// %lhs = bitcast void* %arg0 to <type>*
5966/// %rhs = bitcast void* %arg1 to <type>*
5967/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5968/// store <type> %2, <type>* %lhs
5969/// ret void
5970/// }
5971/// \endcode
5972static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5973 SourceLocation Loc,
5974 ReductionCodeGen &RCG, unsigned N,
5975 const Expr *ReductionOp,
5976 const Expr *LHS, const Expr *RHS,
5977 const Expr *PrivateRef) {
5978 ASTContext &C = CGM.getContext();
5979 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHS)->getDecl());
5980 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHS)->getDecl());
5981 FunctionArgList Args;
5982 auto *ParamInOut =
5983 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5984 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5985 auto *ParamIn =
5986 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5987 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5988 Args.emplace_back(Args&: ParamInOut);
5989 Args.emplace_back(Args&: ParamIn);
5990 const auto &FnInfo =
5991 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5992 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5993 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_comb", ""});
5994 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5995 N: Name, M: &CGM.getModule());
5996 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5997 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5998 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5999 Fn->setDoesNotRecurse();
6000 CodeGenFunction CGF(CGM);
6001 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
6002 llvm::Value *Size = nullptr;
6003 // If the size of the reduction item is non-constant, load it from global
6004 // threadprivate variable.
6005 if (RCG.getSizes(N).second) {
6006 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6007 CGF, VarType: CGM.getContext().getSizeType(),
6008 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6009 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
6010 Ty: CGM.getContext().getSizeType(), Loc);
6011 }
6012 RCG.emitAggregateType(CGF, N, Size);
6013 // Remap lhs and rhs variables to the addresses of the function arguments.
6014 // %lhs = bitcast void* %arg0 to <type>*
6015 // %rhs = bitcast void* %arg1 to <type>*
6016 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
6017 PrivateScope.addPrivate(
6018 LocalVD: LHSVD,
6019 // Pull out the pointer to the variable.
6020 Addr: CGF.EmitLoadOfPointer(
6021 Ptr: CGF.GetAddrOfLocalVar(VD: ParamInOut)
6022 .withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
6023 PtrTy: C.getPointerType(T: LHSVD->getType())->castAs<PointerType>()));
6024 PrivateScope.addPrivate(
6025 LocalVD: RHSVD,
6026 // Pull out the pointer to the variable.
6027 Addr: CGF.EmitLoadOfPointer(
6028 Ptr: CGF.GetAddrOfLocalVar(VD: ParamIn).withElementType(
6029 ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
6030 PtrTy: C.getPointerType(T: RHSVD->getType())->castAs<PointerType>()));
6031 PrivateScope.Privatize();
6032 // Emit the combiner body:
6033 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
6034 // store <type> %2, <type>* %lhs
6035 CGM.getOpenMPRuntime().emitSingleReductionCombiner(
6036 CGF, ReductionOp, PrivateRef, LHS: cast<DeclRefExpr>(Val: LHS),
6037 RHS: cast<DeclRefExpr>(Val: RHS));
6038 CGF.FinishFunction();
6039 return Fn;
6040}
6041
6042/// Emits reduction finalizer function:
6043/// \code
6044/// void @.red_fini(void* %arg) {
6045/// %0 = bitcast void* %arg to <type>*
6046/// <destroy>(<type>* %0)
6047/// ret void
6048/// }
6049/// \endcode
6050static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
6051 SourceLocation Loc,
6052 ReductionCodeGen &RCG, unsigned N) {
6053 if (!RCG.needCleanups(N))
6054 return nullptr;
6055 ASTContext &C = CGM.getContext();
6056 FunctionArgList Args;
6057 auto *Param =
6058 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
6059 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
6060 Args.emplace_back(Args&: Param);
6061 const auto &FnInfo =
6062 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
6063 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
6064 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_fini", ""});
6065 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
6066 N: Name, M: &CGM.getModule());
6067 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
6068 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
6069 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
6070 Fn->setDoesNotRecurse();
6071 CodeGenFunction CGF(CGM);
6072 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
6073 Address PrivateAddr = CGF.EmitLoadOfPointer(
6074 Ptr: CGF.GetAddrOfLocalVar(VD: Param), PtrTy: C.VoidPtrTy.castAs<PointerType>());
6075 llvm::Value *Size = nullptr;
6076 // If the size of the reduction item is non-constant, load it from global
6077 // threadprivate variable.
6078 if (RCG.getSizes(N).second) {
6079 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
6080 CGF, VarType: CGM.getContext().getSizeType(),
6081 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6082 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
6083 Ty: CGM.getContext().getSizeType(), Loc);
6084 }
6085 RCG.emitAggregateType(CGF, N, Size);
6086 // Emit the finalizer body:
6087 // <destroy>(<type>* %0)
6088 RCG.emitCleanups(CGF, N, PrivateAddr);
6089 CGF.FinishFunction(EndLoc: Loc);
6090 return Fn;
6091}
6092
6093llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
6094 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
6095 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
6096 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
6097 return nullptr;
6098
6099 // Build typedef struct:
6100 // kmp_taskred_input {
6101 // void *reduce_shar; // shared reduction item
6102 // void *reduce_orig; // original reduction item used for initialization
6103 // size_t reduce_size; // size of data item
6104 // void *reduce_init; // data initialization routine
6105 // void *reduce_fini; // data finalization routine
6106 // void *reduce_comb; // data combiner routine
6107 // kmp_task_red_flags_t flags; // flags for additional info from compiler
6108 // } kmp_taskred_input_t;
6109 ASTContext &C = CGM.getContext();
6110 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_taskred_input_t");
6111 RD->startDefinition();
6112 const FieldDecl *SharedFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6113 const FieldDecl *OrigFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6114 const FieldDecl *SizeFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.getSizeType());
6115 const FieldDecl *InitFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6116 const FieldDecl *FiniFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6117 const FieldDecl *CombFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
6118 const FieldDecl *FlagsFD = addFieldToRecordDecl(
6119 C, DC: RD, FieldTy: C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
6120 RD->completeDefinition();
6121 CanQualType RDType = C.getCanonicalTagType(TD: RD);
6122 unsigned Size = Data.ReductionVars.size();
6123 llvm::APInt ArraySize(/*numBits=*/64, Size);
6124 QualType ArrayRDType =
6125 C.getConstantArrayType(EltTy: RDType, ArySize: ArraySize, SizeExpr: nullptr,
6126 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
6127 // kmp_task_red_input_t .rd_input.[Size];
6128 RawAddress TaskRedInput = CGF.CreateMemTemp(T: ArrayRDType, Name: ".rd_input.");
6129 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
6130 Data.ReductionCopies, Data.ReductionOps);
6131 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6132 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
6133 llvm::Value *Idxs[] = {llvm::ConstantInt::get(Ty: CGM.SizeTy, /*V=*/0),
6134 llvm::ConstantInt::get(Ty: CGM.SizeTy, V: Cnt)};
6135 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
6136 ElemTy: TaskRedInput.getElementType(), Ptr: TaskRedInput.getPointer(), IdxList: Idxs,
6137 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
6138 Name: ".rd_input.gep.");
6139 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(V: GEP, T: RDType);
6140 // ElemLVal.reduce_shar = &Shareds[Cnt];
6141 LValue SharedLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SharedFD);
6142 RCG.emitSharedOrigLValue(CGF, N: Cnt);
6143 llvm::Value *Shared = RCG.getSharedLValue(N: Cnt).getPointer(CGF);
6144 CGF.EmitStoreOfScalar(value: Shared, lvalue: SharedLVal);
6145 // ElemLVal.reduce_orig = &Origs[Cnt];
6146 LValue OrigLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: OrigFD);
6147 llvm::Value *Orig = RCG.getOrigLValue(N: Cnt).getPointer(CGF);
6148 CGF.EmitStoreOfScalar(value: Orig, lvalue: OrigLVal);
6149 RCG.emitAggregateType(CGF, N: Cnt);
6150 llvm::Value *SizeValInChars;
6151 llvm::Value *SizeVal;
6152 std::tie(args&: SizeValInChars, args&: SizeVal) = RCG.getSizes(N: Cnt);
6153 // We use delayed creation/initialization for VLAs and array sections. It is
6154 // required because runtime does not provide the way to pass the sizes of
6155 // VLAs/array sections to initializer/combiner/finalizer functions. Instead
6156 // threadprivate global variables are used to store these values and use
6157 // them in the functions.
6158 bool DelayedCreation = !!SizeVal;
6159 SizeValInChars = CGF.Builder.CreateIntCast(V: SizeValInChars, DestTy: CGM.SizeTy,
6160 /*isSigned=*/false);
6161 LValue SizeLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SizeFD);
6162 CGF.EmitStoreOfScalar(value: SizeValInChars, lvalue: SizeLVal);
6163 // ElemLVal.reduce_init = init;
6164 LValue InitLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: InitFD);
6165 llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, N: Cnt);
6166 CGF.EmitStoreOfScalar(value: InitAddr, lvalue: InitLVal);
6167 // ElemLVal.reduce_fini = fini;
6168 LValue FiniLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FiniFD);
6169 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, N: Cnt);
6170 llvm::Value *FiniAddr =
6171 Fini ? Fini : llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
6172 CGF.EmitStoreOfScalar(value: FiniAddr, lvalue: FiniLVal);
6173 // ElemLVal.reduce_comb = comb;
6174 LValue CombLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: CombFD);
6175 llvm::Value *CombAddr = emitReduceCombFunction(
6176 CGM, Loc, RCG, N: Cnt, ReductionOp: Data.ReductionOps[Cnt], LHS: LHSExprs[Cnt],
6177 RHS: RHSExprs[Cnt], PrivateRef: Data.ReductionCopies[Cnt]);
6178 CGF.EmitStoreOfScalar(value: CombAddr, lvalue: CombLVal);
6179 // ElemLVal.flags = 0;
6180 LValue FlagsLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FlagsFD);
6181 if (DelayedCreation) {
6182 CGF.EmitStoreOfScalar(
6183 value: llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/1, /*isSigned=*/IsSigned: true),
6184 lvalue: FlagsLVal);
6185 } else
6186 CGF.EmitNullInitialization(DestPtr: FlagsLVal.getAddress(), Ty: FlagsLVal.getType());
6187 }
6188 if (Data.IsReductionWithTaskMod) {
6189 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6190 // is_ws, int num, void *data);
6191 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6192 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6193 DestTy: CGM.IntTy, /*isSigned=*/true);
6194 llvm::Value *Args[] = {
6195 IdentTLoc, GTid,
6196 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Data.IsWorksharingReduction ? 1 : 0,
6197 /*isSigned=*/IsSigned: true),
6198 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6199 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6200 V: TaskRedInput.getPointer(), DestTy: CGM.VoidPtrTy)};
6201 return CGF.EmitRuntimeCall(
6202 callee: OMPBuilder.getOrCreateRuntimeFunction(
6203 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_modifier_init),
6204 args: Args);
6205 }
6206 // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data);
6207 llvm::Value *Args[] = {
6208 CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc), DestTy: CGM.IntTy,
6209 /*isSigned=*/true),
6210 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6211 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: TaskRedInput.getPointer(),
6212 DestTy: CGM.VoidPtrTy)};
6213 return CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6214 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_init),
6215 args: Args);
6216}
6217
6218void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
6219 SourceLocation Loc,
6220 bool IsWorksharingReduction) {
6221 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6222 // is_ws, int num, void *data);
6223 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6224 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6225 DestTy: CGM.IntTy, /*isSigned=*/true);
6226 llvm::Value *Args[] = {IdentTLoc, GTid,
6227 llvm::ConstantInt::get(Ty: CGM.IntTy,
6228 V: IsWorksharingReduction ? 1 : 0,
6229 /*isSigned=*/IsSigned: true)};
6230 (void)CGF.EmitRuntimeCall(
6231 callee: OMPBuilder.getOrCreateRuntimeFunction(
6232 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_modifier_fini),
6233 args: Args);
6234}
6235
6236void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6237 SourceLocation Loc,
6238 ReductionCodeGen &RCG,
6239 unsigned N) {
6240 auto Sizes = RCG.getSizes(N);
6241 // Emit threadprivate global variable if the type is non-constant
6242 // (Sizes.second = nullptr).
6243 if (Sizes.second) {
6244 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(V: Sizes.second, DestTy: CGM.SizeTy,
6245 /*isSigned=*/false);
6246 Address SizeAddr = getAddrOfArtificialThreadPrivate(
6247 CGF, VarType: CGM.getContext().getSizeType(),
6248 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6249 CGF.Builder.CreateStore(Val: SizeVal, Addr: SizeAddr, /*IsVolatile=*/false);
6250 }
6251}
6252
6253Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6254 SourceLocation Loc,
6255 llvm::Value *ReductionsPtr,
6256 LValue SharedLVal) {
6257 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6258 // *d);
6259 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6260 DestTy: CGM.IntTy,
6261 /*isSigned=*/true),
6262 ReductionsPtr,
6263 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6264 V: SharedLVal.getPointer(CGF), DestTy: CGM.VoidPtrTy)};
6265 return Address(
6266 CGF.EmitRuntimeCall(
6267 callee: OMPBuilder.getOrCreateRuntimeFunction(
6268 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_get_th_data),
6269 args: Args),
6270 CGF.Int8Ty, SharedLVal.getAlignment());
6271}
6272
6273void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
6274 const OMPTaskDataTy &Data) {
6275 if (!CGF.HaveInsertPoint())
6276 return;
6277
6278 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
6279 // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder.
6280 OMPBuilder.createTaskwait(Loc: CGF.Builder);
6281 } else {
6282 llvm::Value *ThreadID = getThreadID(CGF, Loc);
6283 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
6284 auto &M = CGM.getModule();
6285 Address DependenciesArray = Address::invalid();
6286 llvm::Value *NumOfElements;
6287 std::tie(args&: NumOfElements, args&: DependenciesArray) =
6288 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
6289 if (!Data.Dependences.empty()) {
6290 llvm::Value *DepWaitTaskArgs[7];
6291 DepWaitTaskArgs[0] = UpLoc;
6292 DepWaitTaskArgs[1] = ThreadID;
6293 DepWaitTaskArgs[2] = NumOfElements;
6294 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
6295 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
6296 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6297 DepWaitTaskArgs[6] =
6298 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
6299
6300 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
6301
6302 // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
6303 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
6304 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
6305 // kmp_int32 has_no_wait); if dependence info is specified.
6306 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6307 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
6308 args: DepWaitTaskArgs);
6309
6310 } else {
6311
6312 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6313 // global_tid);
6314 llvm::Value *Args[] = {UpLoc, ThreadID};
6315 // Ignore return result until untied tasks are supported.
6316 CGF.EmitRuntimeCall(
6317 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_omp_taskwait),
6318 args: Args);
6319 }
6320 }
6321
6322 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
6323 Region->emitUntiedSwitch(CGF);
6324}
6325
6326void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6327 OpenMPDirectiveKind InnerKind,
6328 const RegionCodeGenTy &CodeGen,
6329 bool HasCancel) {
6330 if (!CGF.HaveInsertPoint())
6331 return;
6332 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
6333 InnerKind != OMPD_critical &&
6334 InnerKind != OMPD_master &&
6335 InnerKind != OMPD_masked);
6336 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6337}
6338
6339namespace {
6340enum RTCancelKind {
6341 CancelNoreq = 0,
6342 CancelParallel = 1,
6343 CancelLoop = 2,
6344 CancelSections = 3,
6345 CancelTaskgroup = 4
6346};
6347} // anonymous namespace
6348
6349static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6350 RTCancelKind CancelKind = CancelNoreq;
6351 if (CancelRegion == OMPD_parallel)
6352 CancelKind = CancelParallel;
6353 else if (CancelRegion == OMPD_for)
6354 CancelKind = CancelLoop;
6355 else if (CancelRegion == OMPD_sections)
6356 CancelKind = CancelSections;
6357 else {
6358 assert(CancelRegion == OMPD_taskgroup);
6359 CancelKind = CancelTaskgroup;
6360 }
6361 return CancelKind;
6362}
6363
6364void CGOpenMPRuntime::emitCancellationPointCall(
6365 CodeGenFunction &CGF, SourceLocation Loc,
6366 OpenMPDirectiveKind CancelRegion) {
6367 if (!CGF.HaveInsertPoint())
6368 return;
6369 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6370 // global_tid, kmp_int32 cncl_kind);
6371 if (auto *OMPRegionInfo =
6372 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6373 // For 'cancellation point taskgroup', the task region info may not have a
6374 // cancel. This may instead happen in another adjacent task.
6375 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6376 llvm::Value *Args[] = {
6377 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6378 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6379 // Ignore return result until untied tasks are supported.
6380 llvm::Value *Result = CGF.EmitRuntimeCall(
6381 callee: OMPBuilder.getOrCreateRuntimeFunction(
6382 M&: CGM.getModule(), FnID: OMPRTL___kmpc_cancellationpoint),
6383 args: Args);
6384 // if (__kmpc_cancellationpoint()) {
6385 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6386 // exit from construct;
6387 // }
6388 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6389 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6390 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6391 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6392 CGF.EmitBlock(BB: ExitBB);
6393 if (CancelRegion == OMPD_parallel)
6394 emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6395 // exit from construct;
6396 CodeGenFunction::JumpDest CancelDest =
6397 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6398 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6399 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6400 }
6401 }
6402}
6403
6404void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6405 const Expr *IfCond,
6406 OpenMPDirectiveKind CancelRegion) {
6407 if (!CGF.HaveInsertPoint())
6408 return;
6409 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6410 // kmp_int32 cncl_kind);
6411 auto &M = CGM.getModule();
6412 if (auto *OMPRegionInfo =
6413 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6414 auto &&ThenGen = [this, &M, Loc, CancelRegion,
6415 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
6416 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6417 llvm::Value *Args[] = {
6418 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6419 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6420 // Ignore return result until untied tasks are supported.
6421 llvm::Value *Result = CGF.EmitRuntimeCall(
6422 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_cancel), args: Args);
6423 // if (__kmpc_cancel()) {
6424 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6425 // exit from construct;
6426 // }
6427 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6428 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6429 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6430 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6431 CGF.EmitBlock(BB: ExitBB);
6432 if (CancelRegion == OMPD_parallel)
6433 RT.emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6434 // exit from construct;
6435 CodeGenFunction::JumpDest CancelDest =
6436 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6437 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6438 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6439 };
6440 if (IfCond) {
6441 emitIfClause(CGF, Cond: IfCond, ThenGen,
6442 ElseGen: [](CodeGenFunction &, PrePostActionTy &) {});
6443 } else {
6444 RegionCodeGenTy ThenRCG(ThenGen);
6445 ThenRCG(CGF);
6446 }
6447 }
6448}
6449
6450namespace {
6451/// Cleanup action for uses_allocators support.
6452class OMPUsesAllocatorsActionTy final : public PrePostActionTy {
6453 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators;
6454 const OMPExecutableDirective &D;
6455 bool IsOffloadEntry;
6456
6457public:
6458 OMPUsesAllocatorsActionTy(
6459 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators,
6460 const OMPExecutableDirective &D, bool IsOffloadEntry)
6461 : Allocators(Allocators), D(D), IsOffloadEntry(IsOffloadEntry) {}
6462 void Enter(CodeGenFunction &CGF) override {
6463 if (!CGF.HaveInsertPoint())
6464 return;
6465 for (const auto &AllocatorData : Allocators) {
6466 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(
6467 CGF, Allocator: AllocatorData.first, AllocatorTraits: AllocatorData.second);
6468 }
6469 // This kernel does not go through the device-side runtime
6470 // init/deinit sequence (that is GPU-only), but the runtime still
6471 // needs a '<kernel>_kernel_environment' global to know how the
6472 // kernel was configured, so emit it directly here.
6473 if (IsOffloadEntry)
6474 CGF.CGM.getOpenMPRuntime().emitHostKernelEnvironment(D, CGF);
6475 }
6476 void Exit(CodeGenFunction &CGF) override {
6477 if (!CGF.HaveInsertPoint())
6478 return;
6479 for (const auto &AllocatorData : Allocators) {
6480 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,
6481 Allocator: AllocatorData.first);
6482 }
6483 }
6484};
6485} // namespace
6486
6487void CGOpenMPRuntime::emitHostKernelEnvironment(const OMPExecutableDirective &D,
6488 CodeGenFunction &CGF) {
6489 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs Attrs;
6490 Attrs.ExecFlags = llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC;
6491 computeMinAndMaxThreadsAndTeams(D, CGF, Attrs);
6492 OMPBuilder.emitKernelEnvironment(Loc: CGF.Builder, Attrs);
6493}
6494
6495StringRef
6496CGOpenMPRuntime::getTargetRegionParentName(const FunctionDecl *FD) const {
6497 // Regions in a closure are named after its enclosing non-closure function
6498 // (CodeGenFunction::CurFuncDecl), if there is one.
6499 if (const auto *Parent =
6500 dyn_cast_or_null<FunctionDecl>(Val: FD->getNonClosureContext()))
6501 FD = Parent;
6502 if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(Val: FD))
6503 return CGM.getMangledName(GD: GlobalDecl(Ctor, Ctor_Complete));
6504 if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(Val: FD))
6505 return CGM.getMangledName(GD: GlobalDecl(Dtor, Dtor_Complete));
6506 return CGM.getMangledName(GD: GlobalDecl(FD));
6507}
6508
6509void CGOpenMPRuntime::emitTargetOutlinedFunction(
6510 const OMPExecutableDirective &D, StringRef ParentName,
6511 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6512 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6513 assert(!ParentName.empty() && "Invalid target entry parent name!");
6514 HasEmittedTargetRegion = true;
6515 SmallVector<std::pair<const Expr *, const Expr *>, 4> Allocators;
6516 for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) {
6517 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
6518 const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
6519 if (!D.AllocatorTraits)
6520 continue;
6521 Allocators.emplace_back(Args: D.Allocator, Args: D.AllocatorTraits);
6522 }
6523 }
6524 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators, D, IsOffloadEntry);
6525 CodeGen.setAction(UsesAllocatorAction);
6526 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6527 IsOffloadEntry, CodeGen);
6528}
6529
6530void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
6531 const Expr *Allocator,
6532 const Expr *AllocatorTraits) {
6533 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6534 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6535 // Use default memspace handle.
6536 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6537 llvm::Value *NumTraits = llvm::ConstantInt::get(
6538 Ty: CGF.IntTy, V: cast<ConstantArrayType>(
6539 Val: AllocatorTraits->getType()->getAsArrayTypeUnsafe())
6540 ->getSize()
6541 .getLimitedValue());
6542 LValue AllocatorTraitsLVal = CGF.EmitLValue(E: AllocatorTraits);
6543 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6544 Addr: AllocatorTraitsLVal.getAddress(), Ty: CGF.VoidPtrPtrTy, ElementTy: CGF.VoidPtrTy);
6545 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, T: CGF.getContext().VoidPtrTy,
6546 BaseInfo: AllocatorTraitsLVal.getBaseInfo(),
6547 TBAAInfo: AllocatorTraitsLVal.getTBAAInfo());
6548 llvm::Value *Traits = Addr.emitRawPointer(CGF);
6549
6550 llvm::Value *AllocatorVal =
6551 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6552 M&: CGM.getModule(), FnID: OMPRTL___kmpc_init_allocator),
6553 args: {ThreadId, MemSpaceHandle, NumTraits, Traits});
6554 // Store to allocator.
6555 CGF.EmitAutoVarAlloca(var: *cast<VarDecl>(
6556 Val: cast<DeclRefExpr>(Val: Allocator->IgnoreParenImpCasts())->getDecl()));
6557 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6558 AllocatorVal =
6559 CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: CGF.getContext().VoidPtrTy,
6560 DstTy: Allocator->getType(), Loc: Allocator->getExprLoc());
6561 CGF.EmitStoreOfScalar(value: AllocatorVal, lvalue: AllocatorLVal);
6562}
6563
6564void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
6565 const Expr *Allocator) {
6566 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6567 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6568 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6569 llvm::Value *AllocatorVal =
6570 CGF.EmitLoadOfScalar(lvalue: AllocatorLVal, Loc: Allocator->getExprLoc());
6571 AllocatorVal = CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: Allocator->getType(),
6572 DstTy: CGF.getContext().VoidPtrTy,
6573 Loc: Allocator->getExprLoc());
6574 (void)CGF.EmitRuntimeCall(
6575 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
6576 FnID: OMPRTL___kmpc_destroy_allocator),
6577 args: {ThreadId, AllocatorVal});
6578}
6579
6580void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams(
6581 const OMPExecutableDirective &D, CodeGenFunction &CGF,
6582 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6583 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6584 "invalid default attrs structure");
6585 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6586 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6587
6588 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: Attrs.MinTeams.front(),
6589 MaxTeamsVal);
6590 getNumThreadsExprForTargetDirective(CGF, D, UpperBound&: MaxThreadsVal,
6591 /*UpperBoundOnly=*/true);
6592
6593 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6594 for (auto *A : C->getAttrs()) {
6595 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6596 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6597 if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(Val: A))
6598 CGM.handleCUDALaunchBoundsAttr(F: nullptr, A: Attr, MaxThreadsVal: &AttrMaxThreadsVal,
6599 MinBlocksVal: &AttrMinBlocksVal, MaxClusterRankVal: &AttrMaxBlocksVal);
6600 else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(Val: A))
6601 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6602 F: nullptr, A: Attr, /*ReqdWGS=*/nullptr, MinThreadsVal: &AttrMinThreadsVal,
6603 MaxThreadsVal: &AttrMaxThreadsVal);
6604 else
6605 continue;
6606
6607 Attrs.MinThreads.front() =
6608 std::max(a: Attrs.MinThreads.front(), b: AttrMinThreadsVal);
6609 if (AttrMaxThreadsVal > 0)
6610 MaxThreadsVal = MaxThreadsVal > 0
6611 ? std::min(a: MaxThreadsVal, b: AttrMaxThreadsVal)
6612 : AttrMaxThreadsVal;
6613 Attrs.MinTeams.front() =
6614 std::max(a: Attrs.MinTeams.front(), b: AttrMinBlocksVal);
6615 if (AttrMaxBlocksVal > 0)
6616 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(a: MaxTeamsVal, b: AttrMaxBlocksVal)
6617 : AttrMaxBlocksVal;
6618 }
6619 }
6620}
6621
6622void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6623 const OMPExecutableDirective &D, StringRef ParentName,
6624 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6625 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6626
6627 llvm::TargetRegionEntryInfo EntryInfo =
6628 getEntryInfoFromPresumedLoc(CGM, OMPBuilder, BeginLoc: D.getBeginLoc(), ParentName);
6629
6630 CodeGenFunction CGF(CGM, true);
6631 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6632 [&CGF, &D, &CodeGen, this](StringRef EntryFnName) {
6633 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
6634
6635 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6636 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6637 if (CGM.getLangOpts().OpenMPIsTargetDevice && !isGPU())
6638 return CGF.GenerateOpenMPCapturedStmtFunctionAggregate(S: CS, D);
6639 return CGF.GenerateOpenMPCapturedStmtFunction(S: CS, D);
6640 };
6641
6642 cantFail(Err: OMPBuilder.emitTargetRegionFunction(
6643 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6644 OutlinedFnID));
6645
6646 if (!OutlinedFn)
6647 return;
6648
6649 // A target body is entered once, from the kernel, and never re-entered by
6650 // the runtime, so it cannot occur in a cycle.
6651 OutlinedFn->setDoesNotRecurse();
6652
6653 CGM.getTargetCodeGenInfo().setTargetAttributes(D: nullptr, GV: OutlinedFn, M&: CGM);
6654
6655 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6656 for (auto *A : C->getAttrs()) {
6657 if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(Val: A))
6658 CGM.handleAMDGPUWavesPerEUAttr(F: OutlinedFn, A: Attr);
6659 }
6660 }
6661 registerVTable(D);
6662}
6663
6664/// Checks if the expression is constant or does not have non-trivial function
6665/// calls.
6666static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6667 // We can skip constant expressions.
6668 // We can skip expressions with trivial calls or simple expressions.
6669 return (E->isEvaluatable(Ctx, AllowSideEffects: Expr::SE_AllowUndefinedBehavior) ||
6670 !E->hasNonTrivialCall(Ctx)) &&
6671 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6672}
6673
6674const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,
6675 const Stmt *Body) {
6676 const Stmt *Child = Body->IgnoreContainers();
6677 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Val: Child)) {
6678 Child = nullptr;
6679 for (const Stmt *S : C->body()) {
6680 if (const auto *E = dyn_cast<Expr>(Val: S)) {
6681 if (isTrivial(Ctx, E))
6682 continue;
6683 }
6684 // Some of the statements can be ignored.
6685 if (isa<AsmStmt>(Val: S) || isa<NullStmt>(Val: S) || isa<OMPFlushDirective>(Val: S) ||
6686 isa<OMPBarrierDirective>(Val: S) || isa<OMPTaskyieldDirective>(Val: S))
6687 continue;
6688 // Analyze declarations.
6689 if (const auto *DS = dyn_cast<DeclStmt>(Val: S)) {
6690 if (llvm::all_of(Range: DS->decls(), P: [](const Decl *D) {
6691 if (isa<EmptyDecl>(Val: D) || isa<DeclContext>(Val: D) ||
6692 isa<TypeDecl>(Val: D) || isa<PragmaCommentDecl>(Val: D) ||
6693 isa<PragmaDetectMismatchDecl>(Val: D) || isa<UsingDecl>(Val: D) ||
6694 isa<UsingDirectiveDecl>(Val: D) ||
6695 isa<OMPDeclareReductionDecl>(Val: D) ||
6696 isa<OMPThreadPrivateDecl>(Val: D) || isa<OMPAllocateDecl>(Val: D))
6697 return true;
6698 const auto *VD = dyn_cast<VarDecl>(Val: D);
6699 if (!VD)
6700 return false;
6701 return VD->hasGlobalStorage() || !VD->isUsed();
6702 }))
6703 continue;
6704 }
6705 // Found multiple children - cannot get the one child only.
6706 if (Child)
6707 return nullptr;
6708 Child = S;
6709 }
6710 if (Child)
6711 Child = Child->IgnoreContainers();
6712 }
6713 return Child;
6714}
6715
6716const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(
6717 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6718 int32_t &MaxTeamsVal) {
6719
6720 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6721 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6722 "Expected target-based executable directive.");
6723 switch (DirectiveKind) {
6724 case OMPD_target: {
6725 const auto *CS = D.getInnermostCapturedStmt();
6726 const auto *Body =
6727 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6728 const Stmt *ChildStmt =
6729 CGOpenMPRuntime::getSingleCompoundChild(Ctx&: CGF.getContext(), Body);
6730 if (const auto *NestedDir =
6731 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
6732 if (isOpenMPTeamsDirective(DKind: NestedDir->getDirectiveKind())) {
6733 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6734 const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>()
6735 ->getNumTeams()
6736 .front();
6737 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6738 if (auto Constant =
6739 NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6740 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6741 return NumTeams;
6742 }
6743 MinTeamsVal = MaxTeamsVal = 0;
6744 return nullptr;
6745 }
6746 MinTeamsVal = MaxTeamsVal = 1;
6747 return nullptr;
6748 }
6749 // A value of -1 is used to check if we need to emit no teams region
6750 MinTeamsVal = MaxTeamsVal = -1;
6751 return nullptr;
6752 }
6753 case OMPD_target_teams_loop:
6754 case OMPD_target_teams:
6755 case OMPD_target_teams_distribute:
6756 case OMPD_target_teams_distribute_simd:
6757 case OMPD_target_teams_distribute_parallel_for:
6758 case OMPD_target_teams_distribute_parallel_for_simd: {
6759 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6760 const Expr *NumTeams =
6761 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front();
6762 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6763 if (auto Constant = NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6764 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6765 return NumTeams;
6766 }
6767 MinTeamsVal = MaxTeamsVal = 0;
6768 return nullptr;
6769 }
6770 case OMPD_target_parallel:
6771 case OMPD_target_parallel_for:
6772 case OMPD_target_parallel_for_simd:
6773 case OMPD_target_parallel_loop:
6774 case OMPD_target_simd:
6775 case OMPD_target_loop:
6776 MinTeamsVal = MaxTeamsVal = 1;
6777 return nullptr;
6778 case OMPD_parallel:
6779 case OMPD_for:
6780 case OMPD_parallel_for:
6781 case OMPD_parallel_loop:
6782 case OMPD_parallel_master:
6783 case OMPD_parallel_sections:
6784 case OMPD_for_simd:
6785 case OMPD_parallel_for_simd:
6786 case OMPD_cancel:
6787 case OMPD_cancellation_point:
6788 case OMPD_ordered_standalone:
6789 case OMPD_ordered_blockassoc:
6790 case OMPD_threadprivate:
6791 case OMPD_allocate:
6792 case OMPD_task:
6793 case OMPD_simd:
6794 case OMPD_tile:
6795 case OMPD_unroll:
6796 case OMPD_sections:
6797 case OMPD_section:
6798 case OMPD_single:
6799 case OMPD_master:
6800 case OMPD_critical:
6801 case OMPD_taskyield:
6802 case OMPD_barrier:
6803 case OMPD_taskwait:
6804 case OMPD_taskgroup:
6805 case OMPD_atomic:
6806 case OMPD_flush:
6807 case OMPD_depobj:
6808 case OMPD_scan:
6809 case OMPD_teams:
6810 case OMPD_target_data:
6811 case OMPD_target_exit_data:
6812 case OMPD_target_enter_data:
6813 case OMPD_distribute:
6814 case OMPD_distribute_simd:
6815 case OMPD_distribute_parallel_for:
6816 case OMPD_distribute_parallel_for_simd:
6817 case OMPD_teams_distribute:
6818 case OMPD_teams_distribute_simd:
6819 case OMPD_teams_distribute_parallel_for:
6820 case OMPD_teams_distribute_parallel_for_simd:
6821 case OMPD_target_update:
6822 case OMPD_declare_simd:
6823 case OMPD_declare_variant:
6824 case OMPD_begin_declare_variant:
6825 case OMPD_end_declare_variant:
6826 case OMPD_declare_target:
6827 case OMPD_end_declare_target:
6828 case OMPD_declare_reduction:
6829 case OMPD_declare_mapper:
6830 case OMPD_taskloop:
6831 case OMPD_taskloop_simd:
6832 case OMPD_master_taskloop:
6833 case OMPD_master_taskloop_simd:
6834 case OMPD_parallel_master_taskloop:
6835 case OMPD_parallel_master_taskloop_simd:
6836 case OMPD_requires:
6837 case OMPD_metadirective:
6838 case OMPD_unknown:
6839 break;
6840 default:
6841 break;
6842 }
6843 llvm_unreachable("Unexpected directive kind.");
6844}
6845
6846llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective(
6847 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6848 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6849 "Clauses associated with the teams directive expected to be emitted "
6850 "only for the host!");
6851 CGBuilderTy &Bld = CGF.Builder;
6852 int32_t MinNT = -1, MaxNT = -1;
6853 const Expr *NumTeams =
6854 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: MinNT, MaxTeamsVal&: MaxNT);
6855 if (NumTeams != nullptr) {
6856 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6857
6858 switch (DirectiveKind) {
6859 case OMPD_target: {
6860 const auto *CS = D.getInnermostCapturedStmt();
6861 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6862 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6863 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6864 /*IgnoreResultAssign*/ true);
6865 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6866 /*isSigned=*/true);
6867 }
6868 case OMPD_target_teams:
6869 case OMPD_target_teams_distribute:
6870 case OMPD_target_teams_distribute_simd:
6871 case OMPD_target_teams_distribute_parallel_for:
6872 case OMPD_target_teams_distribute_parallel_for_simd: {
6873 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6874 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6875 /*IgnoreResultAssign*/ true);
6876 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6877 /*isSigned=*/true);
6878 }
6879 default:
6880 break;
6881 }
6882 }
6883
6884 assert(MinNT == MaxNT && "Num threads ranges require handling here.");
6885 return llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: MinNT);
6886}
6887
6888/// Merge the thread count upper bound \p Val into \p UpperBound.
6889///
6890/// \p UpperBound is -1 while no thread limiting clause has been seen, 0 once
6891/// one has been seen whose value is not known at compile time, and otherwise
6892/// the smallest constant bound found so far.
6893///
6894/// Thread limiting clauses compose by taking the minimum, so a constant bound
6895/// stays valid whatever the clauses that are not compile time constants
6896/// evaluate to. That makes it correct to replace the 0 marker with \p Val, and
6897/// necessary to keep a clause from raising a smaller bound found earlier.
6898static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val) {
6899 UpperBound = UpperBound > 0 ? std::min(a: UpperBound, b: Val) : Val;
6900}
6901
6902/// Check for a num threads constant value (stored in \p DefaultVal), or
6903/// expression (stored in \p E). If the value is conditional (via an if-clause),
6904/// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are
6905/// nullptr, no expression evaluation is perfomed.
6906static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6907 const Expr **E, int32_t &UpperBound,
6908 bool UpperBoundOnly, llvm::Value **CondVal) {
6909 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6910 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6911 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6912 if (!Dir)
6913 return;
6914
6915 if (isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6916 // Handle if clause. If if clause present, the number of threads is
6917 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6918 if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) {
6919 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6920 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6921 const OMPIfClause *IfClause = nullptr;
6922 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6923 if (C->getNameModifier() == OMPD_unknown ||
6924 C->getNameModifier() == OMPD_parallel) {
6925 IfClause = C;
6926 break;
6927 }
6928 }
6929 if (IfClause) {
6930 const Expr *CondExpr = IfClause->getCondition();
6931 bool Result;
6932 if (CondExpr->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6933 if (!Result) {
6934 UpperBound = 1;
6935 return;
6936 }
6937 } else {
6938 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());
6939 if (const auto *PreInit =
6940 cast_or_null<DeclStmt>(Val: IfClause->getPreInitStmt())) {
6941 for (const auto *I : PreInit->decls()) {
6942 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6943 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6944 } else {
6945 CodeGenFunction::AutoVarEmission Emission =
6946 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6947 CGF.EmitAutoVarCleanups(emission: Emission);
6948 }
6949 }
6950 *CondVal = CGF.EvaluateExprAsBool(E: CondExpr);
6951 }
6952 }
6953 }
6954 }
6955 // Check the value of num_threads clause iff if clause was not specified
6956 // or is not evaluated to false.
6957 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6958 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6959 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6960 const auto *NumThreadsClause =
6961 Dir->getSingleClause<OMPNumThreadsClause>();
6962 const Expr *NTExpr = NumThreadsClause->getNumThreads().front();
6963 if (NTExpr->isIntegerConstantExpr(Ctx: CGF.getContext()))
6964 if (auto Constant = NTExpr->getIntegerConstantExpr(Ctx: CGF.getContext()))
6965 mergeThreadCountUpperBound(
6966 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
6967 // If we haven't found a upper bound, remember we saw a thread limiting
6968 // clause.
6969 if (UpperBound == -1)
6970 UpperBound = 0;
6971 if (!E)
6972 return;
6973 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6974 if (const auto *PreInit =
6975 cast_or_null<DeclStmt>(Val: NumThreadsClause->getPreInitStmt())) {
6976 for (const auto *I : PreInit->decls()) {
6977 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6978 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6979 } else {
6980 CodeGenFunction::AutoVarEmission Emission =
6981 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6982 CGF.EmitAutoVarCleanups(emission: Emission);
6983 }
6984 }
6985 }
6986 *E = NTExpr;
6987 }
6988 return;
6989 }
6990 if (isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6991 UpperBound = 1;
6992}
6993
6994const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective(
6995 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6996 bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) {
6997 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6998 "Clauses associated with the teams directive expected to be emitted "
6999 "only for the host!");
7000 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
7001 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
7002 "Expected target-based executable directive.");
7003
7004 const Expr *NT = nullptr;
7005 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
7006
7007 auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) {
7008 if (E->isIntegerConstantExpr(Ctx: CGF.getContext())) {
7009 if (auto Constant = E->getIntegerConstantExpr(Ctx: CGF.getContext()))
7010 mergeThreadCountUpperBound(
7011 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
7012 }
7013 // If we haven't found a upper bound, remember we saw a thread limiting
7014 // clause.
7015 if (UpperBound == -1)
7016 UpperBound = 0;
7017 if (EPtr)
7018 *EPtr = E;
7019 };
7020
7021 auto ReturnSequential = [&]() {
7022 UpperBound = 1;
7023 return NT;
7024 };
7025
7026 switch (DirectiveKind) {
7027 case OMPD_target: {
7028 const CapturedStmt *CS = D.getInnermostCapturedStmt();
7029 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7030 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
7031 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
7032 // TODO: The standard is not clear how to resolve two thread limit clauses,
7033 // let's pick the teams one if it's present, otherwise the target one.
7034 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7035 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
7036 if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) {
7037 ThreadLimitClause = TLC;
7038 if (ThreadLimitExpr) {
7039 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
7040 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
7041 CodeGenFunction::LexicalScope Scope(
7042 CGF,
7043 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
7044 if (const auto *PreInit =
7045 cast_or_null<DeclStmt>(Val: ThreadLimitClause->getPreInitStmt())) {
7046 for (const auto *I : PreInit->decls()) {
7047 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
7048 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
7049 } else {
7050 CodeGenFunction::AutoVarEmission Emission =
7051 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
7052 CGF.EmitAutoVarCleanups(emission: Emission);
7053 }
7054 }
7055 }
7056 }
7057 }
7058 }
7059 if (ThreadLimitClause)
7060 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7061 ThreadLimitExpr);
7062 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
7063 if (isOpenMPTeamsDirective(DKind: Dir->getDirectiveKind()) &&
7064 !isOpenMPDistributeDirective(DKind: Dir->getDirectiveKind())) {
7065 CS = Dir->getInnermostCapturedStmt();
7066 // Now that the 'teams' level has been peeled off, the remainder is
7067 // shaped like a 'target teams' region, so pick up the num_threads of
7068 // the directive nested in it the same way the OMPD_target_teams case
7069 // below does. Without this the upper bound of a construct written as
7070 // 'target' / 'teams' / 'distribute parallel for' would stay at the
7071 // default, while every combined spelling of the same construct honors
7072 // the clause. Only the bound is taken here: passing null for the
7073 // expression and the condition keeps this from emitting anything, so
7074 // the value the host passes to the kernel launch is left as it was.
7075 getNumThreads(CGF, CS, /*E=*/nullptr, UpperBound, UpperBoundOnly,
7076 /*CondVal=*/nullptr);
7077 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
7078 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
7079 Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
7080 }
7081 if (Dir && isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
7082 CS = Dir->getInnermostCapturedStmt();
7083 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7084 } else if (Dir && isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
7085 return ReturnSequential();
7086 }
7087 return NT;
7088 }
7089 case OMPD_target_teams: {
7090 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7091 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7092 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7093 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7094 ThreadLimitExpr);
7095 }
7096 const CapturedStmt *CS = D.getInnermostCapturedStmt();
7097 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7098 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
7099 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
7100 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
7101 if (Dir->getDirectiveKind() == OMPD_distribute) {
7102 CS = Dir->getInnermostCapturedStmt();
7103 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
7104 }
7105 }
7106 return NT;
7107 }
7108 case OMPD_target_teams_distribute:
7109 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7110 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7111 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7112 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7113 ThreadLimitExpr);
7114 }
7115 getNumThreads(CGF, CS: D.getInnermostCapturedStmt(), E: NTPtr, UpperBound,
7116 UpperBoundOnly, CondVal);
7117 return NT;
7118 case OMPD_target_teams_loop:
7119 case OMPD_target_parallel_loop:
7120 case OMPD_target_parallel:
7121 case OMPD_target_parallel_for:
7122 case OMPD_target_parallel_for_simd:
7123 case OMPD_target_teams_distribute_parallel_for:
7124 case OMPD_target_teams_distribute_parallel_for_simd: {
7125 if (CondVal && D.hasClausesOfKind<OMPIfClause>()) {
7126 const OMPIfClause *IfClause = nullptr;
7127 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
7128 if (C->getNameModifier() == OMPD_unknown ||
7129 C->getNameModifier() == OMPD_parallel) {
7130 IfClause = C;
7131 break;
7132 }
7133 }
7134 if (IfClause) {
7135 const Expr *Cond = IfClause->getCondition();
7136 bool Result;
7137 if (Cond->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
7138 if (!Result)
7139 return ReturnSequential();
7140 } else {
7141 CodeGenFunction::RunCleanupsScope Scope(CGF);
7142 *CondVal = CGF.EvaluateExprAsBool(E: Cond);
7143 }
7144 }
7145 }
7146 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
7147 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
7148 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
7149 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
7150 ThreadLimitExpr);
7151 }
7152 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
7153 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
7154 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
7155 CheckForConstExpr(NumThreadsClause->getNumThreads().front(), nullptr);
7156 return NumThreadsClause->getNumThreads().front();
7157 }
7158 return NT;
7159 }
7160 case OMPD_target_teams_distribute_simd:
7161 case OMPD_target_simd:
7162 case OMPD_target_loop:
7163 return ReturnSequential();
7164 default:
7165 break;
7166 }
7167 llvm_unreachable("Unsupported directive kind.");
7168}
7169
7170llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective(
7171 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
7172 llvm::Value *NumThreadsVal = nullptr;
7173 llvm::Value *CondVal = nullptr;
7174 llvm::Value *ThreadLimitVal = nullptr;
7175 const Expr *ThreadLimitExpr = nullptr;
7176 int32_t UpperBound = -1;
7177
7178 const Expr *NT = getNumThreadsExprForTargetDirective(
7179 CGF, D, UpperBound, /* UpperBoundOnly */ false, CondVal: &CondVal,
7180 ThreadLimitExpr: &ThreadLimitExpr);
7181
7182 // Thread limit expressions are used below, emit them.
7183 if (ThreadLimitExpr) {
7184 ThreadLimitVal =
7185 CGF.EmitScalarExpr(E: ThreadLimitExpr, /*IgnoreResultAssign=*/true);
7186 ThreadLimitVal = CGF.Builder.CreateIntCast(V: ThreadLimitVal, DestTy: CGF.Int32Ty,
7187 /*isSigned=*/false);
7188 }
7189
7190 // Generate the num teams expression.
7191 if (UpperBound == 1) {
7192 NumThreadsVal = CGF.Builder.getInt32(C: UpperBound);
7193 } else if (NT) {
7194 NumThreadsVal = CGF.EmitScalarExpr(E: NT, /*IgnoreResultAssign=*/true);
7195 NumThreadsVal = CGF.Builder.CreateIntCast(V: NumThreadsVal, DestTy: CGF.Int32Ty,
7196 /*isSigned=*/false);
7197 } else if (ThreadLimitVal) {
7198 // If we do not have a num threads value but a thread limit, replace the
7199 // former with the latter. We know handled the thread limit expression.
7200 NumThreadsVal = ThreadLimitVal;
7201 ThreadLimitVal = nullptr;
7202 } else {
7203 // Default to "0" which means runtime choice.
7204 assert(!ThreadLimitVal && "Default not applicable with thread limit value");
7205 NumThreadsVal = CGF.Builder.getInt32(C: 0);
7206 }
7207
7208 // Handle if clause. If if clause present, the number of threads is
7209 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
7210 if (CondVal) {
7211 CodeGenFunction::RunCleanupsScope Scope(CGF);
7212 NumThreadsVal = CGF.Builder.CreateSelect(C: CondVal, True: NumThreadsVal,
7213 False: CGF.Builder.getInt32(C: 1));
7214 }
7215
7216 // If the thread limit and num teams expression were present, take the
7217 // minimum.
7218 if (ThreadLimitVal) {
7219 NumThreadsVal = CGF.Builder.CreateSelect(
7220 C: CGF.Builder.CreateICmpULT(LHS: ThreadLimitVal, RHS: NumThreadsVal),
7221 True: ThreadLimitVal, False: NumThreadsVal);
7222 }
7223
7224 return NumThreadsVal;
7225}
7226
7227namespace {
7228LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
7229
7230// Utility to handle information from clauses associated with a given
7231// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
7232// It provides a convenient interface to obtain the information and generate
7233// code for that information.
7234class MappableExprsHandler {
7235public:
7236 /// Custom comparator for attach-pointer expressions that compares them by
7237 /// complexity (i.e. their component-depth) first, then by the order in which
7238 /// they were computed by collectAttachPtrExprInfo(), if they are semantically
7239 /// different.
7240 struct AttachPtrExprComparator {
7241 const MappableExprsHandler &Handler;
7242 // Cache of previous equality comparison results.
7243 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool>
7244 CachedEqualityComparisons;
7245
7246 AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {}
7247 AttachPtrExprComparator() = delete;
7248
7249 // Return true iff LHS is "less than" RHS.
7250 bool operator()(const Expr *LHS, const Expr *RHS) const {
7251 if (LHS == RHS)
7252 return false;
7253
7254 // First, compare by complexity (depth)
7255 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(Val: LHS);
7256 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(Val: RHS);
7257
7258 std::optional<size_t> DepthLHS =
7259 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
7260 : std::nullopt;
7261 std::optional<size_t> DepthRHS =
7262 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
7263 : std::nullopt;
7264
7265 // std::nullopt (no attach pointer) has lowest complexity
7266 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
7267 // Both have same complexity, now check semantic equality
7268 if (areEqual(LHS, RHS))
7269 return false;
7270 // Different semantically, compare by computation order
7271 return wasComputedBefore(LHS, RHS);
7272 }
7273 if (!DepthLHS.has_value())
7274 return true; // LHS has lower complexity
7275 if (!DepthRHS.has_value())
7276 return false; // RHS has lower complexity
7277
7278 // Both have values, compare by depth (lower depth = lower complexity)
7279 if (DepthLHS.value() != DepthRHS.value())
7280 return DepthLHS.value() < DepthRHS.value();
7281
7282 // Same complexity, now check semantic equality
7283 if (areEqual(LHS, RHS))
7284 return false;
7285 // Different semantically, compare by computation order
7286 return wasComputedBefore(LHS, RHS);
7287 }
7288
7289 public:
7290 /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached
7291 /// results, if available, otherwise does a recursive semantic comparison.
7292 bool areEqual(const Expr *LHS, const Expr *RHS) const {
7293 // Check cache first for faster lookup
7294 const auto CachedResultIt = CachedEqualityComparisons.find(Val: {LHS, RHS});
7295 if (CachedResultIt != CachedEqualityComparisons.end())
7296 return CachedResultIt->second;
7297
7298 bool ComparisonResult = areSemanticallyEqual(LHS, RHS);
7299
7300 // Cache the result for future lookups (both orders since semantic
7301 // equality is commutative)
7302 CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult;
7303 CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult;
7304 return ComparisonResult;
7305 }
7306
7307 /// Compare the two attach-ptr expressions by their computation order.
7308 /// Returns true iff LHS was computed before RHS by
7309 /// collectAttachPtrExprInfo().
7310 bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const {
7311 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(Val: LHS);
7312 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(Val: RHS);
7313
7314 return OrderLHS < OrderRHS;
7315 }
7316
7317 private:
7318 /// Helper function to compare attach-pointer expressions semantically.
7319 /// This function handles various expression types that can be part of an
7320 /// attach-pointer.
7321 /// TODO: Not urgent, but we should ideally return true when comparing
7322 /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc.
7323 bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const {
7324 if (LHS == RHS)
7325 return true;
7326
7327 // If only one is null, they aren't equal
7328 if (!LHS || !RHS)
7329 return false;
7330
7331 ASTContext &Ctx = Handler.CGF.getContext();
7332 // Strip away parentheses and no-op casts to get to the core expression
7333 LHS = LHS->IgnoreParenNoopCasts(Ctx);
7334 RHS = RHS->IgnoreParenNoopCasts(Ctx);
7335
7336 // Direct pointer comparison of the underlying expressions
7337 if (LHS == RHS)
7338 return true;
7339
7340 // Check if the expression classes match
7341 if (LHS->getStmtClass() != RHS->getStmtClass())
7342 return false;
7343
7344 // Handle DeclRefExpr (variable references)
7345 if (const auto *LD = dyn_cast<DeclRefExpr>(Val: LHS)) {
7346 const auto *RD = dyn_cast<DeclRefExpr>(Val: RHS);
7347 if (!RD)
7348 return false;
7349 return LD->getDecl()->getCanonicalDecl() ==
7350 RD->getDecl()->getCanonicalDecl();
7351 }
7352
7353 // Handle ArraySubscriptExpr (array indexing like a[i])
7354 if (const auto *LA = dyn_cast<ArraySubscriptExpr>(Val: LHS)) {
7355 const auto *RA = dyn_cast<ArraySubscriptExpr>(Val: RHS);
7356 if (!RA)
7357 return false;
7358 return areSemanticallyEqual(LHS: LA->getBase(), RHS: RA->getBase()) &&
7359 areSemanticallyEqual(LHS: LA->getIdx(), RHS: RA->getIdx());
7360 }
7361
7362 // Handle MemberExpr (member access like s.m or p->m)
7363 if (const auto *LM = dyn_cast<MemberExpr>(Val: LHS)) {
7364 const auto *RM = dyn_cast<MemberExpr>(Val: RHS);
7365 if (!RM)
7366 return false;
7367 if (LM->getMemberDecl()->getCanonicalDecl() !=
7368 RM->getMemberDecl()->getCanonicalDecl())
7369 return false;
7370 return areSemanticallyEqual(LHS: LM->getBase(), RHS: RM->getBase());
7371 }
7372
7373 // Handle UnaryOperator (unary operations like *p, &x, etc.)
7374 if (const auto *LU = dyn_cast<UnaryOperator>(Val: LHS)) {
7375 const auto *RU = dyn_cast<UnaryOperator>(Val: RHS);
7376 if (!RU)
7377 return false;
7378 if (LU->getOpcode() != RU->getOpcode())
7379 return false;
7380 return areSemanticallyEqual(LHS: LU->getSubExpr(), RHS: RU->getSubExpr());
7381 }
7382
7383 // Handle BinaryOperator (binary operations like p + offset)
7384 if (const auto *LB = dyn_cast<BinaryOperator>(Val: LHS)) {
7385 const auto *RB = dyn_cast<BinaryOperator>(Val: RHS);
7386 if (!RB)
7387 return false;
7388 if (LB->getOpcode() != RB->getOpcode())
7389 return false;
7390 return areSemanticallyEqual(LHS: LB->getLHS(), RHS: RB->getLHS()) &&
7391 areSemanticallyEqual(LHS: LB->getRHS(), RHS: RB->getRHS());
7392 }
7393
7394 // Handle ArraySectionExpr (array sections like a[0:1])
7395 // Attach pointers should not contain array-sections, but currently we
7396 // don't emit an error.
7397 if (const auto *LAS = dyn_cast<ArraySectionExpr>(Val: LHS)) {
7398 const auto *RAS = dyn_cast<ArraySectionExpr>(Val: RHS);
7399 if (!RAS)
7400 return false;
7401 return areSemanticallyEqual(LHS: LAS->getBase(), RHS: RAS->getBase()) &&
7402 areSemanticallyEqual(LHS: LAS->getLowerBound(),
7403 RHS: RAS->getLowerBound()) &&
7404 areSemanticallyEqual(LHS: LAS->getLength(), RHS: RAS->getLength());
7405 }
7406
7407 // Handle CastExpr (explicit casts)
7408 if (const auto *LC = dyn_cast<CastExpr>(Val: LHS)) {
7409 const auto *RC = dyn_cast<CastExpr>(Val: RHS);
7410 if (!RC)
7411 return false;
7412 if (LC->getCastKind() != RC->getCastKind())
7413 return false;
7414 return areSemanticallyEqual(LHS: LC->getSubExpr(), RHS: RC->getSubExpr());
7415 }
7416
7417 // Handle CXXThisExpr (this pointer)
7418 if (isa<CXXThisExpr>(Val: LHS) && isa<CXXThisExpr>(Val: RHS))
7419 return true;
7420
7421 // Handle IntegerLiteral (integer constants)
7422 if (const auto *LI = dyn_cast<IntegerLiteral>(Val: LHS)) {
7423 const auto *RI = dyn_cast<IntegerLiteral>(Val: RHS);
7424 if (!RI)
7425 return false;
7426 return LI->getValue() == RI->getValue();
7427 }
7428
7429 // Handle CharacterLiteral (character constants)
7430 if (const auto *LC = dyn_cast<CharacterLiteral>(Val: LHS)) {
7431 const auto *RC = dyn_cast<CharacterLiteral>(Val: RHS);
7432 if (!RC)
7433 return false;
7434 return LC->getValue() == RC->getValue();
7435 }
7436
7437 // Handle FloatingLiteral (floating point constants)
7438 if (const auto *LF = dyn_cast<FloatingLiteral>(Val: LHS)) {
7439 const auto *RF = dyn_cast<FloatingLiteral>(Val: RHS);
7440 if (!RF)
7441 return false;
7442 // Use bitwise comparison for floating point literals
7443 return LF->getValue().bitwiseIsEqual(RHS: RF->getValue());
7444 }
7445
7446 // Handle StringLiteral (string constants)
7447 if (const auto *LS = dyn_cast<StringLiteral>(Val: LHS)) {
7448 const auto *RS = dyn_cast<StringLiteral>(Val: RHS);
7449 if (!RS)
7450 return false;
7451 return LS->getString() == RS->getString();
7452 }
7453
7454 // Handle CXXNullPtrLiteralExpr (nullptr)
7455 if (isa<CXXNullPtrLiteralExpr>(Val: LHS) && isa<CXXNullPtrLiteralExpr>(Val: RHS))
7456 return true;
7457
7458 // Handle CXXBoolLiteralExpr (true/false)
7459 if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(Val: LHS)) {
7460 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(Val: RHS);
7461 if (!RB)
7462 return false;
7463 return LB->getValue() == RB->getValue();
7464 }
7465
7466 // Fallback for other forms - use the existing comparison method
7467 return Expr::isSameComparisonOperand(E1: LHS, E2: RHS);
7468 }
7469 };
7470
7471 /// Get the offset of the OMP_MAP_MEMBER_OF field.
7472 static unsigned getFlagMemberOffset() {
7473 unsigned Offset = 0;
7474 for (uint64_t Remain =
7475 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
7476 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7477 !(Remain & 1); Remain = Remain >> 1)
7478 Offset++;
7479 return Offset;
7480 }
7481
7482 /// Class that holds debugging information for a data mapping to be passed to
7483 /// the runtime library.
7484 class MappingExprInfo {
7485 /// The variable declaration used for the data mapping.
7486 const ValueDecl *MapDecl = nullptr;
7487 /// The original expression used in the map clause, or null if there is
7488 /// none.
7489 const Expr *MapExpr = nullptr;
7490
7491 public:
7492 MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr)
7493 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7494
7495 const ValueDecl *getMapDecl() const { return MapDecl; }
7496 const Expr *getMapExpr() const { return MapExpr; }
7497 };
7498
7499 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7500 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7501 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7502 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7503 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7504 using MapNonContiguousArrayTy =
7505 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7506 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7507 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7508 using MapData =
7509 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7510 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,
7511 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;
7512 using MapDataArrayTy = SmallVector<MapData, 4>;
7513
7514 /// This structure contains combined information generated for mappable
7515 /// clauses, including base pointers, pointers, sizes, map types, user-defined
7516 /// mappers, and non-contiguous information.
7517 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7518 MapExprsArrayTy Exprs;
7519 MapValueDeclsArrayTy Mappers;
7520 MapValueDeclsArrayTy DevicePtrDecls;
7521
7522 /// Append arrays in \a CurInfo.
7523 void append(MapCombinedInfoTy &CurInfo) {
7524 Exprs.append(in_start: CurInfo.Exprs.begin(), in_end: CurInfo.Exprs.end());
7525 DevicePtrDecls.append(in_start: CurInfo.DevicePtrDecls.begin(),
7526 in_end: CurInfo.DevicePtrDecls.end());
7527 Mappers.append(in_start: CurInfo.Mappers.begin(), in_end: CurInfo.Mappers.end());
7528 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7529 }
7530 };
7531
7532 /// Map between a struct and the its lowest & highest elements which have been
7533 /// mapped.
7534 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7535 /// HE(FieldIndex, Pointer)}
7536 struct StructRangeInfoTy {
7537 MapCombinedInfoTy PreliminaryMapData;
7538 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7539 0, Address::invalid()};
7540 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7541 0, Address::invalid()};
7542 Address Base = Address::invalid();
7543 Address LB = Address::invalid();
7544 bool IsArraySection = false;
7545 bool HasCompleteRecord = false;
7546 };
7547
7548 /// A struct to store the attach pointer and pointee information, to be used
7549 /// when emitting an attach entry.
7550 struct AttachInfoTy {
7551 Address AttachPtrAddr = Address::invalid();
7552 Address AttachPteeAddr = Address::invalid();
7553 const ValueDecl *AttachPtrDecl = nullptr;
7554 const Expr *AttachMapExpr = nullptr;
7555
7556 bool isValid() const {
7557 return AttachPtrAddr.isValid() && AttachPteeAddr.isValid();
7558 }
7559 };
7560
7561 /// Check if there's any component list where the attach pointer expression
7562 /// matches the given captured variable.
7563 bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const {
7564 for (const auto &AttachEntry : AttachPtrExprMap) {
7565 if (AttachEntry.second) {
7566 // Check if the attach pointer expression is a DeclRefExpr that
7567 // references the captured variable
7568 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: AttachEntry.second))
7569 if (DRE->getDecl() == VD)
7570 return true;
7571 }
7572 }
7573 return false;
7574 }
7575
7576 /// Get the previously-cached attach pointer for a component list, if-any.
7577 const Expr *getAttachPtrExpr(
7578 OMPClauseMappableExprCommon::MappableExprComponentListRef Components)
7579 const {
7580 const auto It = AttachPtrExprMap.find(Val: Components);
7581 if (It != AttachPtrExprMap.end())
7582 return It->second;
7583
7584 return nullptr;
7585 }
7586
7587private:
7588 /// Kind that defines how a device pointer has to be returned.
7589 struct MapInfo {
7590 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7591 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
7592 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7593 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7594 bool ReturnDevicePointer = false;
7595 bool IsImplicit = false;
7596 const ValueDecl *Mapper = nullptr;
7597 const Expr *VarRef = nullptr;
7598 bool ForDeviceAddr = false;
7599 bool HasUdpFbNullify = false;
7600
7601 MapInfo() = default;
7602 MapInfo(
7603 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7604 OpenMPMapClauseKind MapType,
7605 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7606 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7607 bool ReturnDevicePointer, bool IsImplicit,
7608 const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr,
7609 bool ForDeviceAddr = false, bool HasUdpFbNullify = false)
7610 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7611 MotionModifiers(MotionModifiers),
7612 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7613 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr),
7614 HasUdpFbNullify(HasUdpFbNullify) {}
7615 };
7616
7617 /// The target directive from where the mappable clauses were extracted. It
7618 /// is either a executable directive or a user-defined mapper directive.
7619 llvm::PointerUnion<const OMPExecutableDirective *,
7620 const OMPDeclareMapperDecl *>
7621 CurDir;
7622
7623 /// Function the directive is being generated for.
7624 CodeGenFunction &CGF;
7625
7626 /// Set of all first private variables in the current directive.
7627 /// bool data is set to true if the variable is implicitly marked as
7628 /// firstprivate, false otherwise.
7629 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7630
7631 /// Set of defaultmap clause kinds that use firstprivate behavior.
7632 llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
7633
7634 /// Map between device pointer declarations and their expression components.
7635 /// The key value for declarations in 'this' is null.
7636 llvm::DenseMap<
7637 const ValueDecl *,
7638 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7639 DevPointersMap;
7640
7641 /// Map between device addr declarations and their expression components.
7642 /// The key value for declarations in 'this' is null.
7643 llvm::DenseMap<
7644 const ValueDecl *,
7645 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7646 HasDevAddrsMap;
7647
7648 /// Map between lambda declarations and their map type.
7649 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7650
7651 /// Map from component lists to their attach pointer expressions.
7652 llvm::DenseMap<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7653 const Expr *>
7654 AttachPtrExprMap;
7655
7656 /// Map from attach pointer expressions to their component depth.
7657 /// nullptr key has std::nullopt depth. This can be used to order attach-ptr
7658 /// expressions with increasing/decreasing depth.
7659 /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`.
7660 /// TODO: Not urgent, but we should ideally use the number of pointer
7661 /// dereferences in an expr as an indicator of its complexity, instead of the
7662 /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`,
7663 /// `*(p + 5 + 5)` together.
7664 llvm::DenseMap<const Expr *, std::optional<size_t>>
7665 AttachPtrComponentDepthMap = {{nullptr, std::nullopt}};
7666
7667 /// Map from attach pointer expressions to the order they were computed in, in
7668 /// collectAttachPtrExprInfo().
7669 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7670 {nullptr, 0}};
7671
7672 /// An instance of attach-ptr-expr comparator that can be used throughout the
7673 /// lifetime of this handler.
7674 AttachPtrExprComparator AttachPtrComparator;
7675
7676 llvm::Value *getExprTypeSize(const Expr *E) const {
7677 QualType ExprTy = E->getType().getCanonicalType();
7678
7679 // Calculate the size for array shaping expression.
7680 if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(Val: E)) {
7681 llvm::Value *Size =
7682 CGF.getTypeSize(Ty: OAE->getBase()->getType()->getPointeeType());
7683 for (const Expr *SE : OAE->getDimensions()) {
7684 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
7685 Sz = CGF.EmitScalarConversion(Src: Sz, SrcTy: SE->getType(),
7686 DstTy: CGF.getContext().getSizeType(),
7687 Loc: SE->getExprLoc());
7688 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: Sz);
7689 }
7690 return Size;
7691 }
7692
7693 // Reference types are ignored for mapping purposes.
7694 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7695 ExprTy = RefTy->getPointeeType().getCanonicalType();
7696
7697 // Given that an array section is considered a built-in type, we need to
7698 // do the calculation based on the length of the section instead of relying
7699 // on CGF.getTypeSize(E->getType()).
7700 if (const auto *OAE = dyn_cast<ArraySectionExpr>(Val: E)) {
7701 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(
7702 Base: OAE->getBase()->IgnoreParenImpCasts())
7703 .getCanonicalType();
7704
7705 // If there is no length associated with the expression and lower bound is
7706 // not specified too, that means we are using the whole length of the
7707 // base.
7708 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7709 !OAE->getLowerBound())
7710 return CGF.getTypeSize(Ty: BaseTy);
7711
7712 llvm::Value *ElemSize;
7713 if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7714 ElemSize = CGF.getTypeSize(Ty: PTy->getPointeeType().getCanonicalType());
7715 } else {
7716 const auto *ATy = cast<ArrayType>(Val: BaseTy.getTypePtr());
7717 assert(ATy && "Expecting array type if not a pointer type.");
7718 ElemSize = CGF.getTypeSize(Ty: ATy->getElementType().getCanonicalType());
7719 }
7720
7721 // If we don't have a length at this point, that is because we have an
7722 // array section with a single element.
7723 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7724 return ElemSize;
7725
7726 if (const Expr *LenExpr = OAE->getLength()) {
7727 llvm::Value *LengthVal = CGF.EmitScalarExpr(E: LenExpr);
7728 LengthVal = CGF.EmitScalarConversion(Src: LengthVal, SrcTy: LenExpr->getType(),
7729 DstTy: CGF.getContext().getSizeType(),
7730 Loc: LenExpr->getExprLoc());
7731 return CGF.Builder.CreateNUWMul(LHS: LengthVal, RHS: ElemSize);
7732 }
7733 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7734 OAE->getLowerBound() && "expected array_section[lb:].");
7735 // Size = sizetype - lb * elemtype;
7736 llvm::Value *LengthVal = CGF.getTypeSize(Ty: BaseTy);
7737 llvm::Value *LBVal = CGF.EmitScalarExpr(E: OAE->getLowerBound());
7738 LBVal = CGF.EmitScalarConversion(Src: LBVal, SrcTy: OAE->getLowerBound()->getType(),
7739 DstTy: CGF.getContext().getSizeType(),
7740 Loc: OAE->getLowerBound()->getExprLoc());
7741 LBVal = CGF.Builder.CreateNUWMul(LHS: LBVal, RHS: ElemSize);
7742 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LHS: LengthVal, RHS: LBVal);
7743 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LHS: LengthVal, RHS: LBVal);
7744 LengthVal = CGF.Builder.CreateSelect(
7745 C: Cmp, True: TrueVal, False: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0));
7746 return LengthVal;
7747 }
7748 return CGF.getTypeSize(Ty: ExprTy);
7749 }
7750
7751 /// Return the corresponding bits for a given map clause modifier. Add
7752 /// a flag marking the map as a pointer if requested. Add a flag marking the
7753 /// map as the first one of a series of maps that relate to the same map
7754 /// expression.
7755 OpenMPOffloadMappingFlags getMapTypeBits(
7756 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7757 ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,
7758 bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const {
7759 OpenMPOffloadMappingFlags Bits =
7760 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7761 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7762 switch (MapType) {
7763 case OMPC_MAP_alloc:
7764 case OMPC_MAP_release:
7765 // alloc and release is the default behavior in the runtime library, i.e.
7766 // if we don't pass any bits alloc/release that is what the runtime is
7767 // going to do. Therefore, we don't need to signal anything for these two
7768 // type modifiers.
7769 break;
7770 case OMPC_MAP_to:
7771 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7772 break;
7773 case OMPC_MAP_from:
7774 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7775 break;
7776 case OMPC_MAP_tofrom:
7777 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7778 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7779 break;
7780 case OMPC_MAP_delete:
7781 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7782 break;
7783 case OMPC_MAP_unknown:
7784 llvm_unreachable("Unexpected map type!");
7785 }
7786 if (AddPtrFlag)
7787 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7788 if (AddIsTargetParamFlag)
7789 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7790 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_always))
7791 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7792 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_close))
7793 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7794 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_present) ||
7795 llvm::is_contained(Range&: MotionModifiers, Element: OMPC_MOTION_MODIFIER_present))
7796 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7797 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_ompx_hold))
7798 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7799 if (IsNonContiguous)
7800 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7801 return Bits;
7802 }
7803
7804 /// Return true if the provided expression is a final array section. A
7805 /// final array section, is one whose length can't be proved to be one.
7806 bool isFinalArraySectionExpression(const Expr *E) const {
7807 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E);
7808
7809 // It is not an array section and therefore not a unity-size one.
7810 if (!OASE)
7811 return false;
7812
7813 // An array section with no colon always refer to a single element.
7814 if (OASE->getColonLocFirst().isInvalid())
7815 return false;
7816
7817 const Expr *Length = OASE->getLength();
7818
7819 // If we don't have a length we have to check if the array has size 1
7820 // for this dimension. Also, we should always expect a length if the
7821 // base type is pointer.
7822 if (!Length) {
7823 QualType BaseQTy = ArraySectionExpr::getBaseOriginalType(
7824 Base: OASE->getBase()->IgnoreParenImpCasts())
7825 .getCanonicalType();
7826 if (const auto *ATy = dyn_cast<ConstantArrayType>(Val: BaseQTy.getTypePtr()))
7827 return ATy->getSExtSize() != 1;
7828 // If we don't have a constant dimension length, we have to consider
7829 // the current section as having any size, so it is not necessarily
7830 // unitary. If it happen to be unity size, that's user fault.
7831 return true;
7832 }
7833
7834 // Check if the length evaluates to 1.
7835 Expr::EvalResult Result;
7836 if (!Length->EvaluateAsInt(Result, Ctx: CGF.getContext()))
7837 return true; // Can have more that size 1.
7838
7839 llvm::APSInt ConstLength = Result.Val.getInt();
7840 return ConstLength.getSExtValue() != 1;
7841 }
7842
7843 /// Emit an attach entry into \p CombinedInfo, using the information from \p
7844 /// AttachInfo. For example, for a map of form `int *p; ... map(p[1:10])`,
7845 /// an attach entry has the following form:
7846 /// &p, &p[1], sizeof(void*), ATTACH
7847 void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo,
7848 const AttachInfoTy &AttachInfo) const {
7849 assert(AttachInfo.isValid() &&
7850 "Expected valid attach pointer/pointee information!");
7851
7852 // Size is the size of the pointer itself - use pointer size, not BaseDecl
7853 // size
7854 llvm::Value *PointerSize = CGF.Builder.CreateIntCast(
7855 V: llvm::ConstantInt::get(
7856 Ty: CGF.CGM.SizeTy, V: CGF.getContext()
7857 .getTypeSizeInChars(T: CGF.getContext().VoidPtrTy)
7858 .getQuantity()),
7859 DestTy: CGF.Int64Ty, /*isSigned=*/true);
7860
7861 CombinedInfo.Exprs.emplace_back(Args: AttachInfo.AttachPtrDecl,
7862 Args: AttachInfo.AttachMapExpr);
7863 CombinedInfo.BasePointers.push_back(
7864 Elt: AttachInfo.AttachPtrAddr.emitRawPointer(CGF));
7865 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7866 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7867 CombinedInfo.Pointers.push_back(
7868 Elt: AttachInfo.AttachPteeAddr.emitRawPointer(CGF));
7869 CombinedInfo.Sizes.push_back(Elt: PointerSize);
7870 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
7871 // ATTACH entries themselves don't "have" a base attach-ptr.
7872 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7873 CombinedInfo.Mappers.push_back(Elt: nullptr);
7874 CombinedInfo.NonContigInfo.Dims.push_back(Elt: 1);
7875 }
7876
7877 /// A helper class to copy structures with overlapped elements, i.e. those
7878 /// which have mappings of both "s" and "s.mem". Consecutive elements that
7879 /// are not explicitly copied have mapping nodes synthesized for them,
7880 /// taking care to avoid generating zero-sized copies.
7881 class CopyOverlappedEntryGaps {
7882 CodeGenFunction &CGF;
7883 MapCombinedInfoTy &CombinedInfo;
7884 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7885 const ValueDecl *MapDecl = nullptr;
7886 const Expr *MapExpr = nullptr;
7887 Address BP = Address::invalid();
7888 bool IsNonContiguous = false;
7889 uint64_t DimSize = 0;
7890 // These elements track the position as the struct is iterated over
7891 // (in order of increasing element address).
7892 const RecordDecl *LastParent = nullptr;
7893 uint64_t Cursor = 0;
7894 unsigned LastIndex = -1u;
7895 Address LB = Address::invalid();
7896
7897 public:
7898 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7899 MapCombinedInfoTy &CombinedInfo,
7900 OpenMPOffloadMappingFlags Flags,
7901 const ValueDecl *MapDecl, const Expr *MapExpr,
7902 Address BP, Address LB, bool IsNonContiguous,
7903 uint64_t DimSize)
7904 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7905 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7906 DimSize(DimSize), LB(LB) {}
7907
7908 void processField(
7909 const OMPClauseMappableExprCommon::MappableComponent &MC,
7910 const FieldDecl *FD,
7911 llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)>
7912 EmitMemberExprBase) {
7913 const RecordDecl *RD = FD->getParent();
7914 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(D: RD);
7915 uint64_t FieldOffset = RL.getFieldOffset(FieldNo: FD->getFieldIndex());
7916 uint64_t FieldSize =
7917 CGF.getContext().getTypeSize(T: FD->getType().getCanonicalType());
7918 Address ComponentLB = Address::invalid();
7919
7920 if (FD->getType()->isLValueReferenceType()) {
7921 const auto *ME = cast<MemberExpr>(Val: MC.getAssociatedExpression());
7922 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7923 ComponentLB =
7924 CGF.EmitLValueForFieldInitialization(Base: BaseLVal, Field: FD).getAddress();
7925 } else {
7926 ComponentLB =
7927 CGF.EmitOMPSharedLValue(E: MC.getAssociatedExpression()).getAddress();
7928 }
7929
7930 if (!LastParent)
7931 LastParent = RD;
7932 if (FD->getParent() == LastParent) {
7933 if (FD->getFieldIndex() != LastIndex + 1)
7934 copyUntilField(FD, ComponentLB);
7935 } else {
7936 LastParent = FD->getParent();
7937 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7938 copyUntilField(FD, ComponentLB);
7939 }
7940 Cursor = FieldOffset + FieldSize;
7941 LastIndex = FD->getFieldIndex();
7942 LB = CGF.Builder.CreateConstGEP(Addr: ComponentLB, Index: 1);
7943 }
7944
7945 void copyUntilField(const FieldDecl *FD, Address ComponentLB) {
7946 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7947 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7948 llvm::Value *Size = CGF.Builder.CreatePtrDiff(LHS: ComponentLBPtr, RHS: LBPtr);
7949 copySizedChunk(Base: LBPtr, Size);
7950 }
7951
7952 void copyUntilEnd(Address HB) {
7953 if (LastParent) {
7954 const ASTRecordLayout &RL =
7955 CGF.getContext().getASTRecordLayout(D: LastParent);
7956 if ((uint64_t)CGF.getContext().toBits(CharSize: RL.getSize()) <= Cursor)
7957 return;
7958 }
7959 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7960 llvm::Value *Size = CGF.Builder.CreatePtrDiff(
7961 LHS: CGF.Builder.CreateConstGEP(Addr: HB, Index: 1).emitRawPointer(CGF), RHS: LBPtr);
7962 copySizedChunk(Base: LBPtr, Size);
7963 }
7964
7965 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7966 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
7967 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
7968 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7969 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7970 CombinedInfo.Pointers.push_back(Elt: Base);
7971 CombinedInfo.Sizes.push_back(
7972 Elt: CGF.Builder.CreateIntCast(V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/false));
7973 CombinedInfo.Types.push_back(Elt: Flags);
7974 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7975 CombinedInfo.Mappers.push_back(Elt: nullptr);
7976 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize : 1);
7977 }
7978 };
7979
7980 /// Generate the base pointers, section pointers, sizes, map type bits, and
7981 /// user-defined mappers (all included in \a CombinedInfo) for the provided
7982 /// map type, map or motion modifiers, and expression components.
7983 /// \a IsFirstComponent should be set to true if the provided set of
7984 /// components is the first associated with a capture.
7985 void generateInfoForComponentList(
7986 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7987 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7988 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7989 MapCombinedInfoTy &CombinedInfo,
7990 MapCombinedInfoTy &StructBaseCombinedInfo,
7991 StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo,
7992 bool IsFirstComponentList, bool IsImplicit,
7993 bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr,
7994 bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr,
7995 const Expr *MapExpr = nullptr,
7996 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7997 OverlappedElements = {}) const {
7998
7999 // The following summarizes what has to be generated for each map and the
8000 // types below. The generated information is expressed in this order:
8001 // base pointer, section pointer, size, flags
8002 // (to add to the ones that come from the map type and modifier).
8003 // Entries annotated with (+) are only generated for "target" constructs,
8004 // and only if the variable at the beginning of the expression is used in
8005 // the region.
8006 //
8007 // double d;
8008 // int i[100];
8009 // float *p;
8010 // int **a = &i;
8011 //
8012 // struct S1 {
8013 // int i;
8014 // float f[50];
8015 // }
8016 // struct S2 {
8017 // int i;
8018 // float f[50];
8019 // S1 s;
8020 // double *p;
8021 // double *&pref;
8022 // struct S2 *ps;
8023 // int &ref;
8024 // }
8025 // S2 s;
8026 // S2 *ps;
8027 //
8028 // map(d)
8029 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
8030 //
8031 // map(i)
8032 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
8033 //
8034 // map(i[1:23])
8035 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
8036 //
8037 // map(p)
8038 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
8039 //
8040 // map(p[1:24])
8041 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM // map pointee
8042 // &p, &p[1], sizeof(void*), ATTACH // attach pointer/pointee, if both
8043 // // are present, and either is new
8044 //
8045 // map(([22])p)
8046 // p, p, 22*sizeof(float), TARGET_PARAM | TO | FROM
8047 // &p, p, sizeof(void*), ATTACH
8048 //
8049 // map((*a)[0:3])
8050 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
8051 // (*a)[0], &(*a)[0], 3 * sizeof(int), TO | FROM
8052 // &(*a), &(*a)[0], sizeof(void*), ATTACH
8053 // (+) Only on target, if a is used in the region
8054 // Note: Since the attach base-pointer is `*a`, which is not a scalar
8055 // variable, it doesn't determine the clause on `a`. `a` is mapped using
8056 // a zero-length-array-section map by generateDefaultMapInfo, if it is
8057 // referenced in the target region, because it is a pointer.
8058 //
8059 // map(**a)
8060 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
8061 // &(*a)[0], &(*a)[0], sizeof(int), TO | FROM
8062 // &(*a), &(*a)[0], sizeof(void*), ATTACH
8063 // (+) Only on target, if a is used in the region
8064 //
8065 // map(s)
8066 // FIXME: This needs to also imply map(ref_ptr_ptee: s.ref), since the
8067 // effect is supposed to be same as if the user had a map for every element
8068 // of the struct. We currently do a shallow-map of s.
8069 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
8070 //
8071 // map(s.i)
8072 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
8073 //
8074 // map(s.s.f)
8075 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
8076 //
8077 // map(s.p)
8078 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
8079 //
8080 // map(to: s.p[:22])
8081 // &s, &(s.p), sizeof(double*), TARGET_PARAM | IMPLICIT // (+)
8082 // &(s.p[0]), &(s.p[0]), 22 * sizeof(double*), TO | FROM
8083 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8084 //
8085 // map(to: s.ref)
8086 // &s, &(ptr(s.ref)), sizeof(int*), TARGET_PARAM (*)
8087 // &s, &(ptee(s.ref)), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
8088 // (*) alloc space for struct members, only this is a target parameter.
8089 // (**) map the pointer (nothing to be mapped in this example) (the compiler
8090 // optimizes this entry out, same in the examples below)
8091 // (***) map the pointee (map: to)
8092 // Note: ptr(s.ref) represents the referring pointer of s.ref
8093 // ptee(s.ref) represents the referenced pointee of s.ref
8094 //
8095 // map(to: s.pref)
8096 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM
8097 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO
8098 //
8099 // map(to: s.pref[:22])
8100 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | IMPLICIT // (+)
8101 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO |
8102 // FROM | IMPLICIT // (+)
8103 // &(ptee(s.pref)[0]), &(ptee(s.pref)[0]), 22 * sizeof(double), TO
8104 // &(ptee(s.pref)), &(ptee(s.pref)[0]), sizeof(void*), ATTACH
8105 //
8106 // map(s.ps)
8107 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
8108 //
8109 // map(from: s.ps->s.i)
8110 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8111 // &(s.ps[0]), &(s.ps->s.i), sizeof(int), FROM
8112 // &(s.ps), &(s.ps->s.i), sizeof(void*), ATTACH
8113 //
8114 // map(to: s.ps->ps)
8115 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8116 // &(s.ps[0]), &(s.ps->ps), sizeof(S2*), TO
8117 // &(s.ps), &(s.ps->ps), sizeof(void*), ATTACH
8118 //
8119 // map(s.ps->ps->ps)
8120 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8121 // &(s.ps->ps[0]), &(s.ps->ps->ps), sizeof(S2*), TO
8122 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(void*), ATTACH
8123 //
8124 // map(to: s.ps->ps->s.f[:22])
8125 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
8126 // &(s.ps->ps[0]), &(s.ps->ps->s.f[0]), 22*sizeof(float), TO
8127 // &(s.ps->ps), &(s.ps->ps->s.f[0]), sizeof(void*), ATTACH
8128 //
8129 // map(ps)
8130 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
8131 //
8132 // map(ps->i)
8133 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
8134 // &ps, &(ps->i), sizeof(void*), ATTACH
8135 //
8136 // map(ps->s.f)
8137 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
8138 // &ps, &(ps->s.f[0]), sizeof(ps), ATTACH
8139 //
8140 // map(from: ps->p)
8141 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
8142 // &ps, &(ps->p), sizeof(ps), ATTACH
8143 //
8144 // map(to: ps->p[:22])
8145 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
8146 // &(ps->p[0]), &(ps->p[0]), 22*sizeof(double), TO
8147 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8148 //
8149 // map(ps->ps)
8150 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
8151 // &ps, &(ps->ps), sizeof(ps), ATTACH
8152 //
8153 // map(from: ps->ps->s.i)
8154 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
8155 // &(ps->ps[0]), &(ps->ps->s.i), sizeof(int), FROM
8156 // &(ps->ps), &(ps->ps->s.i), sizeof(void*), ATTACH
8157 //
8158 // map(from: ps->ps->ps)
8159 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8160 // &(ps->ps[0]), &(ps->ps->ps), sizeof(S2*), FROM
8161 // &(ps->ps), &(ps->ps->ps), sizeof(void*), ATTACH
8162 //
8163 // map(ps->ps->ps->ps)
8164 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8165 // &(ps->ps->ps[0]), &(ps->ps->ps->ps), sizeof(S2*), FROM
8166 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(void*), ATTACH
8167 //
8168 // map(to: ps->ps->ps->s.f[:22])
8169 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8170 // &(ps->ps->ps[0]), &(ps->ps->ps->s.f[0]), 22*sizeof(float), TO
8171 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), sizeof(void*), ATTACH
8172 //
8173 // map(to: s.f[:22]) map(from: s.p[:33])
8174 // On target, and if s is used in the region:
8175 //
8176 // &s, &(s.f[0]), 50*sizeof(float) +
8177 // sizeof(struct S1) +
8178 // sizeof(double*) (**), TARGET_PARAM
8179 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
8180 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) | TO |
8181 // FROM | IMPLICIT
8182 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
8183 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8184 // (**) allocate contiguous space needed to fit all mapped members even if
8185 // we allocate space for members not mapped (in this example,
8186 // s.f[22..49] and s.s are not mapped, yet we must allocate space for
8187 // them as well because they fall between &s.f[0] and &s.p)
8188 //
8189 // On other constructs, and, if s is not used in the region, on target:
8190 // &s, &(s.f[0]), 22*sizeof(float), TO
8191 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
8192 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
8193 //
8194 // map(from: s.f[:22]) map(to: ps->p[:33])
8195 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
8196 // &ps[0], &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8197 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
8198 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8199 //
8200 // map(from: s.f[:22], s.s) map(to: ps->p[:33])
8201 // &s, &(s.f[0]), 50*sizeof(float) +
8202 // sizeof(struct S1), TARGET_PARAM
8203 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
8204 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
8205 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
8206 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
8207 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
8208 //
8209 // map(p[:100], p)
8210 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
8211 // p, &p[0], 100*sizeof(float), TO | FROM
8212 // &p, &p[0], sizeof(float*), ATTACH
8213
8214 // Track if the map information being generated is the first for a capture.
8215 bool IsCaptureFirstInfo = IsFirstComponentList;
8216 // When the variable is on a declare target link or in a to clause with
8217 // unified memory, a reference is needed to hold the host/device address
8218 // of the variable.
8219 bool RequiresReference = false;
8220
8221 // Scan the components from the base to the complete expression.
8222 auto CI = Components.rbegin();
8223 auto CE = Components.rend();
8224 auto I = CI;
8225
8226 // Track if the map information being generated is the first for a list of
8227 // components.
8228 bool IsExpressionFirstInfo = true;
8229 bool FirstPointerInComplexData = false;
8230 Address BP = Address::invalid();
8231 Address FinalLowestElem = Address::invalid();
8232 const Expr *AssocExpr = I->getAssociatedExpression();
8233 const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr);
8234 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8235 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(Val: AssocExpr);
8236
8237 // Get the pointer-attachment base-pointer for the given list, if any.
8238 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
8239 auto [AttachPtrAddr, AttachPteeBaseAddr] =
8240 getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF);
8241
8242 bool HasAttachPtr = AttachPtrExpr != nullptr;
8243 bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr;
8244 bool SeenAttachPtr = FirstComponentIsForAttachPtr;
8245
8246 if (FirstComponentIsForAttachPtr) {
8247 // No need to process AttachPtr here. It will be processed at the end
8248 // after we have computed the pointee's address.
8249 ++I;
8250 } else if (isa<MemberExpr>(Val: AssocExpr)) {
8251 // The base is the 'this' pointer. The content of the pointer is going
8252 // to be the base of the field being mapped.
8253 BP = CGF.LoadCXXThisAddress();
8254 } else if ((AE && isa<CXXThisExpr>(Val: AE->getBase()->IgnoreParenImpCasts())) ||
8255 (OASE &&
8256 isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))) {
8257 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8258 } else if (OAShE &&
8259 isa<CXXThisExpr>(Val: OAShE->getBase()->IgnoreParenCasts())) {
8260 BP = Address(
8261 CGF.EmitScalarExpr(E: OAShE->getBase()),
8262 CGF.ConvertTypeForMem(T: OAShE->getBase()->getType()->getPointeeType()),
8263 CGF.getContext().getTypeAlignInChars(T: OAShE->getBase()->getType()));
8264 } else {
8265 // The base is the reference to the variable.
8266 // BP = &Var.
8267 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8268 if (const auto *VD =
8269 dyn_cast_or_null<VarDecl>(Val: I->getAssociatedDeclaration())) {
8270 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8271 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
8272 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
8273 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
8274 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
8275 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
8276 RequiresReference = true;
8277 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
8278 }
8279 }
8280 }
8281
8282 // If the variable is a pointer and is being dereferenced (i.e. is not
8283 // the last component), the base has to be the pointer itself, not its
8284 // reference. References are ignored for mapping purposes.
8285 QualType Ty =
8286 I->getAssociatedDeclaration()->getType().getNonReferenceType();
8287 if (Ty->isAnyPointerType() && std::next(x: I) != CE) {
8288 // No need to generate individual map information for the pointer, it
8289 // can be associated with the combined storage if shared memory mode is
8290 // active or the base declaration is not global variable.
8291 const auto *VD = dyn_cast<VarDecl>(Val: I->getAssociatedDeclaration());
8292 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8293 !VD || VD->hasLocalStorage() || HasAttachPtr)
8294 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8295 else
8296 FirstPointerInComplexData = true;
8297 ++I;
8298 }
8299 }
8300
8301 // Track whether a component of the list should be marked as MEMBER_OF some
8302 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
8303 // in a component list should be marked as MEMBER_OF, all subsequent entries
8304 // do not belong to the base struct. E.g.
8305 // struct S2 s;
8306 // s.ps->ps->ps->f[:]
8307 // (1) (2) (3) (4)
8308 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
8309 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
8310 // is the pointee of ps(2) which is not member of struct s, so it should not
8311 // be marked as such (it is still PTR_AND_OBJ).
8312 // The variable is initialized to false so that PTR_AND_OBJ entries which
8313 // are not struct members are not considered (e.g. array of pointers to
8314 // data).
8315 bool ShouldBeMemberOf = false;
8316
8317 // Variable keeping track of whether or not we have encountered a component
8318 // in the component list which is a member expression. Useful when we have a
8319 // pointer or a final array section, in which case it is the previous
8320 // component in the list which tells us whether we have a member expression.
8321 // E.g. X.f[:]
8322 // While processing the final array section "[:]" it is "f" which tells us
8323 // whether we are dealing with a member of a declared struct.
8324 const MemberExpr *EncounteredME = nullptr;
8325
8326 // Track for the total number of dimension. Start from one for the dummy
8327 // dimension.
8328 uint64_t DimSize = 1;
8329
8330 // Detects non-contiguous updates due to strided accesses.
8331 // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set
8332 // correctly when generating information to be passed to the runtime. The
8333 // flag is set to true if any array section has a stride not equal to 1, or
8334 // if the stride is not a constant expression (conservatively assumed
8335 // non-contiguous).
8336 bool IsNonContiguous =
8337 CombinedInfo.NonContigInfo.IsNonContiguous ||
8338 any_of(Range&: Components, P: [&](const auto &Component) {
8339 const auto *OASE =
8340 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
8341 if (!OASE)
8342 return false;
8343
8344 const Expr *StrideExpr = OASE->getStride();
8345 if (!StrideExpr)
8346 return false;
8347
8348 assert(StrideExpr->getType()->isIntegerType() &&
8349 "Stride expression must be of integer type");
8350
8351 // If stride is not evaluatable as a constant, treat as
8352 // non-contiguous.
8353 const auto Constant =
8354 StrideExpr->getIntegerConstantExpr(Ctx: CGF.getContext());
8355 if (!Constant)
8356 return true;
8357
8358 // Treat non-unitary strides as non-contiguous.
8359 return !Constant->isOne();
8360 });
8361
8362 bool IsPrevMemberReference = false;
8363
8364 bool IsPartialMapped =
8365 !PartialStruct.PreliminaryMapData.BasePointers.empty();
8366
8367 // We need to check if we will be encountering any MEs. If we do not
8368 // encounter any ME expression it means we will be mapping the whole struct.
8369 // In that case we need to skip adding an entry for the struct to the
8370 // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo
8371 // list only when generating all info for clauses.
8372 bool IsMappingWholeStruct = true;
8373 if (!GenerateAllInfoForClauses) {
8374 IsMappingWholeStruct = false;
8375 } else {
8376 for (auto TempI = I; TempI != CE; ++TempI) {
8377 const MemberExpr *PossibleME =
8378 dyn_cast<MemberExpr>(Val: TempI->getAssociatedExpression());
8379 if (PossibleME) {
8380 IsMappingWholeStruct = false;
8381 break;
8382 }
8383 }
8384 }
8385
8386 bool SeenFirstNonBinOpExprAfterAttachPtr = false;
8387 for (; I != CE; ++I) {
8388 // If we have a valid attach-ptr, we skip processing all components until
8389 // after the attach-ptr.
8390 if (HasAttachPtr && !SeenAttachPtr) {
8391 SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr;
8392 continue;
8393 }
8394
8395 // After finding the attach pointer, skip binary-ops, to skip past
8396 // expressions like (p + 10), for a map like map(*(p + 10)), where p is
8397 // the attach-ptr.
8398 if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) {
8399 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8400 if (BO)
8401 continue;
8402
8403 // Found the first non-binary-operator component after attach
8404 SeenFirstNonBinOpExprAfterAttachPtr = true;
8405 BP = AttachPteeBaseAddr;
8406 }
8407
8408 // If the current component is member of a struct (parent struct) mark it.
8409 if (!EncounteredME) {
8410 EncounteredME = dyn_cast<MemberExpr>(Val: I->getAssociatedExpression());
8411 // If we encounter a PTR_AND_OBJ entry from now on it should be marked
8412 // as MEMBER_OF the parent struct.
8413 if (EncounteredME) {
8414 ShouldBeMemberOf = true;
8415 // Do not emit as complex pointer if this is actually not array-like
8416 // expression.
8417 if (FirstPointerInComplexData) {
8418 QualType Ty = std::prev(x: I)
8419 ->getAssociatedDeclaration()
8420 ->getType()
8421 .getNonReferenceType();
8422 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8423 FirstPointerInComplexData = false;
8424 }
8425 }
8426 }
8427
8428 auto Next = std::next(x: I);
8429
8430 // We need to generate the addresses and sizes if this is the last
8431 // component, if the component is a pointer or if it is an array section
8432 // whose length can't be proved to be one. If this is a pointer, it
8433 // becomes the base address for the following components.
8434
8435 // A final array section, is one whose length can't be proved to be one.
8436 // If the map item is non-contiguous then we don't treat any array section
8437 // as final array section.
8438 bool IsFinalArraySection =
8439 !IsNonContiguous &&
8440 isFinalArraySectionExpression(E: I->getAssociatedExpression());
8441
8442 // If we have a declaration for the mapping use that, otherwise use
8443 // the base declaration of the map clause.
8444 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
8445 ? I->getAssociatedDeclaration()
8446 : BaseDecl;
8447 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
8448 : MapExpr;
8449
8450 // Get information on whether the element is a pointer. Have to do a
8451 // special treatment for array sections given that they are built-in
8452 // types.
8453 const auto *OASE =
8454 dyn_cast<ArraySectionExpr>(Val: I->getAssociatedExpression());
8455 const auto *OAShE =
8456 dyn_cast<OMPArrayShapingExpr>(Val: I->getAssociatedExpression());
8457 const auto *UO = dyn_cast<UnaryOperator>(Val: I->getAssociatedExpression());
8458 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8459 bool IsPointer =
8460 OAShE ||
8461 (OASE && ArraySectionExpr::getBaseOriginalType(Base: OASE)
8462 .getCanonicalType()
8463 ->isAnyPointerType()) ||
8464 I->getAssociatedExpression()->getType()->isAnyPointerType();
8465 bool IsMemberReference = isa<MemberExpr>(Val: I->getAssociatedExpression()) &&
8466 MapDecl &&
8467 MapDecl->getType()->isLValueReferenceType();
8468 bool IsNonDerefPointer = IsPointer &&
8469 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
8470 !IsNonContiguous;
8471
8472 if (OASE)
8473 ++DimSize;
8474
8475 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
8476 IsFinalArraySection) {
8477 // If this is not the last component, we expect the pointer to be
8478 // associated with an array expression or member expression.
8479 assert((Next == CE ||
8480 isa<MemberExpr>(Next->getAssociatedExpression()) ||
8481 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
8482 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
8483 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
8484 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
8485 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
8486 "Unexpected expression");
8487
8488 Address LB = Address::invalid();
8489 Address LowestElem = Address::invalid();
8490 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
8491 const MemberExpr *E) {
8492 const Expr *BaseExpr = E->getBase();
8493 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a
8494 // scalar.
8495 LValue BaseLV;
8496 if (E->isArrow()) {
8497 LValueBaseInfo BaseInfo;
8498 TBAAAccessInfo TBAAInfo;
8499 Address Addr =
8500 CGF.EmitPointerWithAlignment(Addr: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
8501 QualType PtrTy = BaseExpr->getType()->getPointeeType();
8502 BaseLV = CGF.MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
8503 } else {
8504 BaseLV = CGF.EmitOMPSharedLValue(E: BaseExpr);
8505 }
8506 return BaseLV;
8507 };
8508 if (OAShE) {
8509 LowestElem = LB =
8510 Address(CGF.EmitScalarExpr(E: OAShE->getBase()),
8511 CGF.ConvertTypeForMem(
8512 T: OAShE->getBase()->getType()->getPointeeType()),
8513 CGF.getContext().getTypeAlignInChars(
8514 T: OAShE->getBase()->getType()));
8515 } else if (IsMemberReference) {
8516 const auto *ME = cast<MemberExpr>(Val: I->getAssociatedExpression());
8517 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
8518 LowestElem = CGF.EmitLValueForFieldInitialization(
8519 Base: BaseLVal, Field: cast<FieldDecl>(Val: MapDecl))
8520 .getAddress();
8521 LB = CGF.EmitLoadOfReferenceLValue(RefAddr: LowestElem, RefTy: MapDecl->getType())
8522 .getAddress();
8523 } else {
8524 LowestElem = LB =
8525 CGF.EmitOMPSharedLValue(E: I->getAssociatedExpression())
8526 .getAddress();
8527 }
8528
8529 // Save the final LowestElem, to use it as the pointee in attach maps,
8530 // if emitted.
8531 if (Next == CE)
8532 FinalLowestElem = LowestElem;
8533
8534 // If this component is a pointer inside the base struct then we don't
8535 // need to create any entry for it - it will be combined with the object
8536 // it is pointing to into a single PTR_AND_OBJ entry.
8537 bool IsMemberPointerOrAddr =
8538 EncounteredME &&
8539 (((IsPointer || ForDeviceAddr) &&
8540 I->getAssociatedExpression() == EncounteredME) ||
8541 (IsPrevMemberReference && !IsPointer) ||
8542 (IsMemberReference && Next != CE &&
8543 !Next->getAssociatedExpression()->getType()->isPointerType()));
8544 if (!OverlappedElements.empty() && Next == CE) {
8545 // Handle base element with the info for overlapped elements.
8546 assert(!PartialStruct.Base.isValid() && "The base element is set.");
8547 assert(!IsPointer &&
8548 "Unexpected base element with the pointer type.");
8549 // Mark the whole struct as the struct that requires allocation on the
8550 // device.
8551 PartialStruct.LowestElem = {0, LowestElem};
8552 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
8553 T: I->getAssociatedExpression()->getType());
8554 Address HB = CGF.Builder.CreateConstGEP(
8555 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8556 Addr: LowestElem, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty),
8557 Index: TypeSize.getQuantity() - 1);
8558 PartialStruct.HighestElem = {
8559 std::numeric_limits<decltype(
8560 PartialStruct.HighestElem.first)>::max(),
8561 HB};
8562 PartialStruct.Base = BP;
8563 PartialStruct.LB = LB;
8564 assert(
8565 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8566 "Overlapped elements must be used only once for the variable.");
8567 std::swap(a&: PartialStruct.PreliminaryMapData, b&: CombinedInfo);
8568 // Emit data for non-overlapped data.
8569 OpenMPOffloadMappingFlags Flags =
8570 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8571 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8572 /*AddPtrFlag=*/false,
8573 /*AddIsTargetParamFlag=*/false, IsNonContiguous);
8574 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8575 MapExpr, BP, LB, IsNonContiguous,
8576 DimSize);
8577 // Do bitcopy of all non-overlapped structure elements.
8578 for (OMPClauseMappableExprCommon::MappableExprComponentListRef
8579 Component : OverlappedElements) {
8580 for (const OMPClauseMappableExprCommon::MappableComponent &MC :
8581 Component) {
8582 if (const ValueDecl *VD = MC.getAssociatedDeclaration()) {
8583 if (const auto *FD = dyn_cast<FieldDecl>(Val: VD)) {
8584 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8585 }
8586 }
8587 }
8588 }
8589 CopyGaps.copyUntilEnd(HB);
8590 break;
8591 }
8592 llvm::Value *Size = getExprTypeSize(E: I->getAssociatedExpression());
8593 // Skip adding an entry in the CurInfo of this combined entry if the
8594 // whole struct is currently being mapped. The struct needs to be added
8595 // in the first position before any data internal to the struct is being
8596 // mapped.
8597 // Skip adding an entry in the CurInfo of this combined entry if the
8598 // PartialStruct.PreliminaryMapData.BasePointers has been mapped.
8599 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8600 (Next == CE && MapType != OMPC_MAP_unknown)) {
8601 if (!IsMappingWholeStruct) {
8602 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8603 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
8604 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8605 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8606 CombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8607 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8608 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8609 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize
8610 : 1);
8611 } else {
8612 StructBaseCombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8613 StructBaseCombinedInfo.BasePointers.push_back(
8614 Elt: BP.emitRawPointer(CGF));
8615 StructBaseCombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8616 StructBaseCombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8617 StructBaseCombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8618 StructBaseCombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8619 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8620 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8621 Elt: IsNonContiguous ? DimSize : 1);
8622 }
8623
8624 // If Mapper is valid, the last component inherits the mapper.
8625 bool HasMapper = Mapper && Next == CE;
8626 if (!IsMappingWholeStruct)
8627 CombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper : nullptr);
8628 else
8629 StructBaseCombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper
8630 : nullptr);
8631
8632 // We need to add a pointer flag for each map that comes from the
8633 // same expression except for the first one. We also need to signal
8634 // this map is the first one that relates with the current capture
8635 // (there is a set of entries for each capture).
8636 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
8637 MapType, MapModifiers, MotionModifiers, IsImplicit,
8638 AddPtrFlag: !IsExpressionFirstInfo || RequiresReference ||
8639 FirstPointerInComplexData || IsMemberReference,
8640 AddIsTargetParamFlag: IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
8641
8642 if (!IsExpressionFirstInfo || IsMemberReference) {
8643 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
8644 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
8645 if (IsPointer || (IsMemberReference && Next != CE))
8646 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8647 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8648 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8649 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8650 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8651
8652 if (ShouldBeMemberOf) {
8653 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
8654 // should be later updated with the correct value of MEMBER_OF.
8655 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8656 // From now on, all subsequent PTR_AND_OBJ entries should not be
8657 // marked as MEMBER_OF.
8658 ShouldBeMemberOf = false;
8659 }
8660 }
8661
8662 if (!IsMappingWholeStruct) {
8663 CombinedInfo.Types.push_back(Elt: Flags);
8664 // HasAttachPtr marks pointee entries, which have a base attach-ptr.
8665 CombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8666 } else {
8667 StructBaseCombinedInfo.Types.push_back(Elt: Flags);
8668 StructBaseCombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8669 }
8670 }
8671
8672 // If we have encountered a member expression so far, keep track of the
8673 // mapped member. If the parent is "*this", then the value declaration
8674 // is nullptr.
8675 if (EncounteredME) {
8676 const auto *FD = cast<FieldDecl>(Val: EncounteredME->getMemberDecl());
8677 unsigned FieldIndex = FD->getFieldIndex();
8678
8679 // Update info about the lowest and highest elements for this struct
8680 if (!PartialStruct.Base.isValid()) {
8681 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8682 if (IsFinalArraySection && OASE) {
8683 Address HB =
8684 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8685 .getAddress();
8686 PartialStruct.HighestElem = {FieldIndex, HB};
8687 } else {
8688 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8689 }
8690 PartialStruct.Base = BP;
8691 PartialStruct.LB = BP;
8692 } else if (FieldIndex < PartialStruct.LowestElem.first) {
8693 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8694 } else if (FieldIndex > PartialStruct.HighestElem.first) {
8695 if (IsFinalArraySection && OASE) {
8696 Address HB =
8697 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8698 .getAddress();
8699 PartialStruct.HighestElem = {FieldIndex, HB};
8700 } else {
8701 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8702 }
8703 }
8704 }
8705
8706 // Need to emit combined struct for array sections.
8707 if (IsFinalArraySection || IsNonContiguous)
8708 PartialStruct.IsArraySection = true;
8709
8710 // If we have a final array section, we are done with this expression.
8711 if (IsFinalArraySection)
8712 break;
8713
8714 // The pointer becomes the base for the next element.
8715 if (Next != CE)
8716 BP = IsMemberReference ? LowestElem : LB;
8717 if (!IsPartialMapped)
8718 IsExpressionFirstInfo = false;
8719 IsCaptureFirstInfo = false;
8720 FirstPointerInComplexData = false;
8721 IsPrevMemberReference = IsMemberReference;
8722 } else if (FirstPointerInComplexData) {
8723 QualType Ty = Components.rbegin()
8724 ->getAssociatedDeclaration()
8725 ->getType()
8726 .getNonReferenceType();
8727 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8728 FirstPointerInComplexData = false;
8729 }
8730 }
8731 // If ran into the whole component - allocate the space for the whole
8732 // record.
8733 if (!EncounteredME)
8734 PartialStruct.HasCompleteRecord = true;
8735
8736 // Populate ATTACH information for later processing by emitAttachEntry.
8737 if (shouldEmitAttachEntry(PointerExpr: AttachPtrExpr, MapBaseDecl: BaseDecl, CGF, CurDir)) {
8738 AttachInfo.AttachPtrAddr = AttachPtrAddr;
8739 AttachInfo.AttachPteeAddr = FinalLowestElem;
8740 AttachInfo.AttachPtrDecl = BaseDecl;
8741 AttachInfo.AttachMapExpr = MapExpr;
8742 }
8743
8744 if (!IsNonContiguous)
8745 return;
8746
8747 const ASTContext &Context = CGF.getContext();
8748
8749 // For supporting stride in array section, we need to initialize the first
8750 // dimension size as 1, first offset as 0, and first count as 1
8751 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 0)};
8752 MapValuesArrayTy CurCounts;
8753 MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8754 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8755 uint64_t ElementTypeSize;
8756
8757 // Collect Size information for each dimension and get the element size as
8758 // the first Stride. For example, for `int arr[10][10]`, the DimSizes
8759 // should be [10, 10] and the first stride is 4 btyes.
8760 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8761 Components) {
8762 const Expr *AssocExpr = Component.getAssociatedExpression();
8763 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8764
8765 if (!OASE)
8766 continue;
8767
8768 QualType Ty = ArraySectionExpr::getBaseOriginalType(Base: OASE->getBase());
8769 auto *CAT = Context.getAsConstantArrayType(T: Ty);
8770 auto *VAT = Context.getAsVariableArrayType(T: Ty);
8771
8772 // We need all the dimension size except for the last dimension.
8773 assert((VAT || CAT || &Component == &*Components.begin()) &&
8774 "Should be either ConstantArray or VariableArray if not the "
8775 "first Component");
8776
8777 // Get element size if CurCounts is empty.
8778 if (CurCounts.empty()) {
8779 const Type *ElementType = nullptr;
8780 if (CAT)
8781 ElementType = CAT->getElementType().getTypePtr();
8782 else if (VAT)
8783 ElementType = VAT->getElementType().getTypePtr();
8784 else if (&Component == &*Components.begin()) {
8785 // If the base is a raw pointer (e.g. T *data with data[a:b:c]),
8786 // there was no earlier CAT/VAT/array handling to establish
8787 // ElementType. Capture the pointee type now so that subsequent
8788 // components (offset/length/stride) have a concrete element type to
8789 // work with. This makes pointer-backed sections behave consistently
8790 // with CAT/VAT/array bases.
8791 if (const auto *PtrType = Ty->getAs<PointerType>())
8792 ElementType = PtrType->getPointeeType().getTypePtr();
8793 } else {
8794 // Any component after the first should never have a raw pointer type;
8795 // by this point. ElementType must already be known (set above or in
8796 // prior array / CAT / VAT handling).
8797 assert(!Ty->isPointerType() &&
8798 "Non-first components should not be raw pointers");
8799 }
8800
8801 // At this stage, if ElementType was a base pointer and we are in the
8802 // first iteration, it has been computed.
8803 if (ElementType) {
8804 // For the case that having pointer as base, we need to remove one
8805 // level of indirection.
8806 if (&Component != &*Components.begin())
8807 ElementType = ElementType->getPointeeOrArrayElementType();
8808 ElementTypeSize =
8809 Context.getTypeSizeInChars(T: ElementType).getQuantity();
8810 CurCounts.push_back(
8811 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: ElementTypeSize));
8812 }
8813 }
8814 // Get dimension value except for the last dimension since we don't need
8815 // it.
8816 if (DimSizes.size() < Components.size() - 1) {
8817 if (CAT)
8818 DimSizes.push_back(
8819 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: CAT->getZExtSize()));
8820 else if (VAT)
8821 DimSizes.push_back(Elt: CGF.Builder.CreateIntCast(
8822 V: CGF.EmitScalarExpr(E: VAT->getSizeExpr()), DestTy: CGF.Int64Ty,
8823 /*IsSigned=*/isSigned: false));
8824 }
8825 }
8826
8827 // Skip the dummy dimension since we have already have its information.
8828 auto *DI = DimSizes.begin() + 1;
8829 // Product of dimension.
8830 llvm::Value *DimProd =
8831 llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: ElementTypeSize);
8832
8833 // Collect info for non-contiguous. Notice that offset, count, and stride
8834 // are only meaningful for array-section, so we insert a null for anything
8835 // other than array-section.
8836 // Also, the size of offset, count, and stride are not the same as
8837 // pointers, base_pointers, sizes, or dims. Instead, the size of offset,
8838 // count, and stride are the same as the number of non-contiguous
8839 // declaration in target update to/from clause.
8840 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8841 Components) {
8842 const Expr *AssocExpr = Component.getAssociatedExpression();
8843
8844 if (const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr)) {
8845 llvm::Value *Offset = CGF.Builder.CreateIntCast(
8846 V: CGF.EmitScalarExpr(E: AE->getIdx()), DestTy: CGF.Int64Ty,
8847 /*isSigned=*/false);
8848 CurOffsets.push_back(Elt: Offset);
8849 CurCounts.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/1));
8850 CurStrides.push_back(Elt: CurStrides.back());
8851 continue;
8852 }
8853
8854 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8855
8856 if (!OASE)
8857 continue;
8858
8859 // Offset
8860 const Expr *OffsetExpr = OASE->getLowerBound();
8861 llvm::Value *Offset = nullptr;
8862 if (!OffsetExpr) {
8863 // If offset is absent, then we just set it to zero.
8864 Offset = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
8865 } else {
8866 Offset = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: OffsetExpr),
8867 DestTy: CGF.Int64Ty,
8868 /*isSigned=*/false);
8869 }
8870
8871 // Count
8872 const Expr *CountExpr = OASE->getLength();
8873 llvm::Value *Count = nullptr;
8874 if (!CountExpr) {
8875 // In Clang, once a high dimension is an array section, we construct all
8876 // the lower dimension as array section, however, for case like
8877 // arr[0:2][2], Clang construct the inner dimension as an array section
8878 // but it actually is not in an array section form according to spec.
8879 if (!OASE->getColonLocFirst().isValid() &&
8880 !OASE->getColonLocSecond().isValid()) {
8881 Count = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 1);
8882 } else {
8883 // OpenMP 5.0, 2.1.5 Array Sections, Description.
8884 // When the length is absent it defaults to ⌈(size −
8885 // lower-bound)/stride⌉, where size is the size of the array
8886 // dimension.
8887 const Expr *StrideExpr = OASE->getStride();
8888 llvm::Value *Stride =
8889 StrideExpr
8890 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8891 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8892 : nullptr;
8893 if (Stride)
8894 Count = CGF.Builder.CreateUDiv(
8895 LHS: CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset), RHS: Stride);
8896 else
8897 Count = CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset);
8898 }
8899 } else {
8900 Count = CGF.EmitScalarExpr(E: CountExpr);
8901 }
8902 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: CGF.Int64Ty, /*isSigned=*/false);
8903 CurCounts.push_back(Elt: Count);
8904
8905 // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size
8906 // Offset_n' = Offset_n * (D_0 * D_1 ... * D_n-1) * Unit size
8907 // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example:
8908 // Offset Count Stride
8909 // D0 0 4 1 (int) <- dummy dimension
8910 // D1 0 2 8 (2 * (1) * 4)
8911 // D2 100 2 20 (1 * (1 * 5) * 4)
8912 // D3 0 2 200 (2 * (1 * 5 * 4) * 4)
8913 const Expr *StrideExpr = OASE->getStride();
8914 llvm::Value *Stride =
8915 StrideExpr
8916 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8917 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8918 : nullptr;
8919 DimProd = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: *(DI - 1));
8920 if (Stride)
8921 CurStrides.push_back(Elt: CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Stride));
8922 else
8923 CurStrides.push_back(Elt: DimProd);
8924
8925 Offset = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Offset);
8926 CurOffsets.push_back(Elt: Offset);
8927
8928 if (DI != DimSizes.end())
8929 ++DI;
8930 }
8931
8932 CombinedInfo.NonContigInfo.Offsets.push_back(Elt: CurOffsets);
8933 CombinedInfo.NonContigInfo.Counts.push_back(Elt: CurCounts);
8934 CombinedInfo.NonContigInfo.Strides.push_back(Elt: CurStrides);
8935 }
8936
8937 /// Return the adjusted map modifiers if the declaration a capture refers to
8938 /// appears in a first-private clause. This is expected to be used only with
8939 /// directives that start with 'target'.
8940 OpenMPOffloadMappingFlags
8941 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
8942 assert(Cap.capturesVariable() && "Expected capture by reference only!");
8943
8944 // A first private variable captured by reference will use only the
8945 // 'private ptr' and 'map to' flag. Return the right flags if the captured
8946 // declaration is known as first-private in this handler.
8947 if (FirstPrivateDecls.count(Val: Cap.getCapturedVar())) {
8948 if (Cap.getCapturedVar()->getType()->isAnyPointerType())
8949 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8950 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8951 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8952 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8953 }
8954 auto I = LambdasMap.find(Val: Cap.getCapturedVar()->getCanonicalDecl());
8955 if (I != LambdasMap.end())
8956 // for map(to: lambda): using user specified map type.
8957 return getMapTypeBits(
8958 MapType: I->getSecond()->getMapType(), MapModifiers: I->getSecond()->getMapTypeModifiers(),
8959 /*MotionModifiers=*/{}, IsImplicit: I->getSecond()->isImplicit(),
8960 /*AddPtrFlag=*/false,
8961 /*AddIsTargetParamFlag=*/false,
8962 /*isNonContiguous=*/IsNonContiguous: false);
8963 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8964 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8965 }
8966
8967 void getPlainLayout(const CXXRecordDecl *RD,
8968 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8969 bool AsBase) const {
8970 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
8971
8972 llvm::StructType *St =
8973 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
8974
8975 unsigned NumElements = St->getNumElements();
8976 llvm::SmallVector<
8977 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8978 RecordLayout(NumElements);
8979
8980 // Fill bases.
8981 for (const auto &I : RD->bases()) {
8982 if (I.isVirtual())
8983 continue;
8984
8985 QualType BaseTy = I.getType();
8986 const auto *Base = BaseTy->getAsCXXRecordDecl();
8987 // Ignore empty bases.
8988 if (CodeGenUtils::isEmptyRecordForLayout(Ctx: CGF.getContext(), T: BaseTy) ||
8989 CGF.getContext()
8990 .getASTRecordLayout(D: Base)
8991 .getNonVirtualSize()
8992 .isZero())
8993 continue;
8994
8995 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(RD: Base);
8996 RecordLayout[FieldIndex] = Base;
8997 }
8998 // Fill in virtual bases.
8999 for (const auto &I : RD->vbases()) {
9000 QualType BaseTy = I.getType();
9001 // Ignore empty bases.
9002 if (CodeGenUtils::isEmptyRecordForLayout(Ctx: CGF.getContext(), T: BaseTy))
9003 continue;
9004
9005 const auto *Base = BaseTy->getAsCXXRecordDecl();
9006 unsigned FieldIndex = RL.getVirtualBaseIndex(base: Base);
9007 if (RecordLayout[FieldIndex])
9008 continue;
9009 RecordLayout[FieldIndex] = Base;
9010 }
9011 // Fill in all the fields.
9012 assert(!RD->isUnion() && "Unexpected union.");
9013 for (const auto *Field : RD->fields()) {
9014 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
9015 // will fill in later.)
9016 if (!Field->isBitField() &&
9017 !CodeGenUtils::isEmptyFieldForLayout(Ctx: CGF.getContext(), FD: Field)) {
9018 unsigned FieldIndex = RL.getLLVMFieldNo(FD: Field);
9019 RecordLayout[FieldIndex] = Field;
9020 }
9021 }
9022 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
9023 &Data : RecordLayout) {
9024 if (Data.isNull())
9025 continue;
9026 if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Val: Data))
9027 getPlainLayout(RD: Base, Layout, /*AsBase=*/true);
9028 else
9029 Layout.push_back(Elt: cast<const FieldDecl *>(Val: Data));
9030 }
9031 }
9032
9033 /// Returns the address corresponding to \p PointerExpr.
9034 static Address getAttachPtrAddr(const Expr *PointerExpr,
9035 CodeGenFunction &CGF) {
9036 assert(PointerExpr && "Cannot get addr from null attach-ptr expr");
9037 Address AttachPtrAddr = Address::invalid();
9038
9039 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: PointerExpr)) {
9040 // If the pointer is a variable, we can use its address directly.
9041 AttachPtrAddr = CGF.EmitLValue(E: DRE).getAddress();
9042 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(Val: PointerExpr)) {
9043 AttachPtrAddr =
9044 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/true).getAddress();
9045 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: PointerExpr)) {
9046 AttachPtrAddr = CGF.EmitLValue(E: ASE).getAddress();
9047 } else if (auto *ME = dyn_cast<MemberExpr>(Val: PointerExpr)) {
9048 AttachPtrAddr = CGF.EmitMemberExpr(E: ME).getAddress();
9049 } else if (auto *UO = dyn_cast<UnaryOperator>(Val: PointerExpr)) {
9050 assert(UO->getOpcode() == UO_Deref &&
9051 "Unexpected unary-operator on attach-ptr-expr");
9052 AttachPtrAddr = CGF.EmitLValue(E: UO).getAddress();
9053 }
9054 assert(AttachPtrAddr.isValid() &&
9055 "Failed to get address for attach pointer expression");
9056 return AttachPtrAddr;
9057 }
9058
9059 /// Get the address of the attach pointer, and a load from it, to get the
9060 /// pointee base address.
9061 /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair
9062 /// contains invalid addresses if \p AttachPtrExpr is null.
9063 static std::pair<Address, Address>
9064 getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr,
9065 CodeGenFunction &CGF) {
9066
9067 if (!AttachPtrExpr)
9068 return {Address::invalid(), Address::invalid()};
9069
9070 Address AttachPtrAddr = getAttachPtrAddr(PointerExpr: AttachPtrExpr, CGF);
9071 assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr");
9072
9073 QualType AttachPtrType =
9074 OMPClauseMappableExprCommon::getComponentExprElementType(Exp: AttachPtrExpr)
9075 .getCanonicalType();
9076
9077 Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer(
9078 Ptr: AttachPtrAddr, PtrTy: AttachPtrType->castAs<PointerType>());
9079 assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr");
9080
9081 return {AttachPtrAddr, AttachPteeBaseAddr};
9082 }
9083
9084 /// Returns whether an attach entry should be emitted for a map on
9085 /// \p MapBaseDecl on the directive \p CurDir.
9086 static bool
9087 shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl,
9088 CodeGenFunction &CGF,
9089 llvm::PointerUnion<const OMPExecutableDirective *,
9090 const OMPDeclareMapperDecl *>
9091 CurDir) {
9092 if (!PointerExpr)
9093 return false;
9094
9095 // Pointer attachment is needed at map-entering time or for declare
9096 // mappers.
9097 return isa<const OMPDeclareMapperDecl *>(Val: CurDir) ||
9098 isOpenMPTargetMapEnteringDirective(
9099 DKind: cast<const OMPExecutableDirective *>(Val&: CurDir)
9100 ->getDirectiveKind());
9101 }
9102
9103 /// Computes the attach-ptr expr for \p Components, and updates various maps
9104 /// with the information.
9105 /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr()
9106 /// with the OpenMPDirectiveKind extracted from \p CurDir.
9107 /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and
9108 /// AttachPtrExprMap.
9109 void collectAttachPtrExprInfo(
9110 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
9111 llvm::PointerUnion<const OMPExecutableDirective *,
9112 const OMPDeclareMapperDecl *>
9113 CurDir) {
9114
9115 OpenMPDirectiveKind CurDirectiveID =
9116 isa<const OMPDeclareMapperDecl *>(Val: CurDir)
9117 ? OMPD_declare_mapper
9118 : cast<const OMPExecutableDirective *>(Val&: CurDir)->getDirectiveKind();
9119
9120 const auto &[AttachPtrExpr, Depth] =
9121 OMPClauseMappableExprCommon::findAttachPtrExpr(Components,
9122 CurDirKind: CurDirectiveID);
9123
9124 AttachPtrComputationOrderMap.try_emplace(
9125 Key: AttachPtrExpr, Args: AttachPtrComputationOrderMap.size());
9126 AttachPtrComponentDepthMap.try_emplace(Key: AttachPtrExpr, Args: Depth);
9127 AttachPtrExprMap.try_emplace(Key: Components, Args: AttachPtrExpr);
9128 }
9129
9130 /// Generate all the base pointers, section pointers, sizes, map types, and
9131 /// mappers for the extracted mappable expressions (all included in \a
9132 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9133 /// pair of the relevant declaration and index where it occurs is appended to
9134 /// the device pointers info array.
9135 void generateAllInfoForClauses(
9136 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
9137 llvm::OpenMPIRBuilder &OMPBuilder,
9138 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9139 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9140 // We have to process the component lists that relate with the same
9141 // declaration in a single chunk so that we can generate the map flags
9142 // correctly. Therefore, we organize all lists in a map.
9143 enum MapKind { Present, Allocs, Other, Total };
9144 llvm::MapVector<CanonicalDeclPtr<const Decl>,
9145 SmallVector<SmallVector<MapInfo, 8>, 4>>
9146 Info;
9147
9148 // Helper function to fill the information map for the different supported
9149 // clauses.
9150 auto &&InfoGen =
9151 [&Info, &SkipVarSet](
9152 const ValueDecl *D, MapKind Kind,
9153 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
9154 OpenMPMapClauseKind MapType,
9155 ArrayRef<OpenMPMapModifierKind> MapModifiers,
9156 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
9157 bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,
9158 const Expr *VarRef = nullptr, bool ForDeviceAddr = false) {
9159 if (SkipVarSet.contains(V: D))
9160 return;
9161 auto It = Info.try_emplace(Key: D, Args: Total).first;
9162 It->second[Kind].emplace_back(
9163 Args&: L, Args&: MapType, Args&: MapModifiers, Args&: MotionModifiers, Args&: ReturnDevicePointer,
9164 Args&: IsImplicit, Args&: Mapper, Args&: VarRef, Args&: ForDeviceAddr);
9165 };
9166
9167 for (const auto *Cl : Clauses) {
9168 const auto *C = dyn_cast<OMPMapClause>(Val: Cl);
9169 if (!C)
9170 continue;
9171 MapKind Kind = Other;
9172 if (llvm::is_contained(Range: C->getMapTypeModifiers(),
9173 Element: OMPC_MAP_MODIFIER_present))
9174 Kind = Present;
9175 else if (C->getMapType() == OMPC_MAP_alloc)
9176 Kind = Allocs;
9177 const auto *EI = C->getVarRefs().begin();
9178 for (const auto L : C->component_lists()) {
9179 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
9180 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), C->getMapType(),
9181 C->getMapTypeModifiers(), {},
9182 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
9183 E);
9184 ++EI;
9185 }
9186 }
9187 for (const auto *Cl : Clauses) {
9188 const auto *C = dyn_cast<OMPToClause>(Val: Cl);
9189 if (!C)
9190 continue;
9191 MapKind Kind = Other;
9192 if (llvm::is_contained(Range: C->getMotionModifiers(),
9193 Element: OMPC_MOTION_MODIFIER_present))
9194 Kind = Present;
9195 if (llvm::is_contained(Range: C->getMotionModifiers(),
9196 Element: OMPC_MOTION_MODIFIER_iterator)) {
9197 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9198 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
9199 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
9200 CGF.EmitVarDecl(D: *VD);
9201 }
9202 }
9203
9204 const auto *EI = C->getVarRefs().begin();
9205 for (const auto L : C->component_lists()) {
9206 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_to, {},
9207 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
9208 C->isImplicit(), std::get<2>(t: L), *EI);
9209 ++EI;
9210 }
9211 }
9212 for (const auto *Cl : Clauses) {
9213 const auto *C = dyn_cast<OMPFromClause>(Val: Cl);
9214 if (!C)
9215 continue;
9216 MapKind Kind = Other;
9217 if (llvm::is_contained(Range: C->getMotionModifiers(),
9218 Element: OMPC_MOTION_MODIFIER_present))
9219 Kind = Present;
9220 if (llvm::is_contained(Range: C->getMotionModifiers(),
9221 Element: OMPC_MOTION_MODIFIER_iterator)) {
9222 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
9223 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
9224 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
9225 CGF.EmitVarDecl(D: *VD);
9226 }
9227 }
9228
9229 const auto *EI = C->getVarRefs().begin();
9230 for (const auto L : C->component_lists()) {
9231 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_from, {},
9232 C->getMotionModifiers(),
9233 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
9234 *EI);
9235 ++EI;
9236 }
9237 }
9238
9239 // Look at the use_device_ptr and use_device_addr clauses information and
9240 // mark the existing map entries as such. If there is no map information for
9241 // an entry in the use_device_ptr and use_device_addr list, we create one
9242 // with map type 'return_param' and zero size section. It is the user's
9243 // fault if that was not mapped before. If there is no map information, then
9244 // we defer the emission of that entry until all the maps for the same VD
9245 // have been handled.
9246 MapCombinedInfoTy UseDeviceDataCombinedInfo;
9247
9248 auto &&UseDeviceDataCombinedInfoGen =
9249 [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,
9250 CodeGenFunction &CGF, bool IsDevAddr,
9251 bool HasUdpFbNullify = false) {
9252 UseDeviceDataCombinedInfo.Exprs.push_back(Elt: VD);
9253 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Args&: Ptr);
9254 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(Args&: VD);
9255 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
9256 Args: IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
9257 // FIXME: For use_device_addr on array-sections, this should
9258 // be the starting address of the section.
9259 // e.g. int *p;
9260 // ... use_device_addr(p[3])
9261 // &p[0], &p[3], /*size=*/0, RETURN_PARAM
9262 UseDeviceDataCombinedInfo.Pointers.push_back(Elt: Ptr);
9263 UseDeviceDataCombinedInfo.Sizes.push_back(
9264 Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
9265 OpenMPOffloadMappingFlags Flags =
9266 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9267 if (HasUdpFbNullify)
9268 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9269 UseDeviceDataCombinedInfo.Types.push_back(Elt: Flags);
9270 UseDeviceDataCombinedInfo.HasAttachPtr.push_back(Elt: false);
9271 UseDeviceDataCombinedInfo.Mappers.push_back(Elt: nullptr);
9272 };
9273
9274 auto &&MapInfoGen =
9275 [&UseDeviceDataCombinedInfoGen](
9276 CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
9277 OMPClauseMappableExprCommon::MappableExprComponentListRef
9278 Components,
9279 bool IsDevAddr, bool IEIsAttachPtrForDevAddr = false,
9280 bool HasUdpFbNullify = false) {
9281 // We didn't find any match in our map information - generate a zero
9282 // size array section.
9283 llvm::Value *Ptr;
9284 if (IsDevAddr && !IEIsAttachPtrForDevAddr) {
9285 if (IE->isGLValue())
9286 Ptr = CGF.EmitLValue(E: IE).getPointer(CGF);
9287 else
9288 Ptr = CGF.EmitScalarExpr(E: IE);
9289 } else {
9290 Ptr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: IE), Loc: IE->getExprLoc());
9291 }
9292 bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr;
9293 // For the purpose of address-translation, treat something like the
9294 // following:
9295 // int *p;
9296 // ... use_device_addr(p[1])
9297 // equivalent to
9298 // ... use_device_ptr(p)
9299 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, /*IsDevAddr=*/IsDevAddr &&
9300 !TreatDevAddrAsDevPtr,
9301 HasUdpFbNullify);
9302 };
9303
9304 auto &&IsMapInfoExist =
9305 [&Info, this](CodeGenFunction &CGF, const ValueDecl *VD, const Expr *IE,
9306 const Expr *DesiredAttachPtrExpr, bool IsDevAddr,
9307 bool HasUdpFbNullify = false) -> bool {
9308 // We potentially have map information for this declaration already.
9309 // Look for the first set of components that refer to it. If found,
9310 // return true.
9311 // If the first component is a member expression, we have to look into
9312 // 'this', which maps to null in the map of map information. Otherwise
9313 // look directly for the information.
9314 auto It = Info.find(Key: isa<MemberExpr>(Val: IE) ? nullptr : VD);
9315 if (It != Info.end()) {
9316 bool Found = false;
9317 for (auto &Data : It->second) {
9318 MapInfo *CI = nullptr;
9319 // We potentially have multiple maps for the same decl. We need to
9320 // only consider those for which the attach-ptr matches the desired
9321 // attach-ptr.
9322 auto *It = llvm::find_if(Range&: Data, P: [&](const MapInfo &MI) {
9323 if (MI.Components.back().getAssociatedDeclaration() != VD)
9324 return false;
9325
9326 const Expr *MapAttachPtr = getAttachPtrExpr(Components: MI.Components);
9327 bool Match = AttachPtrComparator.areEqual(LHS: MapAttachPtr,
9328 RHS: DesiredAttachPtrExpr);
9329 return Match;
9330 });
9331
9332 if (It != Data.end())
9333 CI = &*It;
9334
9335 if (CI) {
9336 if (IsDevAddr) {
9337 CI->ForDeviceAddr = true;
9338 CI->ReturnDevicePointer = true;
9339 CI->HasUdpFbNullify = HasUdpFbNullify;
9340 Found = true;
9341 break;
9342 } else {
9343 auto PrevCI = std::next(x: CI->Components.rbegin());
9344 const auto *VarD = dyn_cast<VarDecl>(Val: VD);
9345 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: CI->Components);
9346 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
9347 isa<MemberExpr>(Val: IE) ||
9348 !VD->getType().getNonReferenceType()->isPointerType() ||
9349 PrevCI == CI->Components.rend() ||
9350 isa<MemberExpr>(Val: PrevCI->getAssociatedExpression()) || !VarD ||
9351 VarD->hasLocalStorage() ||
9352 (isa_and_nonnull<DeclRefExpr>(Val: AttachPtrExpr) &&
9353 VD == cast<DeclRefExpr>(Val: AttachPtrExpr)->getDecl())) {
9354 CI->ForDeviceAddr = IsDevAddr;
9355 CI->ReturnDevicePointer = true;
9356 CI->HasUdpFbNullify = HasUdpFbNullify;
9357 Found = true;
9358 break;
9359 }
9360 }
9361 }
9362 }
9363 return Found;
9364 }
9365 return false;
9366 };
9367
9368 // Look at the use_device_ptr clause information and mark the existing map
9369 // entries as such. If there is no map information for an entry in the
9370 // use_device_ptr list, we create one with map type 'alloc' and zero size
9371 // section. It is the user fault if that was not mapped before. If there is
9372 // no map information and the pointer is a struct member, then we defer the
9373 // emission of that entry until the whole struct has been processed.
9374 for (const auto *Cl : Clauses) {
9375 const auto *C = dyn_cast<OMPUseDevicePtrClause>(Val: Cl);
9376 if (!C)
9377 continue;
9378 bool HasUdpFbNullify =
9379 C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify;
9380 for (const auto L : C->component_lists()) {
9381 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9382 std::get<1>(t: L);
9383 assert(!Components.empty() &&
9384 "Not expecting empty list of components!");
9385 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
9386 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9387 const Expr *IE = Components.back().getAssociatedExpression();
9388 // For use_device_ptr, we match an existing map clause if its attach-ptr
9389 // is same as the use_device_ptr operand. e.g.
9390 // map expr | use_device_ptr expr | current behavior
9391 // ---------|---------------------|-----------------
9392 // p[1] | p | match
9393 // ps->a | ps | match
9394 // p | p | no match
9395 const Expr *UDPOperandExpr =
9396 Components.front().getAssociatedExpression();
9397 if (IsMapInfoExist(CGF, VD, IE,
9398 /*DesiredAttachPtrExpr=*/UDPOperandExpr,
9399 /*IsDevAddr=*/false, HasUdpFbNullify))
9400 continue;
9401 MapInfoGen(CGF, IE, VD, Components, /*IsDevAddr=*/false,
9402 /*IEIsAttachPtrForDevAddr=*/false, HasUdpFbNullify);
9403 }
9404 }
9405
9406 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
9407 for (const auto *Cl : Clauses) {
9408 const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Val: Cl);
9409 if (!C)
9410 continue;
9411 for (const auto L : C->component_lists()) {
9412 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9413 std::get<1>(t: L);
9414 assert(!std::get<1>(L).empty() &&
9415 "Not expecting empty list of components!");
9416 const ValueDecl *VD = std::get<1>(t: L).back().getAssociatedDeclaration();
9417 if (!Processed.insert(V: VD).second)
9418 continue;
9419 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9420 // For use_device_addr, we match an existing map clause if the
9421 // use_device_addr operand's attach-ptr matches the map operand's
9422 // attach-ptr.
9423 // We chould also restrict to only match cases when there is a full
9424 // match between the map/use_device_addr clause exprs, but that may be
9425 // unnecessary.
9426 //
9427 // map expr | use_device_addr expr | current | possible restrictive/
9428 // | | behavior | safer behavior
9429 // ---------|----------------------|-----------|-----------------------
9430 // p | p | match | match
9431 // p[0] | p[0] | match | match
9432 // p[0:1] | p[0] | match | no match
9433 // p[0:1] | p[2:1] | match | no match
9434 // p[1] | p[0] | match | no match
9435 // ps->a | ps->b | match | no match
9436 // p | p[0] | no match | no match
9437 // pp | pp[0][0] | no match | no match
9438 const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components);
9439 const Expr *IE = std::get<1>(t: L).back().getAssociatedExpression();
9440 assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) &&
9441 "use_device_addr operand has an attach-ptr, but does not match "
9442 "last component's expr.");
9443 if (IsMapInfoExist(CGF, VD, IE,
9444 /*DesiredAttachPtrExpr=*/UDAAttachPtrExpr,
9445 /*IsDevAddr=*/true))
9446 continue;
9447 MapInfoGen(CGF, IE, VD, Components,
9448 /*IsDevAddr=*/true,
9449 /*IEIsAttachPtrForDevAddr=*/UDAAttachPtrExpr != nullptr);
9450 }
9451 }
9452
9453 for (const auto &Data : Info) {
9454 MapCombinedInfoTy CurInfo;
9455 const Decl *D = Data.first;
9456 const ValueDecl *VD = cast_or_null<ValueDecl>(Val: D);
9457 // Group component lists by their AttachPtrExpr and process them in order
9458 // of increasing complexity (nullptr first, then simple expressions like
9459 // p, then more complex ones like p[0], etc.)
9460 //
9461 // This is similar to how generateInfoForCaptureFromClauseInfo handles
9462 // grouping for target constructs.
9463 SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs;
9464
9465 // First, collect all MapData entries with their attach-ptr exprs.
9466 for (const auto &M : Data.second) {
9467 for (const MapInfo &L : M) {
9468 assert(!L.Components.empty() &&
9469 "Not expecting declaration with no component lists.");
9470
9471 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: L.Components);
9472 AttachPtrMapInfoPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
9473 }
9474 }
9475
9476 // Next, sort by increasing order of their complexity.
9477 llvm::stable_sort(Range&: AttachPtrMapInfoPairs,
9478 C: [this](const auto &LHS, const auto &RHS) {
9479 return AttachPtrComparator(LHS.first, RHS.first);
9480 });
9481
9482 // And finally, process them all in order, grouping those with
9483 // equivalent attach-ptr exprs together.
9484 auto *It = AttachPtrMapInfoPairs.begin();
9485 while (It != AttachPtrMapInfoPairs.end()) {
9486 const Expr *AttachPtrExpr = It->first;
9487
9488 SmallVector<MapInfo, 8> GroupLists;
9489 while (It != AttachPtrMapInfoPairs.end() &&
9490 (It->first == AttachPtrExpr ||
9491 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
9492 GroupLists.push_back(Elt: It->second);
9493 ++It;
9494 }
9495 assert(!GroupLists.empty() && "GroupLists should not be empty");
9496
9497 StructRangeInfoTy PartialStruct;
9498 AttachInfoTy AttachInfo;
9499 MapCombinedInfoTy GroupCurInfo;
9500 // Current group's struct base information:
9501 MapCombinedInfoTy GroupStructBaseCurInfo;
9502 for (const MapInfo &L : GroupLists) {
9503 // Remember the current base pointer index.
9504 unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size();
9505 unsigned StructBasePointersIdx =
9506 GroupStructBaseCurInfo.BasePointers.size();
9507
9508 GroupCurInfo.NonContigInfo.IsNonContiguous =
9509 L.Components.back().isNonContiguous();
9510 generateInfoForComponentList(
9511 MapType: L.MapType, MapModifiers: L.MapModifiers, MotionModifiers: L.MotionModifiers, Components: L.Components,
9512 CombinedInfo&: GroupCurInfo, StructBaseCombinedInfo&: GroupStructBaseCurInfo, PartialStruct, AttachInfo,
9513 /*IsFirstComponentList=*/false, IsImplicit: L.IsImplicit,
9514 /*GenerateAllInfoForClauses*/ true, Mapper: L.Mapper, ForDeviceAddr: L.ForDeviceAddr, BaseDecl: VD,
9515 MapExpr: L.VarRef, /*OverlappedElements*/ {});
9516
9517 // If this entry relates to a device pointer, set the relevant
9518 // declaration and add the 'return pointer' flag.
9519 if (L.ReturnDevicePointer) {
9520 // Check whether a value was added to either GroupCurInfo or
9521 // GroupStructBaseCurInfo and error if no value was added to either
9522 // of them:
9523 assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() ||
9524 StructBasePointersIdx <
9525 GroupStructBaseCurInfo.BasePointers.size()) &&
9526 "Unexpected number of mapped base pointers.");
9527
9528 // Choose a base pointer index which is always valid:
9529 const ValueDecl *RelevantVD =
9530 L.Components.back().getAssociatedDeclaration();
9531 assert(RelevantVD &&
9532 "No relevant declaration related with device pointer??");
9533
9534 // If GroupStructBaseCurInfo has been updated this iteration then
9535 // work on the first new entry added to it i.e. make sure that when
9536 // multiple values are added to any of the lists, the first value
9537 // added is being modified by the assignments below (not the last
9538 // value added).
9539 auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info,
9540 unsigned Idx) {
9541 Info.DevicePtrDecls[Idx] = RelevantVD;
9542 Info.DevicePointers[Idx] = L.ForDeviceAddr
9543 ? DeviceInfoTy::Address
9544 : DeviceInfoTy::Pointer;
9545 Info.Types[Idx] |=
9546 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9547 if (L.HasUdpFbNullify)
9548 Info.Types[Idx] |=
9549 OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9550 };
9551
9552 if (StructBasePointersIdx <
9553 GroupStructBaseCurInfo.BasePointers.size())
9554 SetDevicePointerInfo(GroupStructBaseCurInfo,
9555 StructBasePointersIdx);
9556 else
9557 SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx);
9558 }
9559 }
9560
9561 // Unify entries in one list making sure the struct mapping precedes the
9562 // individual fields:
9563 MapCombinedInfoTy GroupUnionCurInfo;
9564 GroupUnionCurInfo.append(CurInfo&: GroupStructBaseCurInfo);
9565 GroupUnionCurInfo.append(CurInfo&: GroupCurInfo);
9566
9567 // If there is an entry in PartialStruct it means we have a struct with
9568 // individual members mapped. Emit an extra combined entry.
9569 if (PartialStruct.Base.isValid()) {
9570 // Prepend a synthetic dimension of length 1 to represent the
9571 // aggregated struct object. Using 1 (not 0, as 0 produced an
9572 // incorrect non-contiguous descriptor (DimSize==1), causing the
9573 // non-contiguous motion clause path to be skipped.) is important:
9574 // * It preserves the correct rank so targetDataUpdate() computes
9575 // DimSize == 2 for cases like strided array sections originating
9576 // from user-defined mappers (e.g. test with s.data[0:8:2]).
9577 GroupUnionCurInfo.NonContigInfo.Dims.insert(
9578 I: GroupUnionCurInfo.NonContigInfo.Dims.begin(), Elt: 1);
9579 emitCombinedEntry(
9580 CombinedInfo&: CurInfo, CurTypes&: GroupUnionCurInfo.Types, PartialStruct, AttachInfo,
9581 /*IsMapThis=*/!VD, OMPBuilder, VD,
9582 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size(),
9583 /*NotTargetParams=*/true);
9584 }
9585
9586 // Append this group's results to the overall CurInfo in the correct
9587 // order: combined-entry -> original-field-entries -> attach-entry
9588 CurInfo.append(CurInfo&: GroupUnionCurInfo);
9589 if (AttachInfo.isValid())
9590 emitAttachEntry(CGF, CombinedInfo&: CurInfo, AttachInfo);
9591 }
9592
9593 // We need to append the results of this capture to what we already have.
9594 CombinedInfo.append(CurInfo);
9595 }
9596 // Append data for use_device_ptr/addr clauses.
9597 CombinedInfo.append(CurInfo&: UseDeviceDataCombinedInfo);
9598 }
9599
9600public:
9601 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
9602 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9603 // Extract firstprivate clause information.
9604 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
9605 for (const auto *D : C->varlist()) {
9606 const ValueDecl *VD = cast<DeclRefExpr>(Val: D)->getDecl();
9607 if (const auto *BD = dyn_cast<BindingDecl>(Val: VD))
9608 VD = cast<VarDecl>(Val: BD->getDecomposedDecl());
9609 FirstPrivateDecls.try_emplace(Key: cast<VarDecl>(Val: VD), Args: C->isImplicit());
9610 }
9611 // Extract implicit firstprivates from uses_allocators clauses.
9612 for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
9613 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
9614 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
9615 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: D.AllocatorTraits))
9616 FirstPrivateDecls.try_emplace(Key: cast<VarDecl>(Val: DRE->getDecl()),
9617 /*Implicit=*/Args: true);
9618 else if (const auto *VD = dyn_cast<VarDecl>(
9619 Val: cast<DeclRefExpr>(Val: D.Allocator->IgnoreParenImpCasts())
9620 ->getDecl()))
9621 FirstPrivateDecls.try_emplace(Key: VD, /*Implicit=*/Args: true);
9622 }
9623 }
9624 // Extract defaultmap clause information.
9625 for (const auto *C : Dir.getClausesOfKind<OMPDefaultmapClause>())
9626 if (C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate)
9627 DefaultmapFirstprivateKinds.insert(V: C->getDefaultmapKind());
9628 // Extract device pointer clause information.
9629 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9630 for (auto L : C->component_lists())
9631 DevPointersMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9632 // Extract device addr clause information.
9633 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9634 for (auto L : C->component_lists())
9635 HasDevAddrsMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9636 // Extract map information.
9637 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) {
9638 if (C->getMapType() != OMPC_MAP_to)
9639 continue;
9640 for (auto L : C->component_lists()) {
9641 const ValueDecl *VD = std::get<0>(t&: L);
9642 const auto *RD = VD ? VD->getType()
9643 .getCanonicalType()
9644 .getNonReferenceType()
9645 ->getAsCXXRecordDecl()
9646 : nullptr;
9647 if (RD && RD->isLambda())
9648 LambdasMap.try_emplace(Key: std::get<0>(t&: L), Args&: C);
9649 }
9650 }
9651
9652 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9653 for (auto L : C->component_lists()) {
9654 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9655 std::get<1>(L);
9656 if (!Components.empty())
9657 collectAttachPtrExprInfo(Components, CurDir);
9658 }
9659 };
9660
9661 // Populate the AttachPtrExprMap for all component lists from map-related
9662 // clauses.
9663 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>())
9664 CollectAttachPtrExprsForClauseComponents(C);
9665 for (const auto *C : Dir.getClausesOfKind<OMPToClause>())
9666 CollectAttachPtrExprsForClauseComponents(C);
9667 for (const auto *C : Dir.getClausesOfKind<OMPFromClause>())
9668 CollectAttachPtrExprsForClauseComponents(C);
9669 for (const auto *C : Dir.getClausesOfKind<OMPUseDevicePtrClause>())
9670 CollectAttachPtrExprsForClauseComponents(C);
9671 for (const auto *C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>())
9672 CollectAttachPtrExprsForClauseComponents(C);
9673 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9674 CollectAttachPtrExprsForClauseComponents(C);
9675 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9676 CollectAttachPtrExprsForClauseComponents(C);
9677 }
9678
9679 /// Constructor for the declare mapper directive.
9680 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
9681 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9682 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9683 for (auto L : C->component_lists()) {
9684 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9685 std::get<1>(L);
9686 if (!Components.empty())
9687 collectAttachPtrExprInfo(Components, CurDir);
9688 }
9689 };
9690
9691 // Populate the AttachPtrExprMap for all component lists from map-related
9692 // clauses in the declare mapper directive, to enable attach-style mapping
9693 // for mappers.
9694 for (const auto *Cl : Dir.clauses()) {
9695 if (const auto *C = dyn_cast<OMPMapClause>(Val: Cl))
9696 CollectAttachPtrExprsForClauseComponents(C);
9697 else if (const auto *C = dyn_cast<OMPToClause>(Val: Cl))
9698 CollectAttachPtrExprsForClauseComponents(C);
9699 else if (const auto *C = dyn_cast<OMPFromClause>(Val: Cl))
9700 CollectAttachPtrExprsForClauseComponents(C);
9701 }
9702 }
9703
9704 /// Generate code for the combined entry if we have a partially mapped struct
9705 /// and take care of the mapping flags of the arguments corresponding to
9706 /// individual struct members.
9707 /// If a valid \p AttachInfo exists, its pointee addr will be updated to point
9708 /// to the combined-entry's begin address, if emitted.
9709 /// \p PartialStruct contains attach base-pointer information.
9710 /// \returns The index of the combined entry if one was added, std::nullopt
9711 /// otherwise.
9712 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9713 MapFlagsArrayTy &CurTypes,
9714 const StructRangeInfoTy &PartialStruct,
9715 AttachInfoTy &AttachInfo, bool IsMapThis,
9716 llvm::OpenMPIRBuilder &OMPBuilder, const ValueDecl *VD,
9717 unsigned OffsetForMemberOfFlag,
9718 bool NotTargetParams) const {
9719 if (CurTypes.size() == 1 &&
9720 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9721 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9722 !PartialStruct.IsArraySection)
9723 return;
9724 Address LBAddr = PartialStruct.LowestElem.second;
9725 Address HBAddr = PartialStruct.HighestElem.second;
9726 if (PartialStruct.HasCompleteRecord) {
9727 LBAddr = PartialStruct.LB;
9728 HBAddr = PartialStruct.LB;
9729 }
9730 CombinedInfo.Exprs.push_back(Elt: VD);
9731 // Base is the base of the struct
9732 CombinedInfo.BasePointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9733 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9734 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9735 // Pointer is the address of the lowest element
9736 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
9737 const CXXMethodDecl *MD =
9738 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(Val: CGF.CurFuncDecl) : nullptr;
9739 const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
9740 bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false;
9741 // There should not be a mapper for a combined entry.
9742 if (HasBaseClass) {
9743 // OpenMP 5.2 148:21:
9744 // If the target construct is within a class non-static member function,
9745 // and a variable is an accessible data member of the object for which the
9746 // non-static data member function is invoked, the variable is treated as
9747 // if the this[:1] expression had appeared in a map clause with a map-type
9748 // of tofrom.
9749 // Emit this[:1]
9750 CombinedInfo.Pointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9751 QualType Ty = MD->getFunctionObjectParameterType();
9752 llvm::Value *Size =
9753 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty), DestTy: CGF.Int64Ty,
9754 /*isSigned=*/true);
9755 CombinedInfo.Sizes.push_back(Elt: Size);
9756 } else {
9757 CombinedInfo.Pointers.push_back(Elt: LB);
9758 // Size is (addr of {highest+1} element) - (addr of lowest element)
9759 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
9760 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
9761 Ty: HBAddr.getElementType(), Ptr: HB, /*Idx0=*/1);
9762 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(V: LB, DestTy: CGF.VoidPtrTy);
9763 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(V: HAddr, DestTy: CGF.VoidPtrTy);
9764 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(LHS: CHAddr, RHS: CLAddr);
9765 llvm::Value *Size = CGF.Builder.CreateIntCast(V: Diff, DestTy: CGF.Int64Ty,
9766 /*isSigned=*/false);
9767 CombinedInfo.Sizes.push_back(Elt: Size);
9768 }
9769 CombinedInfo.Mappers.push_back(Elt: nullptr);
9770 // Map type is always TARGET_PARAM, if generate info for captures.
9771 CombinedInfo.Types.push_back(
9772 Elt: NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9773 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9774 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9775 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9776 // A combined entry has a base attach-ptr if its constituents do. e.g.:
9777 // map(s2.s1p->x, s2.s1p->y)
9778 // combined entry:
9779 // s2.s1p[0], s2.s1p->x, sizeof(s1p->x..y), ALLOC
9780 // here s2.s1p is the attach-ptr for the combined entry.
9781 // See the inline comments in emitUserDefinedMapper's definition for how
9782 // entries with an attach-ptr are treated.
9783 CombinedInfo.HasAttachPtr.push_back(Elt: AttachInfo.isValid());
9784 // If any element has the present modifier, then make sure the runtime
9785 // doesn't attempt to allocate the struct.
9786 if (CurTypes.end() !=
9787 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9788 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9789 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9790 }))
9791 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9792 // Remove TARGET_PARAM flag from the first element
9793 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9794 // If any element has the ompx_hold modifier, then make sure the runtime
9795 // uses the hold reference count for the struct as a whole so that it won't
9796 // be unmapped by an extra dynamic reference count decrement. Add it to all
9797 // elements as well so the runtime knows which reference count to check
9798 // when determining whether it's time for device-to-host transfers of
9799 // individual elements.
9800 if (CurTypes.end() !=
9801 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9802 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9803 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9804 })) {
9805 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9806 for (auto &M : CurTypes)
9807 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9808 }
9809
9810 // All other current entries will be MEMBER_OF the combined entry
9811 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9812 // 0xFFFF in the MEMBER_OF field, or ATTACH entries since they are expected
9813 // to be handled by themselves, after all other maps).
9814 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9815 Position: OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9816 for (auto &M : CurTypes)
9817 OMPBuilder.setCorrectMemberOfFlag(Flags&: M, MemberOfFlag);
9818
9819 // When we are emitting a combined entry. If there were any pending
9820 // attachments to be done, we do them to the begin address of the combined
9821 // entry. Note that this means only one attachment per combined-entry will
9822 // be done. So, for instance, if we have:
9823 // S *ps;
9824 // ... map(ps->a, ps->b)
9825 // When we are emitting a combined entry. If AttachInfo is valid,
9826 // update the pointee address to point to the begin address of the combined
9827 // entry. This ensures that if we have multiple maps like:
9828 // `map(ps->a, ps->b)`, we still get a single ATTACH entry, like:
9829 //
9830 // &ps[0], &ps->a, sizeof(ps->a to ps->b), ALLOC // combined-entry
9831 // &ps[0], &ps->a, sizeof(ps->a), TO | FROM
9832 // &ps[0], &ps->b, sizeof(ps->b), TO | FROM
9833 // &ps, &ps->a, sizeof(void*), ATTACH // Use combined-entry's LB
9834 if (AttachInfo.isValid())
9835 AttachInfo.AttachPteeAddr = LBAddr;
9836 }
9837
9838 /// Generate all the base pointers, section pointers, sizes, map types, and
9839 /// mappers for the extracted mappable expressions (all included in \a
9840 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9841 /// pair of the relevant declaration and index where it occurs is appended to
9842 /// the device pointers info array.
9843 void generateAllInfo(
9844 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9845 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9846 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9847 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9848 "Expect a executable directive");
9849 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9850 generateAllInfoForClauses(Clauses: CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9851 SkipVarSet);
9852 }
9853
9854 /// Generate all the base pointers, section pointers, sizes, map types, and
9855 /// mappers for the extracted map clauses of user-defined mapper (all included
9856 /// in \a CombinedInfo).
9857 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9858 llvm::OpenMPIRBuilder &OMPBuilder) const {
9859 assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&
9860 "Expect a declare mapper directive");
9861 const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(Val: CurDir);
9862 generateAllInfoForClauses(Clauses: CurMapperDir->clauses(), CombinedInfo,
9863 OMPBuilder);
9864 }
9865
9866 /// Emit capture info for lambdas for variables captured by reference.
9867 void generateInfoForLambdaCaptures(
9868 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9869 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
9870 QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();
9871 const auto *RD = VDType->getAsCXXRecordDecl();
9872 if (!RD || !RD->isLambda())
9873 return;
9874 Address VDAddr(Arg, CGF.ConvertTypeForMem(T: VDType),
9875 CGF.getContext().getDeclAlign(D: VD));
9876 LValue VDLVal = CGF.MakeAddrLValue(Addr: VDAddr, T: VDType);
9877 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9878 FieldDecl *ThisCapture = nullptr;
9879 RD->getCaptureFields(Captures, ThisCapture);
9880 if (ThisCapture) {
9881 LValue ThisLVal =
9882 CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: ThisCapture);
9883 LValue ThisLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: ThisCapture);
9884 LambdaPointers.try_emplace(Key: ThisLVal.getPointer(CGF),
9885 Args: VDLVal.getPointer(CGF));
9886 CombinedInfo.Exprs.push_back(Elt: VD);
9887 CombinedInfo.BasePointers.push_back(Elt: ThisLVal.getPointer(CGF));
9888 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9889 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9890 CombinedInfo.Pointers.push_back(Elt: ThisLValVal.getPointer(CGF));
9891 CombinedInfo.Sizes.push_back(
9892 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy),
9893 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9894 CombinedInfo.Types.push_back(
9895 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9896 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9897 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9898 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9899 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9900 CombinedInfo.Mappers.push_back(Elt: nullptr);
9901 }
9902 for (const LambdaCapture &LC : RD->captures()) {
9903 if (!LC.capturesVariable())
9904 continue;
9905 const VarDecl *VD = cast<VarDecl>(Val: LC.getCapturedVar());
9906 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
9907 continue;
9908 auto It = Captures.find(Val: VD);
9909 assert(It != Captures.end() && "Found lambda capture without field.");
9910 LValue VarLVal = CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: It->second);
9911 if (LC.getCaptureKind() == LCK_ByRef) {
9912 LValue VarLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: It->second);
9913 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9914 Args: VDLVal.getPointer(CGF));
9915 CombinedInfo.Exprs.push_back(Elt: VD);
9916 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9917 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9918 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9919 CombinedInfo.Pointers.push_back(Elt: VarLValVal.getPointer(CGF));
9920 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9921 V: CGF.getTypeSize(
9922 Ty: VD->getType().getCanonicalType().getNonReferenceType()),
9923 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9924 } else {
9925 RValue VarRVal = CGF.EmitLoadOfLValue(V: VarLVal, Loc: RD->getLocation());
9926 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9927 Args: VDLVal.getPointer(CGF));
9928 CombinedInfo.Exprs.push_back(Elt: VD);
9929 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9930 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9931 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9932 CombinedInfo.Pointers.push_back(Elt: VarRVal.getScalarVal());
9933 CombinedInfo.Sizes.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0));
9934 }
9935 CombinedInfo.Types.push_back(
9936 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9937 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9938 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9939 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9940 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9941 CombinedInfo.Mappers.push_back(Elt: nullptr);
9942 }
9943 }
9944
9945 /// Set correct indices for lambdas captures.
9946 void adjustMemberOfForLambdaCaptures(
9947 llvm::OpenMPIRBuilder &OMPBuilder,
9948 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9949 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9950 MapFlagsArrayTy &Types) const {
9951 for (unsigned I = 0, E = Types.size(); I < E; ++I) {
9952 // Set correct member_of idx for all implicit lambda captures.
9953 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9954 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9955 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9956 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9957 continue;
9958 llvm::Value *BasePtr = LambdaPointers.lookup(Val: BasePointers[I]);
9959 assert(BasePtr && "Unable to find base lambda address.");
9960 int TgtIdx = -1;
9961 for (unsigned J = I; J > 0; --J) {
9962 unsigned Idx = J - 1;
9963 if (Pointers[Idx] != BasePtr)
9964 continue;
9965 TgtIdx = Idx;
9966 break;
9967 }
9968 assert(TgtIdx != -1 && "Unable to find parent lambda.");
9969 // All other current entries will be MEMBER_OF the combined entry
9970 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9971 // 0xFFFF in the MEMBER_OF field).
9972 OpenMPOffloadMappingFlags MemberOfFlag =
9973 OMPBuilder.getMemberOfFlag(Position: TgtIdx);
9974 OMPBuilder.setCorrectMemberOfFlag(Flags&: Types[I], MemberOfFlag);
9975 }
9976 }
9977
9978 /// Populate component lists for non-lambda captured variables from map,
9979 /// is_device_ptr and has_device_addr clause info.
9980 void populateComponentListsForNonLambdaCaptureFromClauses(
9981 const ValueDecl *VD, MapDataArrayTy &DeclComponentLists,
9982 SmallVectorImpl<
9983 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9984 &StorageForImplicitlyAddedComponentLists) const {
9985 if (VD && LambdasMap.count(Val: VD))
9986 return;
9987
9988 // For member fields list in is_device_ptr, store it in
9989 // DeclComponentLists for generating components info.
9990 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9991 auto It = DevPointersMap.find(Val: VD);
9992 if (It != DevPointersMap.end())
9993 for (const auto &MCL : It->second)
9994 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_to, Args: Unknown,
9995 /*IsImpicit = */ Args: true, Args: nullptr,
9996 Args: nullptr);
9997 auto I = HasDevAddrsMap.find(Val: VD);
9998 if (I != HasDevAddrsMap.end())
9999 for (const auto &MCL : I->second)
10000 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_tofrom, Args: Unknown,
10001 /*IsImpicit = */ Args: true, Args: nullptr,
10002 Args: nullptr);
10003 assert(isa<const OMPExecutableDirective *>(CurDir) &&
10004 "Expect a executable directive");
10005 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
10006 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
10007 const auto *EI = C->getVarRefs().begin();
10008 for (const auto L : C->decl_component_lists(VD)) {
10009 const ValueDecl *VDecl, *Mapper;
10010 // The Expression is not correct if the mapping is implicit
10011 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
10012 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10013 std::tie(args&: VDecl, args&: Components, args&: Mapper) = L;
10014 assert(VDecl == VD && "We got information for the wrong declaration??");
10015 assert(!Components.empty() &&
10016 "Not expecting declaration with no component lists.");
10017 DeclComponentLists.emplace_back(Args&: Components, Args: C->getMapType(),
10018 Args: C->getMapTypeModifiers(),
10019 Args: C->isImplicit(), Args&: Mapper, Args&: E);
10020 ++EI;
10021 }
10022 }
10023
10024 // For the target construct, if there's a map with a base-pointer that's
10025 // a member of an implicitly captured struct, of the current class,
10026 // we need to emit an implicit map on the pointer.
10027 if (isOpenMPTargetExecutionDirective(DKind: CurExecDir->getDirectiveKind()))
10028 addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10029 CapturedVD: VD, DeclComponentLists, ComponentVectorStorage&: StorageForImplicitlyAddedComponentLists);
10030
10031 llvm::stable_sort(Range&: DeclComponentLists, C: [](const MapData &LHS,
10032 const MapData &RHS) {
10033 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(t: LHS);
10034 OpenMPMapClauseKind MapType = std::get<1>(t: RHS);
10035 bool HasPresent =
10036 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
10037 bool HasAllocs = MapType == OMPC_MAP_alloc;
10038 MapModifiers = std::get<2>(t: RHS);
10039 MapType = std::get<1>(t: LHS);
10040 bool HasPresentR =
10041 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
10042 bool HasAllocsR = MapType == OMPC_MAP_alloc;
10043 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
10044 });
10045 }
10046
10047 /// On a target construct, if there's an implicit map on a struct, or that of
10048 /// this[:], and an explicit map with a member of that struct/class as the
10049 /// base-pointer, we need to make sure that base-pointer is implicitly mapped,
10050 /// to make sure we don't map the full struct/class. For example:
10051 ///
10052 /// \code
10053 /// struct S {
10054 /// int dummy[10000];
10055 /// int *p;
10056 /// void f1() {
10057 /// #pragma omp target map(p[0:1])
10058 /// (void)this;
10059 /// }
10060 /// }; S s;
10061 ///
10062 /// void f2() {
10063 /// #pragma omp target map(s.p[0:10])
10064 /// (void)s;
10065 /// }
10066 /// \endcode
10067 ///
10068 /// Only `this-p` and `s.p` should be mapped in the two cases above.
10069 //
10070 // OpenMP 6.0: 7.9.6 map clause, pg 285
10071 // If a list item with an implicitly determined data-mapping attribute does
10072 // not have any corresponding storage in the device data environment prior to
10073 // a task encountering the construct associated with the map clause, and one
10074 // or more contiguous parts of the original storage are either list items or
10075 // base pointers to list items that are explicitly mapped on the construct,
10076 // only those parts of the original storage will have corresponding storage in
10077 // the device data environment as a result of the map clauses on the
10078 // construct.
10079 void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
10080 const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists,
10081 SmallVectorImpl<
10082 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
10083 &ComponentVectorStorage) const {
10084 bool IsThisCapture = CapturedVD == nullptr;
10085
10086 for (const auto &ComponentsAndAttachPtr : AttachPtrExprMap) {
10087 OMPClauseMappableExprCommon::MappableExprComponentListRef
10088 ComponentsWithAttachPtr = ComponentsAndAttachPtr.first;
10089 const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second;
10090 if (!AttachPtrExpr)
10091 continue;
10092
10093 const auto *ME = dyn_cast<MemberExpr>(Val: AttachPtrExpr);
10094 if (!ME)
10095 continue;
10096
10097 const Expr *Base = ME->getBase()->IgnoreParenImpCasts();
10098
10099 // If we are handling a "this" capture, then we are looking for
10100 // attach-ptrs of form `this->p`, either explicitly or implicitly.
10101 if (IsThisCapture && !ME->isImplicitCXXThis() && !isa<CXXThisExpr>(Val: Base))
10102 continue;
10103
10104 if (!IsThisCapture && (!isa<DeclRefExpr>(Val: Base) ||
10105 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
10106 continue;
10107
10108 // For non-this captures, we are looking for attach-ptrs of form
10109 // `s.p`.
10110 // For non-this captures, we are looking for attach-ptrs like `s.p`.
10111 if (!IsThisCapture && (ME->isArrow() || !isa<DeclRefExpr>(Val: Base) ||
10112 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
10113 continue;
10114
10115 // Check if we have an existing map on either:
10116 // this[:], s, this->p, or s.p, in which case, we don't need to add
10117 // an implicit one for the attach-ptr s.p/this->p.
10118 bool FoundExistingMap = false;
10119 for (const MapData &ExistingL : DeclComponentLists) {
10120 OMPClauseMappableExprCommon::MappableExprComponentListRef
10121 ExistingComponents = std::get<0>(t: ExistingL);
10122
10123 if (ExistingComponents.empty())
10124 continue;
10125
10126 // First check if we have a map like map(this->p) or map(s.p).
10127 const auto &FirstComponent = ExistingComponents.front();
10128 const Expr *FirstExpr = FirstComponent.getAssociatedExpression();
10129
10130 if (!FirstExpr)
10131 continue;
10132
10133 // First check if we have a map like map(this->p) or map(s.p).
10134 if (AttachPtrComparator.areEqual(LHS: FirstExpr, RHS: AttachPtrExpr)) {
10135 FoundExistingMap = true;
10136 break;
10137 }
10138
10139 // Check if we have a map like this[0:1]
10140 if (IsThisCapture) {
10141 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: FirstExpr)) {
10142 if (isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts())) {
10143 FoundExistingMap = true;
10144 break;
10145 }
10146 }
10147 continue;
10148 }
10149
10150 // When the attach-ptr is something like `s.p`, check if
10151 // `s` itself is mapped explicitly.
10152 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: FirstExpr)) {
10153 if (DRE->getDecl() == CapturedVD) {
10154 FoundExistingMap = true;
10155 break;
10156 }
10157 }
10158 }
10159
10160 if (FoundExistingMap)
10161 continue;
10162
10163 // If no base map is found, we need to create an implicit map for the
10164 // attach-pointer expr.
10165
10166 ComponentVectorStorage.emplace_back();
10167 auto &AttachPtrComponents = ComponentVectorStorage.back();
10168
10169 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
10170 bool SeenAttachPtrComponent = false;
10171 // For creating a map on the attach-ptr `s.p/this->p`, we copy all
10172 // components from the component-list which has `s.p/this->p`
10173 // as the attach-ptr, starting from the component which matches
10174 // `s.p/this->p`. This way, we'll have component-lists of
10175 // `s.p` -> `s`, and `this->p` -> `this`.
10176 for (size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) {
10177 const auto &Component = ComponentsWithAttachPtr[i];
10178 const Expr *ComponentExpr = Component.getAssociatedExpression();
10179
10180 if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr)
10181 continue;
10182 SeenAttachPtrComponent = true;
10183
10184 AttachPtrComponents.emplace_back(Args: Component.getAssociatedExpression(),
10185 Args: Component.getAssociatedDeclaration(),
10186 Args: Component.isNonContiguous());
10187 }
10188 assert(!AttachPtrComponents.empty() &&
10189 "Could not populate component-lists for mapping attach-ptr");
10190
10191 DeclComponentLists.emplace_back(
10192 Args&: AttachPtrComponents, Args: OMPC_MAP_tofrom, Args: Unknown,
10193 /*IsImplicit=*/Args: true, /*mapper=*/Args: nullptr, Args&: AttachPtrExpr);
10194 }
10195 }
10196
10197 /// For a capture that has an associated clause, generate the base pointers,
10198 /// section pointers, sizes, map types, and mappers (all included in
10199 /// \a CurCaptureVarInfo).
10200 void generateInfoForCaptureFromClauseInfo(
10201 const MapDataArrayTy &DeclComponentListsFromClauses,
10202 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
10203 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
10204 unsigned OffsetForMemberOfFlag) const {
10205 assert(!Cap->capturesVariableArrayType() &&
10206 "Not expecting to generate map info for a variable array type!");
10207
10208 // We need to know when we generating information for the first component
10209 const ValueDecl *VD = Cap->capturesThis()
10210 ? nullptr
10211 : Cap->getCapturedVar()->getCanonicalDecl();
10212
10213 // for map(to: lambda): skip here, processing it in
10214 // generateDefaultMapInfo
10215 if (LambdasMap.count(Val: VD))
10216 return;
10217
10218 // If this declaration appears in a is_device_ptr clause we just have to
10219 // pass the pointer by value. If it is a reference to a declaration, we just
10220 // pass its value.
10221 if (VD && (DevPointersMap.count(Val: VD) || HasDevAddrsMap.count(Val: VD))) {
10222 CurCaptureVarInfo.Exprs.push_back(Elt: VD);
10223 CurCaptureVarInfo.BasePointers.emplace_back(Args&: Arg);
10224 CurCaptureVarInfo.DevicePtrDecls.emplace_back(Args&: VD);
10225 CurCaptureVarInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::Pointer);
10226 CurCaptureVarInfo.Pointers.push_back(Elt: Arg);
10227 CurCaptureVarInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10228 V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy), DestTy: CGF.Int64Ty,
10229 /*isSigned=*/true));
10230 CurCaptureVarInfo.Types.push_back(
10231 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10232 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
10233 CurCaptureVarInfo.HasAttachPtr.push_back(Elt: false);
10234 CurCaptureVarInfo.Mappers.push_back(Elt: nullptr);
10235 return;
10236 }
10237
10238 auto GenerateInfoForComponentLists =
10239 [&](ArrayRef<MapData> DeclComponentListsFromClauses,
10240 bool IsEligibleForTargetParamFlag) {
10241 MapCombinedInfoTy CurInfoForComponentLists;
10242 StructRangeInfoTy PartialStruct;
10243 AttachInfoTy AttachInfo;
10244
10245 if (DeclComponentListsFromClauses.empty())
10246 return;
10247
10248 generateInfoForCaptureFromComponentLists(
10249 VD, DeclComponentLists: DeclComponentListsFromClauses, CurComponentListInfo&: CurInfoForComponentLists,
10250 PartialStruct, AttachInfo, IsListEligibleForTargetParamFlag: IsEligibleForTargetParamFlag);
10251
10252 // If there is an entry in PartialStruct it means we have a
10253 // struct with individual members mapped. Emit an extra combined
10254 // entry.
10255 if (PartialStruct.Base.isValid()) {
10256 CurCaptureVarInfo.append(CurInfo&: PartialStruct.PreliminaryMapData);
10257 emitCombinedEntry(
10258 CombinedInfo&: CurCaptureVarInfo, CurTypes&: CurInfoForComponentLists.Types,
10259 PartialStruct, AttachInfo, IsMapThis: Cap->capturesThis(), OMPBuilder,
10260 /*VD=*/nullptr, OffsetForMemberOfFlag,
10261 /*NotTargetParams*/ !IsEligibleForTargetParamFlag);
10262 }
10263
10264 // We do the appends to get the entries in the following order:
10265 // combined-entry -> individual-field-entries -> attach-entry,
10266 CurCaptureVarInfo.append(CurInfo&: CurInfoForComponentLists);
10267 if (AttachInfo.isValid())
10268 emitAttachEntry(CGF, CombinedInfo&: CurCaptureVarInfo, AttachInfo);
10269 };
10270
10271 // Group component lists by their AttachPtrExpr and process them in order
10272 // of increasing complexity (nullptr first, then simple expressions like p,
10273 // then more complex ones like p[0], etc.)
10274 //
10275 // This ensure that we:
10276 // * handle maps that can contribute towards setting the kernel argument,
10277 // (e.g. map(ps), or map(ps[0])), before any that cannot (e.g. ps->pt->d).
10278 // * allocate a single contiguous storage for all exprs with the same
10279 // captured var and having the same attach-ptr.
10280 //
10281 // Example: The map clauses below should be handled grouped together based
10282 // on their attachable-base-pointers:
10283 // map-clause | attachable-base-pointer
10284 // --------------------------+------------------------
10285 // map(p, ps) | nullptr
10286 // map(p[0]) | p
10287 // map(p[0]->b, p[0]->c) | p[0]
10288 // map(ps->d, ps->e, ps->pt) | ps
10289 // map(ps->pt->d, ps->pt->e) | ps->pt
10290
10291 // First, collect all MapData entries with their attach-ptr exprs.
10292 SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs;
10293
10294 for (const MapData &L : DeclComponentListsFromClauses) {
10295 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
10296 std::get<0>(t: L);
10297 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
10298 AttachPtrMapDataPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
10299 }
10300
10301 // Next, sort by increasing order of their complexity.
10302 llvm::stable_sort(Range&: AttachPtrMapDataPairs,
10303 C: [this](const auto &LHS, const auto &RHS) {
10304 return AttachPtrComparator(LHS.first, RHS.first);
10305 });
10306
10307 bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty();
10308 bool IsFirstGroup = true;
10309
10310 // And finally, process them all in order, grouping those with
10311 // equivalent attach-ptr exprs together.
10312 auto *It = AttachPtrMapDataPairs.begin();
10313 while (It != AttachPtrMapDataPairs.end()) {
10314 const Expr *AttachPtrExpr = It->first;
10315
10316 MapDataArrayTy GroupLists;
10317 while (It != AttachPtrMapDataPairs.end() &&
10318 (It->first == AttachPtrExpr ||
10319 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
10320 GroupLists.push_back(Elt: It->second);
10321 ++It;
10322 }
10323 assert(!GroupLists.empty() && "GroupLists should not be empty");
10324
10325 // Determine if this group of component-lists is eligible for TARGET_PARAM
10326 // flag. Only the first group processed should be eligible, and only if no
10327 // default mapping was done.
10328 bool IsEligibleForTargetParamFlag =
10329 IsFirstGroup && NoDefaultMappingDoneForVD;
10330
10331 GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag);
10332 IsFirstGroup = false;
10333 }
10334 }
10335
10336 /// Generate the base pointers, section pointers, sizes, map types, and
10337 /// mappers associated to \a DeclComponentLists for a given capture
10338 /// \a VD (all included in \a CurComponentListInfo).
10339 void generateInfoForCaptureFromComponentLists(
10340 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
10341 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
10342 AttachInfoTy &AttachInfo, bool IsListEligibleForTargetParamFlag) const {
10343 // Find overlapping elements (including the offset from the base element).
10344 llvm::SmallDenseMap<
10345 const MapData *,
10346 llvm::SmallVector<
10347 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
10348 4>
10349 OverlappedData;
10350 size_t Count = 0;
10351 for (const MapData &L : DeclComponentLists) {
10352 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10353 OpenMPMapClauseKind MapType;
10354 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10355 bool IsImplicit;
10356 const ValueDecl *Mapper;
10357 const Expr *VarRef;
10358 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10359 L;
10360 ++Count;
10361 for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(N: Count)) {
10362 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
10363 std::tie(args&: Components1, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper,
10364 args&: VarRef) = L1;
10365 auto CI = Components.rbegin();
10366 auto CE = Components.rend();
10367 auto SI = Components1.rbegin();
10368 auto SE = Components1.rend();
10369 for (; CI != CE && SI != SE; ++CI, ++SI) {
10370 if (CI->getAssociatedExpression()->getStmtClass() !=
10371 SI->getAssociatedExpression()->getStmtClass())
10372 break;
10373 // Are we dealing with different variables/fields?
10374 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
10375 break;
10376 }
10377 // Found overlapping if, at least for one component, reached the head
10378 // of the components list.
10379 if (CI == CE || SI == SE) {
10380 // Ignore it if it is the same component.
10381 if (CI == CE && SI == SE)
10382 continue;
10383 const auto It = (SI == SE) ? CI : SI;
10384 // If one component is a pointer and another one is a kind of
10385 // dereference of this pointer (array subscript, section, dereference,
10386 // etc.), it is not an overlapping.
10387 // Same, if one component is a base and another component is a
10388 // dereferenced pointer memberexpr with the same base.
10389 if (!isa<MemberExpr>(Val: It->getAssociatedExpression()) ||
10390 (std::prev(x: It)->getAssociatedDeclaration() &&
10391 std::prev(x: It)
10392 ->getAssociatedDeclaration()
10393 ->getType()
10394 ->isPointerType()) ||
10395 (It->getAssociatedDeclaration() &&
10396 It->getAssociatedDeclaration()->getType()->isPointerType() &&
10397 std::next(x: It) != CE && std::next(x: It) != SE))
10398 continue;
10399 const MapData &BaseData = CI == CE ? L : L1;
10400 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
10401 SI == SE ? Components : Components1;
10402 OverlappedData[&BaseData].push_back(Elt: SubData);
10403 }
10404 }
10405 }
10406 // Sort the overlapped elements for each item.
10407 llvm::SmallVector<const FieldDecl *, 4> Layout;
10408 if (!OverlappedData.empty()) {
10409 const Type *BaseType = VD->getType().getCanonicalType().getTypePtr();
10410 const Type *OrigType = BaseType->getPointeeOrArrayElementType();
10411 while (BaseType != OrigType) {
10412 BaseType = OrigType->getCanonicalTypeInternal().getTypePtr();
10413 OrigType = BaseType->getPointeeOrArrayElementType();
10414 }
10415
10416 if (const auto *CRD = BaseType->getAsCXXRecordDecl())
10417 getPlainLayout(RD: CRD, Layout, /*AsBase=*/false);
10418 else {
10419 const auto *RD = BaseType->getAsRecordDecl();
10420 Layout.append(in_start: RD->field_begin(), in_end: RD->field_end());
10421 }
10422 }
10423 for (auto &Pair : OverlappedData) {
10424 llvm::stable_sort(
10425 Range&: Pair.getSecond(),
10426 C: [&Layout](
10427 OMPClauseMappableExprCommon::MappableExprComponentListRef First,
10428 OMPClauseMappableExprCommon::MappableExprComponentListRef
10429 Second) {
10430 auto CI = First.rbegin();
10431 auto CE = First.rend();
10432 auto SI = Second.rbegin();
10433 auto SE = Second.rend();
10434 for (; CI != CE && SI != SE; ++CI, ++SI) {
10435 if (CI->getAssociatedExpression()->getStmtClass() !=
10436 SI->getAssociatedExpression()->getStmtClass())
10437 break;
10438 // Are we dealing with different variables/fields?
10439 if (CI->getAssociatedDeclaration() !=
10440 SI->getAssociatedDeclaration())
10441 break;
10442 }
10443
10444 // Lists contain the same elements.
10445 if (CI == CE && SI == SE)
10446 return false;
10447
10448 // List with less elements is less than list with more elements.
10449 if (CI == CE || SI == SE)
10450 return CI == CE;
10451
10452 const auto *FD1 = cast<FieldDecl>(Val: CI->getAssociatedDeclaration());
10453 const auto *FD2 = cast<FieldDecl>(Val: SI->getAssociatedDeclaration());
10454 if (FD1->getParent() == FD2->getParent())
10455 return FD1->getFieldIndex() < FD2->getFieldIndex();
10456 const auto *It =
10457 llvm::find_if(Range&: Layout, P: [FD1, FD2](const FieldDecl *FD) {
10458 return FD == FD1 || FD == FD2;
10459 });
10460 return *It == FD1;
10461 });
10462 }
10463
10464 // Associated with a capture, because the mapping flags depend on it.
10465 // Go through all of the elements with the overlapped elements.
10466 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
10467 MapCombinedInfoTy StructBaseCombinedInfo;
10468 for (const auto &Pair : OverlappedData) {
10469 const MapData &L = *Pair.getFirst();
10470 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10471 OpenMPMapClauseKind MapType;
10472 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10473 bool IsImplicit;
10474 const ValueDecl *Mapper;
10475 const Expr *VarRef;
10476 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10477 L;
10478 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
10479 OverlappedComponents = Pair.getSecond();
10480 generateInfoForComponentList(
10481 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10482 StructBaseCombinedInfo, PartialStruct, AttachInfo, IsFirstComponentList: AddTargetParamFlag,
10483 IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
10484 /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef, OverlappedElements: OverlappedComponents);
10485 AddTargetParamFlag = false;
10486 }
10487 // Go through other elements without overlapped elements.
10488 for (const MapData &L : DeclComponentLists) {
10489 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10490 OpenMPMapClauseKind MapType;
10491 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10492 bool IsImplicit;
10493 const ValueDecl *Mapper;
10494 const Expr *VarRef;
10495 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10496 L;
10497 auto It = OverlappedData.find(Val: &L);
10498 if (It == OverlappedData.end())
10499 generateInfoForComponentList(
10500 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10501 StructBaseCombinedInfo, PartialStruct, AttachInfo,
10502 IsFirstComponentList: AddTargetParamFlag, IsImplicit, /*GenerateAllInfoForClauses*/ false,
10503 Mapper, /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef,
10504 /*OverlappedElements*/ {});
10505 AddTargetParamFlag = false;
10506 }
10507 }
10508
10509 /// Check if a variable should be treated as firstprivate due to explicit
10510 /// firstprivate clause or defaultmap(firstprivate:...).
10511 bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const {
10512 // Check explicit firstprivate clauses (not implicit from defaultmap)
10513 auto I = FirstPrivateDecls.find(Val: VD);
10514 if (I != FirstPrivateDecls.end() && !I->getSecond())
10515 return true; // Explicit firstprivate only
10516
10517 // Check defaultmap(firstprivate:scalar) for scalar types
10518 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_scalar)) {
10519 if (Type->isScalarType())
10520 return true;
10521 }
10522
10523 // Check defaultmap(firstprivate:pointer) for pointer types
10524 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_pointer)) {
10525 if (Type->isAnyPointerType())
10526 return true;
10527 }
10528
10529 // Check defaultmap(firstprivate:aggregate) for aggregate types
10530 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_aggregate)) {
10531 if (Type->isAggregateType())
10532 return true;
10533 }
10534
10535 // Check defaultmap(firstprivate:all) for all types
10536 return DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_all);
10537 }
10538
10539 /// Generate the default map information for a given capture \a CI,
10540 /// record field declaration \a RI and captured value \a CV.
10541 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
10542 const FieldDecl &RI, llvm::Value *CV,
10543 MapCombinedInfoTy &CombinedInfo) const {
10544 bool IsImplicit = true;
10545 // Do the default mapping.
10546 if (CI.capturesThis()) {
10547 CombinedInfo.Exprs.push_back(Elt: nullptr);
10548 CombinedInfo.BasePointers.push_back(Elt: CV);
10549 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10550 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10551 CombinedInfo.Pointers.push_back(Elt: CV);
10552 const auto *PtrTy = cast<PointerType>(Val: RI.getType().getTypePtr());
10553 CombinedInfo.Sizes.push_back(
10554 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: PtrTy->getPointeeType()),
10555 DestTy: CGF.Int64Ty, /*isSigned=*/true));
10556 // Default map type.
10557 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TO |
10558 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
10559 } else if (CI.capturesVariableByCopy()) {
10560 const VarDecl *VD = CI.getCapturedVar();
10561 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10562 CombinedInfo.BasePointers.push_back(Elt: CV);
10563 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10564 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10565 CombinedInfo.Pointers.push_back(Elt: CV);
10566 bool IsFirstprivate =
10567 isEffectivelyFirstprivate(VD, Type: RI.getType().getNonReferenceType());
10568
10569 if (!RI.getType()->isAnyPointerType()) {
10570 // We have to signal to the runtime captures passed by value that are
10571 // not pointers.
10572 CombinedInfo.Types.push_back(
10573 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10574 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10575 V: CGF.getTypeSize(Ty: RI.getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10576 } else if (IsFirstprivate) {
10577 // Firstprivate pointers should be passed by value (as literals)
10578 // without performing a present table lookup at runtime.
10579 CombinedInfo.Types.push_back(
10580 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10581 // Use zero size for pointer literals (just passing the pointer value)
10582 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10583 } else {
10584 // Pointers are implicitly mapped with a zero size and no flags
10585 // (other than first map that is added for all implicit maps).
10586 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_NONE);
10587 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10588 }
10589 auto I = FirstPrivateDecls.find(Val: VD);
10590 if (I != FirstPrivateDecls.end())
10591 IsImplicit = I->getSecond();
10592 } else {
10593 assert(CI.capturesVariable() && "Expected captured reference.");
10594 const auto *PtrTy = cast<ReferenceType>(Val: RI.getType().getTypePtr());
10595 QualType ElementType = PtrTy->getPointeeType();
10596 const VarDecl *VD = CI.getCapturedVar();
10597 bool IsFirstprivate = isEffectivelyFirstprivate(VD, Type: ElementType);
10598 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10599 CombinedInfo.BasePointers.push_back(Elt: CV);
10600 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10601 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10602
10603 // For firstprivate pointers, pass by value instead of dereferencing
10604 if (IsFirstprivate && ElementType->isAnyPointerType()) {
10605 // Treat as a literal value (pass the pointer value itself)
10606 CombinedInfo.Pointers.push_back(Elt: CV);
10607 // Use zero size for pointer literals
10608 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10609 CombinedInfo.Types.push_back(
10610 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10611 } else {
10612 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10613 V: CGF.getTypeSize(Ty: ElementType), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10614 // The default map type for a scalar/complex type is 'to' because by
10615 // default the value doesn't have to be retrieved. For an aggregate
10616 // type, the default is 'tofrom'.
10617 CombinedInfo.Types.push_back(Elt: getMapModifiersForPrivateClauses(Cap: CI));
10618 CombinedInfo.Pointers.push_back(Elt: CV);
10619 }
10620 auto I = FirstPrivateDecls.find(Val: VD);
10621 if (I != FirstPrivateDecls.end())
10622 IsImplicit = I->getSecond();
10623 }
10624 // Every default map produces a single argument which is a target parameter.
10625 CombinedInfo.Types.back() |=
10626 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
10627
10628 // Add flag stating this is an implicit map.
10629 if (IsImplicit)
10630 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
10631
10632 CombinedInfo.HasAttachPtr.push_back(Elt: false);
10633 // No user-defined mapper for default mapping.
10634 CombinedInfo.Mappers.push_back(Elt: nullptr);
10635 }
10636};
10637} // anonymous namespace
10638
10639// Try to extract the base declaration from a `this->x` expression if possible.
10640static ValueDecl *getDeclFromThisExpr(const Expr *E) {
10641 if (!E)
10642 return nullptr;
10643
10644 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenCasts()))
10645 if (const MemberExpr *ME =
10646 dyn_cast<MemberExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))
10647 return ME->getMemberDecl();
10648 return nullptr;
10649}
10650
10651/// Emit a string constant containing the names of the values mapped to the
10652/// offloading runtime library.
10653static llvm::Constant *
10654emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
10655 MappableExprsHandler::MappingExprInfo &MapExprs) {
10656
10657 uint32_t SrcLocStrSize;
10658 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
10659 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10660
10661 SourceLocation Loc;
10662 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
10663 if (const ValueDecl *VD = getDeclFromThisExpr(E: MapExprs.getMapExpr()))
10664 Loc = VD->getLocation();
10665 else
10666 Loc = MapExprs.getMapExpr()->getExprLoc();
10667 } else {
10668 Loc = MapExprs.getMapDecl()->getLocation();
10669 }
10670
10671 std::string ExprName;
10672 if (MapExprs.getMapExpr()) {
10673 PrintingPolicy P(CGF.getContext().getLangOpts());
10674 llvm::raw_string_ostream OS(ExprName);
10675 MapExprs.getMapExpr()->printPretty(OS, Helper: nullptr, Policy: P);
10676 } else {
10677 ExprName = MapExprs.getMapDecl()->getNameAsString();
10678 }
10679
10680 std::string FileName;
10681 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
10682 if (auto *DbgInfo = CGF.getDebugInfo())
10683 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
10684 else
10685 FileName = PLoc.getFilename();
10686 return OMPBuilder.getOrCreateSrcLocStr(FunctionName: FileName, FileName: ExprName, Line: PLoc.getLine(),
10687 Column: PLoc.getColumn(), SrcLocStrSize);
10688}
10689/// Emit the arrays used to pass the captures and map information to the
10690/// offloading runtime library. If there is no map or capture information,
10691/// return nullptr by reference.
10692static void emitOffloadingArraysAndArgs(
10693 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10694 CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder,
10695 bool IsNonContiguous = false, bool ForEndCall = false) {
10696 CodeGenModule &CGM = CGF.CGM;
10697
10698 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10699 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
10700 InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
10701
10702 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
10703 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10704 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
10705 }
10706 };
10707
10708 auto CustomMapperCB = [&](unsigned int I) {
10709 llvm::Function *MFunc = nullptr;
10710 if (CombinedInfo.Mappers[I]) {
10711 Info.HasMapper = true;
10712 MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
10713 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10714 }
10715 return MFunc;
10716 };
10717 cantFail(Err: OMPBuilder.emitOffloadingArraysAndArgs(
10718 AllocaIP, CodeGenIP, Info, RTArgs&: Info.RTArgs, CombinedInfo, CustomMapperCB,
10719 IsNonContiguous, ForEndCall, DeviceAddrCB));
10720}
10721
10722/// Check for inner distribute directive.
10723static const OMPExecutableDirective *
10724getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
10725 const auto *CS = D.getInnermostCapturedStmt();
10726 const auto *Body =
10727 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
10728 const Stmt *ChildStmt =
10729 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10730
10731 if (const auto *NestedDir =
10732 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10733 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
10734 switch (D.getDirectiveKind()) {
10735 case OMPD_target:
10736 // For now, treat 'target' with nested 'teams loop' as if it's
10737 // distributed (target teams distribute).
10738 if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop)
10739 return NestedDir;
10740 if (DKind == OMPD_teams) {
10741 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
10742 /*IgnoreCaptured=*/true);
10743 if (!Body)
10744 return nullptr;
10745 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10746 if (const auto *NND =
10747 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10748 DKind = NND->getDirectiveKind();
10749 if (isOpenMPDistributeDirective(DKind))
10750 return NND;
10751 }
10752 }
10753 return nullptr;
10754 case OMPD_target_teams:
10755 if (isOpenMPDistributeDirective(DKind))
10756 return NestedDir;
10757 return nullptr;
10758 case OMPD_target_parallel:
10759 case OMPD_target_simd:
10760 case OMPD_target_loop:
10761 case OMPD_target_parallel_for:
10762 case OMPD_target_parallel_for_simd:
10763 return nullptr;
10764 case OMPD_target_teams_distribute:
10765 case OMPD_target_teams_distribute_simd:
10766 case OMPD_target_teams_distribute_parallel_for:
10767 case OMPD_target_teams_distribute_parallel_for_simd:
10768 case OMPD_parallel:
10769 case OMPD_for:
10770 case OMPD_parallel_for:
10771 case OMPD_parallel_master:
10772 case OMPD_parallel_sections:
10773 case OMPD_for_simd:
10774 case OMPD_parallel_for_simd:
10775 case OMPD_cancel:
10776 case OMPD_cancellation_point:
10777 case OMPD_ordered_standalone:
10778 case OMPD_ordered_blockassoc:
10779 case OMPD_threadprivate:
10780 case OMPD_allocate:
10781 case OMPD_task:
10782 case OMPD_simd:
10783 case OMPD_tile:
10784 case OMPD_unroll:
10785 case OMPD_sections:
10786 case OMPD_section:
10787 case OMPD_single:
10788 case OMPD_master:
10789 case OMPD_critical:
10790 case OMPD_taskyield:
10791 case OMPD_barrier:
10792 case OMPD_taskwait:
10793 case OMPD_taskgroup:
10794 case OMPD_atomic:
10795 case OMPD_flush:
10796 case OMPD_depobj:
10797 case OMPD_scan:
10798 case OMPD_teams:
10799 case OMPD_target_data:
10800 case OMPD_target_exit_data:
10801 case OMPD_target_enter_data:
10802 case OMPD_distribute:
10803 case OMPD_distribute_simd:
10804 case OMPD_distribute_parallel_for:
10805 case OMPD_distribute_parallel_for_simd:
10806 case OMPD_teams_distribute:
10807 case OMPD_teams_distribute_simd:
10808 case OMPD_teams_distribute_parallel_for:
10809 case OMPD_teams_distribute_parallel_for_simd:
10810 case OMPD_target_update:
10811 case OMPD_declare_simd:
10812 case OMPD_declare_variant:
10813 case OMPD_begin_declare_variant:
10814 case OMPD_end_declare_variant:
10815 case OMPD_declare_target:
10816 case OMPD_end_declare_target:
10817 case OMPD_declare_reduction:
10818 case OMPD_declare_mapper:
10819 case OMPD_taskloop:
10820 case OMPD_taskloop_simd:
10821 case OMPD_master_taskloop:
10822 case OMPD_master_taskloop_simd:
10823 case OMPD_parallel_master_taskloop:
10824 case OMPD_parallel_master_taskloop_simd:
10825 case OMPD_requires:
10826 case OMPD_metadirective:
10827 case OMPD_unknown:
10828 default:
10829 llvm_unreachable("Unexpected directive.");
10830 }
10831 }
10832
10833 return nullptr;
10834}
10835
10836/// Emit the user-defined mapper function. The code generation follows the
10837/// pattern in the example below.
10838/// \code
10839/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
10840/// void *base, void *begin,
10841/// int64_t size, int64_t type,
10842/// void *name = nullptr) {
10843/// // Allocate space for an array section first.
10844/// if ((size > 1 || (base != begin)) && !maptype.IsDelete)
10845/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10846/// size*sizeof(Ty), clearToFromMember(type));
10847/// // Map members.
10848/// for (unsigned i = 0; i < size; i++) {
10849/// N = __tgt_mapper_num_components(rt_mapper_handle);
10850/// // For each component specified by this mapper:
10851/// for (auto c : begin[i]->all_components) {
10852/// // MEMBER_OF grouping: tie this component to the current array element
10853/// // (component N) by adding N<<48. Exceptions:
10854/// // - ATTACH entries are not members of any struct storage range.
10855/// // - Pointee entries (reached via a pointer member) occupy separate
10856/// // storage; their inner MEMBER_OF bits are shifted by N instead.
10857/// if (c.isAttach() || c.isPointee())
10858/// member_type = c.arg_type + (c.hasInnerMemberOf() ? N<<48 : 0);
10859/// else
10860/// member_type = c.arg_type + N<<48;
10861/// // Map-type-modifying bits (ALWAYS, DELETE, CLOSE) from the outer map
10862/// // clause are propagated to each component, except ATTACH entries
10863/// // (ATTACH|ALWAYS is reserved for attach(always), and other modifier
10864/// // bits have no meaning for ATTACH). PRESENT is additionally
10865/// // propagated to components with HasAttachPtr (the pointee data) at
10866/// // OpenMP >= 6.0.
10867/// present_bit = (v60 && c.hasAttachPtr()) ? PRESENT : 0;
10868/// imported_modifier_bits =
10869/// type & (ALWAYS | DELETE | CLOSE | present_bit);
10870/// effective_type = c.isAttach() ? member_type
10871/// : member_type | imported_modifier_bits;
10872/// if (c.hasMapper())
10873/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
10874/// effective_type, c.arg_name);
10875/// else
10876/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
10877/// c.arg_begin, c.arg_size, effective_type,
10878/// c.arg_name);
10879/// }
10880/// }
10881/// // Delete the array section.
10882/// if (size > 1 && maptype.IsDelete)
10883/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10884/// size*sizeof(Ty), clearToFromMember(type));
10885/// }
10886/// \endcode
10887void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
10888 CodeGenFunction *CGF) {
10889 if (UDMMap.count(Val: D) > 0)
10890 return;
10891 ASTContext &C = CGM.getContext();
10892 QualType Ty = D->getType();
10893 auto *MapperVarDecl =
10894 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getMapperVarRef())->getDecl());
10895 CharUnits ElementSize = C.getTypeSizeInChars(T: Ty);
10896 llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
10897
10898 CodeGenFunction MapperCGF(CGM);
10899 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10900 auto PrivatizeAndGenMapInfoCB =
10901 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10902 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10903 MapperCGF.Builder.restoreIP(IP: CodeGenIP);
10904
10905 // Privatize the declared variable of mapper to be the current array
10906 // element.
10907 Address PtrCurrent(
10908 PtrPHI, ElemTy,
10909 Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign())
10910 .getAlignment()
10911 .alignmentOfArrayElement(elementSize: ElementSize));
10912 CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
10913 Scope.addPrivate(LocalVD: MapperVarDecl, Addr: PtrCurrent);
10914 (void)Scope.Privatize();
10915
10916 // Get map clause information.
10917 MappableExprsHandler MEHandler(*D, MapperCGF);
10918 MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder);
10919
10920 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10921 return emitMappingInformation(CGF&: MapperCGF, OMPBuilder, MapExprs&: MapExpr);
10922 };
10923 if (CGM.getCodeGenOpts().getDebugInfo() !=
10924 llvm::codegenoptions::NoDebugInfo) {
10925 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10926 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10927 F: FillInfoMap);
10928 }
10929
10930 return CombinedInfo;
10931 };
10932
10933 auto CustomMapperCB = [&](unsigned I) {
10934 llvm::Function *MapperFunc = nullptr;
10935 if (CombinedInfo.Mappers[I]) {
10936 // Call the corresponding mapper function.
10937 MapperFunc = getOrCreateUserDefinedMapperFunc(
10938 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10939 assert(MapperFunc && "Expect a valid mapper function is available.");
10940 }
10941 return MapperFunc;
10942 };
10943
10944 SmallString<64> TyStr;
10945 llvm::raw_svector_ostream Out(TyStr);
10946 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out);
10947 std::string Name = getName(Parts: {"omp_mapper", TyStr, D->getName()});
10948
10949 // Propagate the PRESENT modifier to the pointee entries (those with
10950 // HasAttachPtr) only for OpenMP >= 6.0; before 6.0 the present modifier does
10951 // not apply to the pointee (see the OpenMP 6.0 erratum on the present motion
10952 // vs. map-type modifier divergence).
10953 bool PropagatePresentToPointeeStorage = CGM.getLangOpts().OpenMP >= 60;
10954 // Keep non-pointee storage present propagation disabled until Clang stops
10955 // emitting PTR_AND_OBJ maps and uses attach-style maps throughout.
10956 bool PropagatePresentToNonPointeeStorage = false;
10957 llvm::Function *NewFn = cantFail(ValOrErr: OMPBuilder.emitUserDefinedMapper(
10958 PrivAndGenMapInfoCB: PrivatizeAndGenMapInfoCB, ElemTy, FuncName: Name, CustomMapperCB,
10959 /*PreserveMemberOfFlags=*/false, PropagatePresentToPointeeStorage,
10960 PropagatePresentToNonPointeeStorage));
10961 UDMMap.try_emplace(Key: D, Args&: NewFn);
10962 if (CGF)
10963 FunctionUDMMap[CGF->CurFn].push_back(Elt: D);
10964}
10965
10966llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc(
10967 const OMPDeclareMapperDecl *D) {
10968 auto I = UDMMap.find(Val: D);
10969 if (I != UDMMap.end())
10970 return I->second;
10971 emitUserDefinedMapper(D);
10972 return UDMMap.lookup(Val: D);
10973}
10974
10975llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall(
10976 CodeGenFunction &CGF, const OMPExecutableDirective &D,
10977 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10978 const OMPLoopDirective &D)>
10979 SizeEmitter) {
10980 OpenMPDirectiveKind Kind = D.getDirectiveKind();
10981 const OMPExecutableDirective *TD = &D;
10982 // Get nested teams distribute kind directive, if any. For now, treat
10983 // 'target_teams_loop' as if it's really a target_teams_distribute.
10984 if ((!isOpenMPDistributeDirective(DKind: Kind) || !isOpenMPTeamsDirective(DKind: Kind)) &&
10985 Kind != OMPD_target_teams_loop)
10986 TD = getNestedDistributeDirective(Ctx&: CGM.getContext(), D);
10987 if (!TD)
10988 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10989
10990 const auto *LD = cast<OMPLoopDirective>(Val: TD);
10991 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
10992 return NumIterations;
10993 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10994}
10995
10996static void
10997emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10998 const OMPExecutableDirective &D,
10999 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11000 bool RequiresOuterTask, const CapturedStmt &CS,
11001 bool OffloadingMandatory, CodeGenFunction &CGF) {
11002 if (OffloadingMandatory) {
11003 CGF.Builder.CreateUnreachable();
11004 } else {
11005 if (RequiresOuterTask) {
11006 CapturedVars.clear();
11007 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
11008 }
11009 llvm::SmallVector<llvm::Value *, 16> Args(CapturedVars.begin(),
11010 CapturedVars.end());
11011 Args.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy()));
11012 OMPRuntime->emitOutlinedFunctionCall(CGF, Loc: D.getBeginLoc(), OutlinedFn,
11013 Args);
11014 }
11015}
11016
11017static llvm::Value *emitDeviceID(
11018 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11019 CodeGenFunction &CGF) {
11020 // Emit device ID if any.
11021 llvm::Value *DeviceID;
11022 if (Device.getPointer()) {
11023 assert((Device.getInt() == OMPC_DEVICE_unknown ||
11024 Device.getInt() == OMPC_DEVICE_device_num) &&
11025 "Expected device_num modifier.");
11026 llvm::Value *DevVal = CGF.EmitScalarExpr(E: Device.getPointer());
11027 DeviceID =
11028 CGF.Builder.CreateIntCast(V: DevVal, DestTy: CGF.Int64Ty, /*isSigned=*/true);
11029 } else {
11030 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11031 }
11032 return DeviceID;
11033}
11034
11035static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>
11036emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) {
11037 llvm::Value *DynGP = CGF.Builder.getInt32(C: 0);
11038 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11039
11040 if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) {
11041 CodeGenFunction::RunCleanupsScope DynGPScope(CGF);
11042 llvm::Value *DynGPVal =
11043 CGF.EmitScalarExpr(E: DynGPClause->getSize(), /*IgnoreResultAssign=*/true);
11044 DynGP = CGF.Builder.CreateIntCast(V: DynGPVal, DestTy: CGF.Int32Ty,
11045 /*isSigned=*/false);
11046 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();
11047 switch (FallbackModifier) {
11048 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:
11049 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
11050 break;
11051 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:
11052 DynGPFallback = OMPDynGroupprivateFallbackType::Null;
11053 break;
11054 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:
11055 case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown:
11056 // This is the default for dyn_groupprivate.
11057 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;
11058 break;
11059 default:
11060 llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate");
11061 }
11062 } else if (auto *OMPXDynCGClause =
11063 D.getSingleClause<OMPXDynCGroupMemClause>()) {
11064 CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF);
11065 llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(E: OMPXDynCGClause->getSize(),
11066 /*IgnoreResultAssign=*/true);
11067 DynGP = CGF.Builder.CreateIntCast(V: DynCGMemVal, DestTy: CGF.Int32Ty,
11068 /*isSigned=*/false);
11069 }
11070 return {DynGP, DynGPFallback};
11071}
11072
11073static void genMapInfoForCaptures(
11074 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
11075 const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11076 llvm::OpenMPIRBuilder &OMPBuilder,
11077 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet,
11078 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11079
11080 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
11081 auto RI = CS.getCapturedRecordDecl()->field_begin();
11082 auto *CV = CapturedVars.begin();
11083 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
11084 CE = CS.capture_end();
11085 CI != CE; ++CI, ++RI, ++CV) {
11086 MappableExprsHandler::MapCombinedInfoTy CurInfo;
11087
11088 // VLA sizes are passed to the outlined region by copy and do not have map
11089 // information associated.
11090 if (CI->capturesVariableArrayType()) {
11091 CurInfo.Exprs.push_back(Elt: nullptr);
11092 CurInfo.BasePointers.push_back(Elt: *CV);
11093 CurInfo.DevicePtrDecls.push_back(Elt: nullptr);
11094 CurInfo.DevicePointers.push_back(
11095 Elt: MappableExprsHandler::DeviceInfoTy::None);
11096 CurInfo.Pointers.push_back(Elt: *CV);
11097 CurInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
11098 V: CGF.getTypeSize(Ty: RI->getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
11099 // Copy to the device as an argument. No need to retrieve it.
11100 CurInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
11101 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11102 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
11103 CurInfo.HasAttachPtr.push_back(Elt: false);
11104 CurInfo.Mappers.push_back(Elt: nullptr);
11105 } else {
11106 const ValueDecl *CapturedVD =
11107 CI->capturesThis() ? nullptr
11108 : CI->getCapturedVar()->getCanonicalDecl();
11109 bool HasEntryWithCVAsAttachPtr = false;
11110 if (CapturedVD)
11111 HasEntryWithCVAsAttachPtr =
11112 MEHandler.hasAttachEntryForCapturedVar(VD: CapturedVD);
11113
11114 // Populate component lists for the captured variable from clauses.
11115 MappableExprsHandler::MapDataArrayTy DeclComponentLists;
11116 SmallVector<
11117 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>, 4>
11118 StorageForImplicitlyAddedComponentLists;
11119 MEHandler.populateComponentListsForNonLambdaCaptureFromClauses(
11120 VD: CapturedVD, DeclComponentLists,
11121 StorageForImplicitlyAddedComponentLists);
11122
11123 // OpenMP 6.0, 15.8, target construct, restrictions:
11124 // * A list item in a map clause that is specified on a target construct
11125 // must have a base variable or base pointer.
11126 //
11127 // Map clauses on a target construct must either have a base pointer, or a
11128 // base-variable. So, if we don't have a base-pointer, that means that it
11129 // must have a base-variable, i.e. we have a map like `map(s)`, `map(s.x)`
11130 // etc. In such cases, we do not need to handle default map generation
11131 // for `s`.
11132 bool HasEntryWithoutAttachPtr =
11133 llvm::any_of(Range&: DeclComponentLists, P: [&](const auto &MapData) {
11134 OMPClauseMappableExprCommon::MappableExprComponentListRef
11135 Components = std::get<0>(MapData);
11136 return !MEHandler.getAttachPtrExpr(Components);
11137 });
11138
11139 // Generate default map info first if there's no direct map with CV as
11140 // the base-variable, or attach pointer.
11141 if (DeclComponentLists.empty() ||
11142 (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr))
11143 MEHandler.generateDefaultMapInfo(CI: *CI, RI: **RI, CV: *CV, CombinedInfo&: CurInfo);
11144
11145 // If we have any information in the map clause, we use it, otherwise we
11146 // just do a default mapping.
11147 MEHandler.generateInfoForCaptureFromClauseInfo(
11148 DeclComponentListsFromClauses: DeclComponentLists, Cap: CI, Arg: *CV, CurCaptureVarInfo&: CurInfo, OMPBuilder,
11149 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());
11150
11151 if (!CI->capturesThis())
11152 MappedVarSet.insert(V: CI->getCapturedVar());
11153 else
11154 MappedVarSet.insert(V: nullptr);
11155
11156 // Generate correct mapping for variables captured by reference in
11157 // lambdas.
11158 if (CI->capturesVariable())
11159 MEHandler.generateInfoForLambdaCaptures(VD: CI->getCapturedVar(), Arg: *CV,
11160 CombinedInfo&: CurInfo, LambdaPointers);
11161 }
11162 // We expect to have at least an element of information for this capture.
11163 assert(!CurInfo.BasePointers.empty() &&
11164 "Non-existing map pointer for capture!");
11165 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
11166 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
11167 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
11168 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
11169 "Inconsistent map information sizes!");
11170
11171 // We need to append the results of this capture to what we already have.
11172 CombinedInfo.append(CurInfo);
11173 }
11174 // Adjust MEMBER_OF flags for the lambdas captures.
11175 MEHandler.adjustMemberOfForLambdaCaptures(
11176 OMPBuilder, LambdaPointers, BasePointers&: CombinedInfo.BasePointers,
11177 Pointers&: CombinedInfo.Pointers, Types&: CombinedInfo.Types);
11178}
11179static void
11180genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
11181 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
11182 llvm::OpenMPIRBuilder &OMPBuilder,
11183 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet =
11184 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) {
11185
11186 CodeGenModule &CGM = CGF.CGM;
11187 // Map any list items in a map clause that were not captures because they
11188 // weren't referenced within the construct.
11189 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkipVarSet: SkippedVarSet);
11190
11191 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11192 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
11193 };
11194 if (CGM.getCodeGenOpts().getDebugInfo() !=
11195 llvm::codegenoptions::NoDebugInfo) {
11196 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
11197 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
11198 F: FillInfoMap);
11199 }
11200}
11201
11202static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
11203 const CapturedStmt &CS,
11204 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11205 llvm::OpenMPIRBuilder &OMPBuilder,
11206 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
11207 // Get mappable expression information.
11208 MappableExprsHandler MEHandler(D, CGF);
11209 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
11210
11211 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,
11212 MappedVarSet, CombinedInfo);
11213 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, SkippedVarSet: MappedVarSet);
11214}
11215
11216template <typename ClauseTy>
11217static void
11218emitClauseForBareTargetDirective(CodeGenFunction &CGF,
11219 const OMPExecutableDirective &D,
11220 llvm::SmallVectorImpl<llvm::Value *> &Values) {
11221 const auto *C = D.getSingleClause<ClauseTy>();
11222 assert(!C->varlist_empty() &&
11223 "ompx_bare requires explicit num_teams and thread_limit");
11224 CodeGenFunction::RunCleanupsScope Scope(CGF);
11225 for (auto *E : C->varlist()) {
11226 llvm::Value *V = CGF.EmitScalarExpr(E);
11227 Values.push_back(
11228 Elt: CGF.Builder.CreateIntCast(V, DestTy: CGF.Int32Ty, /*isSigned=*/true));
11229 }
11230}
11231
11232static void emitTargetCallKernelLaunch(
11233 CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11234 const OMPExecutableDirective &D,
11235 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask,
11236 const CapturedStmt &CS, bool OffloadingMandatory,
11237 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11238 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
11239 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
11240 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11241 const OMPLoopDirective &D)>
11242 SizeEmitter,
11243 CodeGenFunction &CGF, CodeGenModule &CGM) {
11244 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();
11245
11246 // Fill up the arrays with all the captured variables.
11247 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11248 CGOpenMPRuntime::TargetDataInfo Info;
11249 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
11250
11251 // Append a null entry for the implicit dyn_ptr argument.
11252 using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
11253 auto *NullPtr = llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy());
11254 CombinedInfo.BasePointers.push_back(Elt: NullPtr);
11255 CombinedInfo.Pointers.push_back(Elt: NullPtr);
11256 CombinedInfo.DevicePointers.push_back(
11257 Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
11258 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.getInt64(C: 0));
11259 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11260 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
11261 CombinedInfo.HasAttachPtr.push_back(Elt: false);
11262 if (!CombinedInfo.Names.empty())
11263 CombinedInfo.Names.push_back(Elt: NullPtr);
11264 CombinedInfo.Exprs.push_back(Elt: nullptr);
11265 CombinedInfo.Mappers.push_back(Elt: nullptr);
11266 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
11267
11268 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11269 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11270
11271 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11272 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11273 CGF.VoidPtrTy, CGM.getPointerAlign());
11274 InputInfo.PointersArray =
11275 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11276 InputInfo.SizesArray =
11277 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11278 InputInfo.MappersArray =
11279 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11280 MapTypesArray = Info.RTArgs.MapTypesArray;
11281 MapNamesArray = Info.RTArgs.MapNamesArray;
11282
11283 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
11284 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11285 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
11286 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
11287 bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
11288
11289 if (IsReverseOffloading) {
11290 // Reverse offloading is not supported, so just execute on the host.
11291 // FIXME: This fallback solution is incorrect since it ignores the
11292 // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to
11293 // assert here and ensure SEMA emits an error.
11294 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11295 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11296 return;
11297 }
11298
11299 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
11300 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
11301
11302 llvm::Value *BasePointersArray =
11303 InputInfo.BasePointersArray.emitRawPointer(CGF);
11304 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
11305 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
11306 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
11307
11308 auto &&EmitTargetCallFallbackCB =
11309 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11310 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
11311 -> llvm::OpenMPIRBuilder::InsertPointTy {
11312 CGF.Builder.restoreIP(IP);
11313 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11314 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11315 return CGF.Builder.saveIP();
11316 };
11317
11318 bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
11319 SmallVector<llvm::Value *, 3> NumTeams;
11320 SmallVector<llvm::Value *, 3> NumThreads;
11321 if (IsBare) {
11322 emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, Values&: NumTeams);
11323 emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
11324 Values&: NumThreads);
11325 } else {
11326 NumTeams.push_back(Elt: OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
11327 NumThreads.push_back(
11328 Elt: OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
11329 }
11330
11331 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
11332 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11333 llvm::Value *NumIterations =
11334 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
11335 auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF);
11336 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
11337 CGF.AllocaInsertPt->getIterator());
11338
11339 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
11340 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
11341 nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray);
11342
11343 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
11344 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
11345 DynCGroupMem, HasNoWait, /*StrictBlocks=*/IsBare,
11346 /*StrictThreads=*/IsBare, DynCGroupMemFallback);
11347
11348 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11349 cantFail(ValOrErr: OMPRuntime->getOMPBuilder().emitKernelLaunch(
11350 Loc: CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
11351 RTLoc, AllocaIP));
11352 CGF.Builder.restoreIP(IP: AfterIP);
11353 };
11354
11355 if (RequiresOuterTask)
11356 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11357 else
11358 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11359}
11360
11361static void
11362emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11363 const OMPExecutableDirective &D,
11364 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11365 bool RequiresOuterTask, const CapturedStmt &CS,
11366 bool OffloadingMandatory, CodeGenFunction &CGF) {
11367
11368 // Notify that the host version must be executed.
11369 auto &&ElseGen =
11370 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11371 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11372 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11373 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11374 };
11375
11376 if (RequiresOuterTask) {
11377 CodeGenFunction::OMPTargetDataInfo InputInfo;
11378 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ElseGen, InputInfo);
11379 } else {
11380 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ElseGen);
11381 }
11382}
11383
11384void CGOpenMPRuntime::emitTargetCall(
11385 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11386 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11387 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11388 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11389 const OMPLoopDirective &D)>
11390 SizeEmitter) {
11391 if (!CGF.HaveInsertPoint())
11392 return;
11393
11394 const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice &&
11395 CGM.getLangOpts().OpenMPOffloadMandatory;
11396
11397 assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
11398
11399 const bool RequiresOuterTask =
11400 D.hasClausesOfKind<OMPDependClause>() ||
11401 D.hasClausesOfKind<OMPNowaitClause>() ||
11402 D.hasClausesOfKind<OMPInReductionClause>() ||
11403 (CGM.getLangOpts().OpenMP >= 51 &&
11404 needsTaskBasedThreadLimit(DKind: D.getDirectiveKind()) &&
11405 D.hasClausesOfKind<OMPThreadLimitClause>());
11406 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
11407 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
11408 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
11409 PrePostActionTy &) {
11410 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
11411 };
11412 emitInlinedDirective(CGF, InnerKind: OMPD_unknown, CodeGen: ArgsCodegen);
11413
11414 CodeGenFunction::OMPTargetDataInfo InputInfo;
11415 llvm::Value *MapTypesArray = nullptr;
11416 llvm::Value *MapNamesArray = nullptr;
11417
11418 auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,
11419 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11420 OutlinedFnID, &InputInfo, &MapTypesArray,
11421 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
11422 PrePostActionTy &) {
11423 emitTargetCallKernelLaunch(OMPRuntime: this, OutlinedFn, D, CapturedVars,
11424 RequiresOuterTask, CS, OffloadingMandatory,
11425 Device, OutlinedFnID, InputInfo, MapTypesArray,
11426 MapNamesArray, SizeEmitter, CGF, CGM);
11427 };
11428
11429 auto &&TargetElseGen =
11430 [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11431 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11432 emitTargetCallElse(OMPRuntime: this, OutlinedFn, D, CapturedVars, RequiresOuterTask,
11433 CS, OffloadingMandatory, CGF);
11434 };
11435
11436 // If we have a target function ID it means that we need to support
11437 // offloading, otherwise, just execute on the host. We need to execute on host
11438 // regardless of the conditional in the if clause if, e.g., the user do not
11439 // specify target triples.
11440 if (OutlinedFnID) {
11441 if (IfCond) {
11442 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen, ElseGen: TargetElseGen);
11443 } else {
11444 RegionCodeGenTy ThenRCG(TargetThenGen);
11445 ThenRCG(CGF);
11446 }
11447 } else {
11448 RegionCodeGenTy ElseRCG(TargetElseGen);
11449 ElseRCG(CGF);
11450 }
11451}
11452
11453void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
11454 StringRef ParentName) {
11455 if (!S)
11456 return;
11457
11458 // Register vtable from device for target data and target directives.
11459 // Add this block here since scanForTargetRegionsFunctions ignores
11460 // target data by checking if S is a executable directive (target).
11461 if (auto *E = dyn_cast<OMPExecutableDirective>(Val: S);
11462 E && isOpenMPTargetDataManagementDirective(DKind: E->getDirectiveKind())) {
11463 // Don't need to check if it's device compile
11464 // since scanForTargetRegionsFunctions currently only called
11465 // in device compilation.
11466 registerVTable(D: *E);
11467 }
11468
11469 // Codegen OMP target directives that offload compute to the device.
11470 bool RequiresDeviceCodegen =
11471 isa<OMPExecutableDirective>(Val: S) &&
11472 isOpenMPTargetExecutionDirective(
11473 DKind: cast<OMPExecutableDirective>(Val: S)->getDirectiveKind());
11474
11475 if (RequiresDeviceCodegen) {
11476 const auto &E = *cast<OMPExecutableDirective>(Val: S);
11477
11478 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
11479 CGM, OMPBuilder, BeginLoc: E.getBeginLoc(), ParentName);
11480
11481 // Is this a target region that should not be emitted as an entry point? If
11482 // so just signal we are done with this target region.
11483 if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
11484 return;
11485
11486 switch (E.getDirectiveKind()) {
11487 case OMPD_target:
11488 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
11489 S: cast<OMPTargetDirective>(Val: E));
11490 break;
11491 case OMPD_target_parallel:
11492 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
11493 CGM, ParentName, S: cast<OMPTargetParallelDirective>(Val: E));
11494 break;
11495 case OMPD_target_teams:
11496 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
11497 CGM, ParentName, S: cast<OMPTargetTeamsDirective>(Val: E));
11498 break;
11499 case OMPD_target_teams_distribute:
11500 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
11501 CGM, ParentName, S: cast<OMPTargetTeamsDistributeDirective>(Val: E));
11502 break;
11503 case OMPD_target_teams_distribute_simd:
11504 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
11505 CGM, ParentName, S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: E));
11506 break;
11507 case OMPD_target_parallel_for:
11508 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
11509 CGM, ParentName, S: cast<OMPTargetParallelForDirective>(Val: E));
11510 break;
11511 case OMPD_target_parallel_for_simd:
11512 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
11513 CGM, ParentName, S: cast<OMPTargetParallelForSimdDirective>(Val: E));
11514 break;
11515 case OMPD_target_simd:
11516 case OMPD_target_loop:
11517 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
11518 CGM, ParentName, S: cast<OMPLoopDirective>(Val: E));
11519 break;
11520 case OMPD_target_teams_distribute_parallel_for:
11521 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
11522 CGM, ParentName,
11523 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: E));
11524 break;
11525 case OMPD_target_teams_distribute_parallel_for_simd:
11526 CodeGenFunction::
11527 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
11528 CGM, ParentName,
11529 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: E));
11530 break;
11531 case OMPD_target_teams_loop:
11532 CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
11533 CGM, ParentName, S: cast<OMPTargetTeamsGenericLoopDirective>(Val: E));
11534 break;
11535 case OMPD_target_parallel_loop:
11536 CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
11537 CGM, ParentName, S: cast<OMPTargetParallelGenericLoopDirective>(Val: E));
11538 break;
11539 case OMPD_parallel:
11540 case OMPD_for:
11541 case OMPD_parallel_for:
11542 case OMPD_parallel_master:
11543 case OMPD_parallel_sections:
11544 case OMPD_for_simd:
11545 case OMPD_parallel_for_simd:
11546 case OMPD_cancel:
11547 case OMPD_cancellation_point:
11548 case OMPD_ordered_standalone:
11549 case OMPD_ordered_blockassoc:
11550 case OMPD_threadprivate:
11551 case OMPD_allocate:
11552 case OMPD_task:
11553 case OMPD_simd:
11554 case OMPD_tile:
11555 case OMPD_unroll:
11556 case OMPD_sections:
11557 case OMPD_section:
11558 case OMPD_single:
11559 case OMPD_master:
11560 case OMPD_critical:
11561 case OMPD_taskyield:
11562 case OMPD_barrier:
11563 case OMPD_taskwait:
11564 case OMPD_taskgroup:
11565 case OMPD_atomic:
11566 case OMPD_flush:
11567 case OMPD_depobj:
11568 case OMPD_scan:
11569 case OMPD_teams:
11570 case OMPD_target_data:
11571 case OMPD_target_exit_data:
11572 case OMPD_target_enter_data:
11573 case OMPD_distribute:
11574 case OMPD_distribute_simd:
11575 case OMPD_distribute_parallel_for:
11576 case OMPD_distribute_parallel_for_simd:
11577 case OMPD_teams_distribute:
11578 case OMPD_teams_distribute_simd:
11579 case OMPD_teams_distribute_parallel_for:
11580 case OMPD_teams_distribute_parallel_for_simd:
11581 case OMPD_target_update:
11582 case OMPD_declare_simd:
11583 case OMPD_declare_variant:
11584 case OMPD_begin_declare_variant:
11585 case OMPD_end_declare_variant:
11586 case OMPD_declare_target:
11587 case OMPD_end_declare_target:
11588 case OMPD_declare_reduction:
11589 case OMPD_declare_mapper:
11590 case OMPD_taskloop:
11591 case OMPD_taskloop_simd:
11592 case OMPD_master_taskloop:
11593 case OMPD_master_taskloop_simd:
11594 case OMPD_parallel_master_taskloop:
11595 case OMPD_parallel_master_taskloop_simd:
11596 case OMPD_requires:
11597 case OMPD_metadirective:
11598 case OMPD_unknown:
11599 default:
11600 llvm_unreachable("Unknown target directive for OpenMP device codegen.");
11601 }
11602 return;
11603 }
11604
11605 // A metadirective has no associated statement; look into its variants.
11606 if (const auto *MD = dyn_cast<OMPMetaDirective>(Val: S)) {
11607 for (const Stmt *V : MD->variants())
11608 scanForTargetRegionsFunctions(S: V, ParentName);
11609 return;
11610 }
11611
11612 if (const auto *E = dyn_cast<OMPExecutableDirective>(Val: S)) {
11613 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
11614 return;
11615
11616 scanForTargetRegionsFunctions(S: E->getRawStmt(), ParentName);
11617 return;
11618 }
11619
11620 // Default arguments and member initializers are not children of their uses.
11621 if (const auto *E = dyn_cast<CXXDefaultArgExpr>(Val: S)) {
11622 scanForTargetRegionsFunctions(S: E->getExpr(), ParentName);
11623 return;
11624 }
11625 if (const auto *E = dyn_cast<CXXDefaultInitExpr>(Val: S)) {
11626 scanForTargetRegionsFunctions(S: E->getExpr(), ParentName);
11627 return;
11628 }
11629
11630 // Implicit and inheriting constructors are only reachable from their uses.
11631 const CXXConstructorDecl *Ctor = nullptr;
11632 if (const auto *E = dyn_cast<CXXConstructExpr>(Val: S))
11633 Ctor = E->getConstructor();
11634 else if (const auto *E = dyn_cast<CXXInheritedCtorInitExpr>(Val: S))
11635 Ctor = E->getConstructor();
11636 if (Ctor && (!Ctor->isUserProvided() || Ctor->isInheritingConstructor()) &&
11637 Ctor->hasBody() && ScannedImplicitCtors.insert(Ptr: Ctor).second)
11638 scanConstructorForTargetRegions(Ctor, ParentName);
11639
11640 // If this is a lambda function, look into its body.
11641 if (const auto *L = dyn_cast<LambdaExpr>(Val: S)) {
11642 const CXXMethodDecl *CallOp = L->getCallOperator();
11643 // Specializations of a generic lambda are scanned as top-level decls.
11644 if (CallOp->isTemplated())
11645 return;
11646 ParentName = getTargetRegionParentName(FD: CallOp);
11647 S = L->getBody();
11648 }
11649
11650 // Keep looking for target regions recursively.
11651 for (const Stmt *II : S->children())
11652 scanForTargetRegionsFunctions(S: II, ParentName);
11653}
11654
11655void CGOpenMPRuntime::scanConstructorForTargetRegions(
11656 const CXXConstructorDecl *Ctor, StringRef ParentName) {
11657 const auto *Def = dyn_cast_or_null<CXXConstructorDecl>(Val: Ctor->getDefinition());
11658 if (!Def)
11659 return;
11660 for (const CXXCtorInitializer *Init : Def->inits())
11661 scanForTargetRegionsFunctions(S: Init->getInit(), ParentName);
11662 scanForTargetRegionsFunctions(S: Def->getBody(), ParentName);
11663}
11664
11665static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) {
11666 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
11667 OMPDeclareTargetDeclAttr::getDeviceType(VD);
11668 if (!DevTy)
11669 return false;
11670 // Do not emit device_type(nohost) functions for the host.
11671 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
11672 return true;
11673 // Do not emit device_type(host) functions for the device.
11674 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
11675 return true;
11676 return false;
11677}
11678
11679bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
11680 // If emitting code for the host, we do not process FD here. Instead we do
11681 // the normal code generation.
11682 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
11683 if (const auto *FD = dyn_cast<FunctionDecl>(Val: GD.getDecl()))
11684 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11685 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11686 return true;
11687 return false;
11688 }
11689
11690 const ValueDecl *VD = cast<ValueDecl>(Val: GD.getDecl());
11691 // Try to detect target regions in the function.
11692 if (const auto *FD = dyn_cast<FunctionDecl>(Val: VD)) {
11693 StringRef Name = getTargetRegionParentName(FD);
11694 if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(Val: FD))
11695 scanConstructorForTargetRegions(Ctor, ParentName: Name);
11696 else
11697 scanForTargetRegionsFunctions(S: FD->getBody(), ParentName: Name);
11698 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11699 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11700 return true;
11701 }
11702
11703 // Do not emit function if it is not marked as declare target.
11704 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
11705 AlreadyEmittedTargetDecls.count(V: VD) == 0;
11706}
11707
11708bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
11709 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
11710 return isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: GD.getDecl()),
11711 /*IsDevice=*/false);
11712
11713 // Check if there are Ctors/Dtors in this declaration and look for target
11714 // regions in it. We use the complete variant to produce the kernel name
11715 // mangling.
11716 const auto *VD = cast<VarDecl>(Val: GD.getDecl());
11717 QualType RDTy = VD->getType();
11718 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
11719 for (const CXXConstructorDecl *Ctor : RD->ctors())
11720 scanConstructorForTargetRegions(Ctor, ParentName: getTargetRegionParentName(FD: Ctor));
11721 if (const CXXDestructorDecl *Dtor = RD->getDestructor())
11722 scanForTargetRegionsFunctions(S: Dtor->getBody(),
11723 ParentName: getTargetRegionParentName(FD: Dtor));
11724 }
11725
11726 // Lambdas in the initializer are reached from nowhere else.
11727 if (const Expr *Init = VD->getInit())
11728 scanForTargetRegionsFunctions(S: Init, ParentName: CGM.getMangledName(GD));
11729
11730 if (isAssumedToBeNotEmitted(VD, /*IsDevice=*/true))
11731 return true;
11732
11733 // Do not emit variable if it is not marked as declare target.
11734 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11735 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11736 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
11737 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11738 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11739 HasRequiresUnifiedSharedMemory)) {
11740 DeferredGlobalVariables.insert(V: VD);
11741 return true;
11742 }
11743 return false;
11744}
11745
11746void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
11747 llvm::Constant *Addr) {
11748 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
11749 !CGM.getLangOpts().OpenMPIsTargetDevice)
11750 return;
11751
11752 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11753 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11754
11755 // If this is an 'extern' declaration we defer to the canonical definition and
11756 // do not emit an offloading entry.
11757 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
11758 VD->hasExternalStorage())
11759 return;
11760
11761 // MT_Local variables use direct access with no host-device mapping.
11762 // No offload entry needed — the device global keeps its own initializer.
11763 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local)
11764 return;
11765
11766 if (!Res) {
11767 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11768 // Register non-target variables being emitted in device code (debug info
11769 // may cause this).
11770 StringRef VarName = CGM.getMangledName(GD: VD);
11771 EmittedNonTargetVariables.try_emplace(Key: VarName, Args&: Addr);
11772 }
11773 return;
11774 }
11775
11776 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
11777 auto LinkageForVariable = [&VD, this]() {
11778 return CGM.getLLVMLinkageVarDefinition(VD);
11779 };
11780
11781 std::vector<llvm::GlobalVariable *> GeneratedRefs;
11782 OMPBuilder.registerTargetGlobalVariable(
11783 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
11784 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
11785 IsExternallyVisible: VD->isExternallyVisible(),
11786 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
11787 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
11788 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
11789 TargetTriple: CGM.getLangOpts().OMPTargetTriples, GlobalInitializer: AddrOfGlobal, VariableLinkage: LinkageForVariable,
11790 LlvmPtrTy: CGM.getTypes().ConvertTypeForMem(
11791 T: CGM.getContext().getPointerType(T: VD->getType())),
11792 Addr);
11793
11794 for (auto *ref : GeneratedRefs)
11795 CGM.addCompilerUsedGlobal(GV: ref);
11796}
11797
11798bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
11799 if (isa<FunctionDecl>(Val: GD.getDecl()) ||
11800 isa<OMPDeclareReductionDecl>(Val: GD.getDecl()))
11801 return emitTargetFunctions(GD);
11802
11803 return emitTargetGlobalVariable(GD);
11804}
11805
11806void CGOpenMPRuntime::emitDeferredTargetDecls() const {
11807 for (const VarDecl *VD : DeferredGlobalVariables) {
11808 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11809 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11810 if (!Res)
11811 continue;
11812 // MT_Local and MT_To/MT_Enter without USM are always emitted.
11813 if (*Res == OMPDeclareTargetDeclAttr::MT_Local ||
11814 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11815 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11816 !HasRequiresUnifiedSharedMemory)) {
11817 CGM.EmitGlobal(D: VD);
11818 } else {
11819 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
11820 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11821 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
11822 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
11823 HasRequiresUnifiedSharedMemory)) &&
11824 "Expected link clause or to clause with unified memory.");
11825 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
11826 }
11827 }
11828}
11829
11830void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
11831 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
11832 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
11833 " Expected target-based directive.");
11834}
11835
11836void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) {
11837 for (const OMPClause *Clause : D->clauselists()) {
11838 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
11839 HasRequiresUnifiedSharedMemory = true;
11840 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
11841 } else if (const auto *AC =
11842 dyn_cast<OMPAtomicDefaultMemOrderClause>(Val: Clause)) {
11843 switch (AC->getAtomicDefaultMemOrderKind()) {
11844 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
11845 RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;
11846 break;
11847 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
11848 RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
11849 break;
11850 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
11851 RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
11852 break;
11853 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown:
11854 break;
11855 }
11856 }
11857 }
11858}
11859
11860llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {
11861 return RequiresAtomicOrdering;
11862}
11863
11864bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
11865 LangAS &AS) {
11866 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
11867 return false;
11868 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
11869 switch(A->getAllocatorType()) {
11870 case OMPAllocateDeclAttr::OMPNullMemAlloc:
11871 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
11872 // Not supported, fallback to the default mem space.
11873 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
11874 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
11875 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
11876 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
11877 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
11878 case OMPAllocateDeclAttr::OMPConstMemAlloc:
11879 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
11880 AS = LangAS::Default;
11881 return true;
11882 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
11883 llvm_unreachable("Expected predefined allocator for the variables with the "
11884 "static storage.");
11885 }
11886 return false;
11887}
11888
11889bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {
11890 return HasRequiresUnifiedSharedMemory;
11891}
11892
11893CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
11894 CodeGenModule &CGM)
11895 : CGM(CGM) {
11896 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11897 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
11898 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
11899 }
11900}
11901
11902CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
11903 if (CGM.getLangOpts().OpenMPIsTargetDevice)
11904 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
11905}
11906
11907bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
11908 if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal)
11909 return true;
11910
11911 const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
11912 // Do not emit function if it is marked as declare target as it was already
11913 // emitted.
11914 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: D)) {
11915 if (D->hasBody() && AlreadyEmittedTargetDecls.count(V: D) == 0) {
11916 if (auto *F = dyn_cast_or_null<llvm::Function>(
11917 Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD))))
11918 return !F->isDeclaration();
11919 return false;
11920 }
11921 return true;
11922 }
11923
11924 return !AlreadyEmittedTargetDecls.insert(V: D).second;
11925}
11926
11927void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
11928 const OMPExecutableDirective &D,
11929 SourceLocation Loc,
11930 llvm::Function *OutlinedFn,
11931 ArrayRef<llvm::Value *> CapturedVars) {
11932 if (!CGF.HaveInsertPoint())
11933 return;
11934
11935 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11936 CodeGenFunction::RunCleanupsScope Scope(CGF);
11937
11938 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
11939 llvm::Value *Args[] = {
11940 RTLoc,
11941 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
11942 OutlinedFn};
11943 llvm::SmallVector<llvm::Value *, 16> RealArgs;
11944 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
11945 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
11946
11947 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
11948 M&: CGM.getModule(), FnID: OMPRTL___kmpc_fork_teams);
11949 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
11950}
11951
11952void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11953 const Expr *NumTeams,
11954 const Expr *ThreadLimit,
11955 SourceLocation Loc) {
11956 if (!CGF.HaveInsertPoint())
11957 return;
11958
11959 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11960
11961 llvm::Value *NumTeamsVal =
11962 NumTeams
11963 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: NumTeams),
11964 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11965 : CGF.Builder.getInt32(C: 0);
11966
11967 llvm::Value *ThreadLimitVal =
11968 ThreadLimit
11969 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11970 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11971 : CGF.Builder.getInt32(C: 0);
11972
11973 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
11974 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
11975 ThreadLimitVal};
11976 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11977 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_num_teams),
11978 args: PushNumTeamsArgs);
11979}
11980
11981void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
11982 const Expr *ThreadLimit,
11983 SourceLocation Loc) {
11984 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11985 llvm::Value *ThreadLimitVal =
11986 ThreadLimit
11987 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11988 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11989 : CGF.Builder.getInt32(C: 0);
11990
11991 // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit)
11992 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
11993 ThreadLimitVal};
11994 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11995 M&: CGM.getModule(), FnID: OMPRTL___kmpc_set_thread_limit),
11996 args: ThreadLimitArgs);
11997}
11998
11999void CGOpenMPRuntime::emitTargetDataCalls(
12000 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
12001 const Expr *Device, const RegionCodeGenTy &CodeGen,
12002 CGOpenMPRuntime::TargetDataInfo &Info) {
12003 if (!CGF.HaveInsertPoint())
12004 return;
12005
12006 // Action used to replace the default codegen action and turn privatization
12007 // off.
12008 PrePostActionTy NoPrivAction;
12009
12010 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
12011
12012 llvm::Value *IfCondVal = nullptr;
12013 if (IfCond)
12014 IfCondVal = CGF.EvaluateExprAsBool(E: IfCond);
12015
12016 // Emit device ID if any.
12017 llvm::Value *DeviceID = nullptr;
12018 if (Device) {
12019 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
12020 DestTy: CGF.Int64Ty, /*isSigned=*/true);
12021 } else {
12022 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
12023 }
12024
12025 // Fill up the arrays with all the mapped variables.
12026 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
12027 auto GenMapInfoCB =
12028 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
12029 CGF.Builder.restoreIP(IP: CodeGenIP);
12030 // Get map clause information.
12031 MappableExprsHandler MEHandler(D, CGF);
12032 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);
12033
12034 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
12035 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
12036 };
12037 if (CGM.getCodeGenOpts().getDebugInfo() !=
12038 llvm::codegenoptions::NoDebugInfo) {
12039 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
12040 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
12041 F: FillInfoMap);
12042 }
12043
12044 return CombinedInfo;
12045 };
12046 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
12047 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
12048 CGF.Builder.restoreIP(IP: CodeGenIP);
12049 switch (BodyGenType) {
12050 case BodyGenTy::Priv:
12051 if (!Info.CaptureDeviceAddrMap.empty())
12052 CodeGen(CGF);
12053 break;
12054 case BodyGenTy::DupNoPriv:
12055 if (!Info.CaptureDeviceAddrMap.empty()) {
12056 CodeGen.setAction(NoPrivAction);
12057 CodeGen(CGF);
12058 }
12059 break;
12060 case BodyGenTy::NoPriv:
12061 if (Info.CaptureDeviceAddrMap.empty()) {
12062 CodeGen.setAction(NoPrivAction);
12063 CodeGen(CGF);
12064 }
12065 break;
12066 }
12067 return InsertPointTy(CGF.Builder.GetInsertPoint());
12068 };
12069
12070 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
12071 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
12072 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
12073 }
12074 };
12075
12076 auto CustomMapperCB = [&](unsigned int I) {
12077 llvm::Function *MFunc = nullptr;
12078 if (CombinedInfo.Mappers[I]) {
12079 Info.HasMapper = true;
12080 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
12081 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
12082 }
12083 return MFunc;
12084 };
12085
12086 // Source location for the ident struct
12087 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
12088
12089 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
12090 InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
12091 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.Builder);
12092 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
12093 cantFail(ValOrErr: OMPBuilder.createTargetData(
12094 Loc: OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
12095 IfCond: IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
12096 /*MapperFunc=*/nullptr, BodyGenCB: BodyCB, DeviceAddrCB, SrcLocInfo: RTLoc));
12097 CGF.Builder.restoreIP(IP: AfterIP);
12098}
12099
12100void CGOpenMPRuntime::emitTargetDataStandAloneCall(
12101 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
12102 const Expr *Device) {
12103 if (!CGF.HaveInsertPoint())
12104 return;
12105
12106 assert((isa<OMPTargetEnterDataDirective>(D) ||
12107 isa<OMPTargetExitDataDirective>(D) ||
12108 isa<OMPTargetUpdateDirective>(D)) &&
12109 "Expecting either target enter, exit data, or update directives.");
12110
12111 CodeGenFunction::OMPTargetDataInfo InputInfo;
12112 llvm::Value *MapTypesArray = nullptr;
12113 llvm::Value *MapNamesArray = nullptr;
12114 // Generate the code for the opening of the data environment.
12115 auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,
12116 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
12117 // Emit device ID if any.
12118 llvm::Value *DeviceID = nullptr;
12119 if (Device) {
12120 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
12121 DestTy: CGF.Int64Ty, /*isSigned=*/true);
12122 } else {
12123 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
12124 }
12125
12126 // Emit the number of elements in the offloading arrays.
12127 llvm::Constant *PointerNum =
12128 CGF.Builder.getInt32(C: InputInfo.NumberOfTargetItems);
12129
12130 // Source location for the ident struct
12131 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
12132
12133 SmallVector<llvm::Value *, 13> OffloadingArgs(
12134 {RTLoc, DeviceID, PointerNum,
12135 InputInfo.BasePointersArray.emitRawPointer(CGF),
12136 InputInfo.PointersArray.emitRawPointer(CGF),
12137 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
12138 InputInfo.MappersArray.emitRawPointer(CGF)});
12139
12140 // Select the right runtime function call for each standalone
12141 // directive.
12142 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
12143 RuntimeFunction RTLFn;
12144 switch (D.getDirectiveKind()) {
12145 case OMPD_target_enter_data:
12146 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
12147 : OMPRTL___tgt_target_data_begin_mapper;
12148 break;
12149 case OMPD_target_exit_data:
12150 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
12151 : OMPRTL___tgt_target_data_end_mapper;
12152 break;
12153 case OMPD_target_update:
12154 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
12155 : OMPRTL___tgt_target_data_update_mapper;
12156 break;
12157 case OMPD_parallel:
12158 case OMPD_for:
12159 case OMPD_parallel_for:
12160 case OMPD_parallel_master:
12161 case OMPD_parallel_sections:
12162 case OMPD_for_simd:
12163 case OMPD_parallel_for_simd:
12164 case OMPD_cancel:
12165 case OMPD_cancellation_point:
12166 case OMPD_ordered_standalone:
12167 case OMPD_ordered_blockassoc:
12168 case OMPD_threadprivate:
12169 case OMPD_allocate:
12170 case OMPD_task:
12171 case OMPD_simd:
12172 case OMPD_tile:
12173 case OMPD_unroll:
12174 case OMPD_sections:
12175 case OMPD_section:
12176 case OMPD_single:
12177 case OMPD_master:
12178 case OMPD_critical:
12179 case OMPD_taskyield:
12180 case OMPD_barrier:
12181 case OMPD_taskwait:
12182 case OMPD_taskgroup:
12183 case OMPD_atomic:
12184 case OMPD_flush:
12185 case OMPD_depobj:
12186 case OMPD_scan:
12187 case OMPD_teams:
12188 case OMPD_target_data:
12189 case OMPD_distribute:
12190 case OMPD_distribute_simd:
12191 case OMPD_distribute_parallel_for:
12192 case OMPD_distribute_parallel_for_simd:
12193 case OMPD_teams_distribute:
12194 case OMPD_teams_distribute_simd:
12195 case OMPD_teams_distribute_parallel_for:
12196 case OMPD_teams_distribute_parallel_for_simd:
12197 case OMPD_declare_simd:
12198 case OMPD_declare_variant:
12199 case OMPD_begin_declare_variant:
12200 case OMPD_end_declare_variant:
12201 case OMPD_declare_target:
12202 case OMPD_end_declare_target:
12203 case OMPD_declare_reduction:
12204 case OMPD_declare_mapper:
12205 case OMPD_taskloop:
12206 case OMPD_taskloop_simd:
12207 case OMPD_master_taskloop:
12208 case OMPD_master_taskloop_simd:
12209 case OMPD_parallel_master_taskloop:
12210 case OMPD_parallel_master_taskloop_simd:
12211 case OMPD_target:
12212 case OMPD_target_simd:
12213 case OMPD_target_teams_distribute:
12214 case OMPD_target_teams_distribute_simd:
12215 case OMPD_target_teams_distribute_parallel_for:
12216 case OMPD_target_teams_distribute_parallel_for_simd:
12217 case OMPD_target_teams:
12218 case OMPD_target_parallel:
12219 case OMPD_target_parallel_for:
12220 case OMPD_target_parallel_for_simd:
12221 case OMPD_requires:
12222 case OMPD_metadirective:
12223 case OMPD_unknown:
12224 default:
12225 llvm_unreachable("Unexpected standalone target data directive.");
12226 break;
12227 }
12228 if (HasNowait) {
12229 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
12230 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
12231 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
12232 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
12233 }
12234 CGF.EmitRuntimeCall(
12235 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID: RTLFn),
12236 args: OffloadingArgs);
12237 };
12238
12239 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
12240 &MapNamesArray](CodeGenFunction &CGF,
12241 PrePostActionTy &) {
12242 // Fill up the arrays with all the mapped variables.
12243 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
12244 CGOpenMPRuntime::TargetDataInfo Info;
12245 MappableExprsHandler MEHandler(D, CGF);
12246 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
12247 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
12248 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
12249
12250 bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
12251 D.hasClausesOfKind<OMPNowaitClause>();
12252
12253 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
12254 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
12255 CGF.VoidPtrTy, CGM.getPointerAlign());
12256 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
12257 CGM.getPointerAlign());
12258 InputInfo.SizesArray =
12259 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
12260 InputInfo.MappersArray =
12261 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
12262 MapTypesArray = Info.RTArgs.MapTypesArray;
12263 MapNamesArray = Info.RTArgs.MapNamesArray;
12264 if (RequiresOuterTask)
12265 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
12266 else
12267 emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
12268 };
12269
12270 if (IfCond) {
12271 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen,
12272 ElseGen: [](CodeGenFunction &CGF, PrePostActionTy &) {});
12273 } else {
12274 RegionCodeGenTy ThenRCG(TargetThenGen);
12275 ThenRCG(CGF);
12276 }
12277}
12278
12279static unsigned
12280evaluateCDTSize(const FunctionDecl *FD,
12281 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12282 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
12283 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
12284 // of that clause. The VLEN value must be power of 2.
12285 // In other case the notion of the function`s "characteristic data type" (CDT)
12286 // is used to compute the vector length.
12287 // CDT is defined in the following order:
12288 // a) For non-void function, the CDT is the return type.
12289 // b) If the function has any non-uniform, non-linear parameters, then the
12290 // CDT is the type of the first such parameter.
12291 // c) If the CDT determined by a) or b) above is struct, union, or class
12292 // type which is pass-by-value (except for the type that maps to the
12293 // built-in complex data type), the characteristic data type is int.
12294 // d) If none of the above three cases is applicable, the CDT is int.
12295 // The VLEN is then determined based on the CDT and the size of vector
12296 // register of that ISA for which current vector version is generated. The
12297 // VLEN is computed using the formula below:
12298 // VLEN = sizeof(vector_register) / sizeof(CDT),
12299 // where vector register size specified in section 3.2.1 Registers and the
12300 // Stack Frame of original AMD64 ABI document.
12301 QualType RetType = FD->getReturnType();
12302 if (RetType.isNull())
12303 return 0;
12304 ASTContext &C = FD->getASTContext();
12305 QualType CDT;
12306 if (!RetType.isNull() && !RetType->isVoidType()) {
12307 CDT = RetType;
12308 } else {
12309 unsigned Offset = 0;
12310 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD)) {
12311 if (ParamAttrs[Offset].Kind ==
12312 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector)
12313 CDT = C.getPointerType(T: C.getCanonicalTagType(TD: MD->getParent()));
12314 ++Offset;
12315 }
12316 if (CDT.isNull()) {
12317 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12318 if (ParamAttrs[I + Offset].Kind ==
12319 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) {
12320 CDT = FD->getParamDecl(i: I)->getType();
12321 break;
12322 }
12323 }
12324 }
12325 }
12326 if (CDT.isNull())
12327 CDT = C.IntTy;
12328 CDT = CDT->getCanonicalTypeUnqualified();
12329 if (CDT->isRecordType() || CDT->isUnionType())
12330 CDT = C.IntTy;
12331 return C.getTypeSize(T: CDT);
12332}
12333
12334// This are the Functions that are needed to mangle the name of the
12335// vector functions generated by the compiler, according to the rules
12336// defined in the "Vector Function ABI specifications for AArch64",
12337// available at
12338// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
12339
12340/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
12341static bool getAArch64MTV(QualType QT,
12342 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) {
12343 QT = QT.getCanonicalType();
12344
12345 if (QT->isVoidType())
12346 return false;
12347
12348 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform)
12349 return false;
12350
12351 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12352 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)
12353 return false;
12354
12355 if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12356 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) &&
12357 !QT->isReferenceType())
12358 return false;
12359
12360 return true;
12361}
12362
12363/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
12364static bool getAArch64PBV(QualType QT, ASTContext &C) {
12365 QT = QT.getCanonicalType();
12366 unsigned Size = C.getTypeSize(T: QT);
12367
12368 // Only scalars and complex within 16 bytes wide set PVB to true.
12369 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
12370 return false;
12371
12372 if (QT->isFloatingType())
12373 return true;
12374
12375 if (QT->isIntegerType())
12376 return true;
12377
12378 if (QT->isPointerType())
12379 return true;
12380
12381 // TODO: Add support for complex types (section 3.1.2, item 2).
12382
12383 return false;
12384}
12385
12386/// Computes the lane size (LS) of a return type or of an input parameter,
12387/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
12388/// TODO: Add support for references, section 3.2.1, item 1.
12389static unsigned getAArch64LS(QualType QT,
12390 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind,
12391 ASTContext &C) {
12392 if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
12393 QualType PTy = QT.getCanonicalType()->getPointeeType();
12394 if (getAArch64PBV(QT: PTy, C))
12395 return C.getTypeSize(T: PTy);
12396 }
12397 if (getAArch64PBV(QT, C))
12398 return C.getTypeSize(T: QT);
12399
12400 return C.getTypeSize(T: C.getUIntPtrType());
12401}
12402
12403// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
12404// signature of the scalar function, as defined in 3.2.2 of the
12405// AAVFABI.
12406static std::tuple<unsigned, unsigned, bool>
12407getNDSWDS(const FunctionDecl *FD,
12408 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12409 QualType RetType = FD->getReturnType().getCanonicalType();
12410
12411 ASTContext &C = FD->getASTContext();
12412
12413 bool OutputBecomesInput = false;
12414
12415 llvm::SmallVector<unsigned, 8> Sizes;
12416 if (!RetType->isVoidType()) {
12417 Sizes.push_back(Elt: getAArch64LS(
12418 QT: RetType, Kind: llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C));
12419 if (!getAArch64PBV(QT: RetType, C) && getAArch64MTV(QT: RetType, Kind: {}))
12420 OutputBecomesInput = true;
12421 }
12422 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12423 QualType QT = FD->getParamDecl(i: I)->getType().getCanonicalType();
12424 Sizes.push_back(Elt: getAArch64LS(QT, Kind: ParamAttrs[I].Kind, C));
12425 }
12426
12427 assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
12428 // The LS of a function parameter / return value can only be a power
12429 // of 2, starting from 8 bits, up to 128.
12430 assert(llvm::all_of(Sizes,
12431 [](unsigned Size) {
12432 return Size == 8 || Size == 16 || Size == 32 ||
12433 Size == 64 || Size == 128;
12434 }) &&
12435 "Invalid size");
12436
12437 return std::make_tuple(args&: *llvm::min_element(Range&: Sizes), args&: *llvm::max_element(Range&: Sizes),
12438 args&: OutputBecomesInput);
12439}
12440
12441static llvm::OpenMPIRBuilder::DeclareSimdBranch
12442convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) {
12443 switch (State) {
12444 case OMPDeclareSimdDeclAttr::BS_Undefined:
12445 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined;
12446 case OMPDeclareSimdDeclAttr::BS_Inbranch:
12447 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch;
12448 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
12449 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch;
12450 }
12451 llvm_unreachable("unexpected declare simd branch state");
12452}
12453
12454// Check the values provided via `simdlen` by the user.
12455static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc,
12456 unsigned UserVLEN, unsigned WDS, char ISA) {
12457 // 1. A `simdlen(1)` doesn't produce vector signatures.
12458 if (UserVLEN == 1) {
12459 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_1_no_effect);
12460 return false;
12461 }
12462
12463 // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced
12464 // SIMD.
12465 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(Value: UserVLEN)) {
12466 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_requires_power_of_2);
12467 return false;
12468 }
12469
12470 // 3. Section 3.4.1: SVE fixed length must obey the architectural limits.
12471 if (ISA == 's' && UserVLEN != 0 &&
12472 ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) {
12473 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_must_fit_lanes) << WDS;
12474 return false;
12475 }
12476
12477 return true;
12478}
12479
12480void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
12481 llvm::Function *Fn) {
12482 ASTContext &C = CGM.getContext();
12483 FD = FD->getMostRecentDecl();
12484 while (FD) {
12485 // Map params to their positions in function decl.
12486 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
12487 if (isa<CXXMethodDecl>(Val: FD))
12488 ParamPositions.try_emplace(Key: FD, Args: 0);
12489 unsigned ParamPos = ParamPositions.size();
12490 for (const ParmVarDecl *P : FD->parameters()) {
12491 ParamPositions.try_emplace(Key: P->getCanonicalDecl(), Args&: ParamPos);
12492 ++ParamPos;
12493 }
12494 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
12495 llvm::SmallVector<llvm::OpenMPIRBuilder::DeclareSimdAttrTy, 8> ParamAttrs(
12496 ParamPositions.size());
12497 // Mark uniform parameters.
12498 for (const Expr *E : Attr->uniforms()) {
12499 E = E->IgnoreParenImpCasts();
12500 unsigned Pos;
12501 if (isa<CXXThisExpr>(Val: E)) {
12502 Pos = ParamPositions[FD];
12503 } else {
12504 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12505 ->getCanonicalDecl();
12506 auto It = ParamPositions.find(Val: PVD);
12507 assert(It != ParamPositions.end() && "Function parameter not found");
12508 Pos = It->second;
12509 }
12510 ParamAttrs[Pos].Kind =
12511 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform;
12512 }
12513 // Get alignment info.
12514 auto *NI = Attr->alignments_begin();
12515 for (const Expr *E : Attr->aligneds()) {
12516 E = E->IgnoreParenImpCasts();
12517 unsigned Pos;
12518 QualType ParmTy;
12519 if (isa<CXXThisExpr>(Val: E)) {
12520 Pos = ParamPositions[FD];
12521 ParmTy = E->getType();
12522 } else {
12523 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12524 ->getCanonicalDecl();
12525 auto It = ParamPositions.find(Val: PVD);
12526 assert(It != ParamPositions.end() && "Function parameter not found");
12527 Pos = It->second;
12528 ParmTy = PVD->getType();
12529 }
12530 ParamAttrs[Pos].Alignment =
12531 (*NI)
12532 ? (*NI)->EvaluateKnownConstInt(Ctx: C)
12533 : llvm::APSInt::getUnsigned(
12534 X: C.toCharUnitsFromBits(BitSize: C.getOpenMPDefaultSimdAlign(T: ParmTy))
12535 .getQuantity());
12536 ++NI;
12537 }
12538 // Mark linear parameters.
12539 auto *SI = Attr->steps_begin();
12540 auto *MI = Attr->modifiers_begin();
12541 for (const Expr *E : Attr->linears()) {
12542 E = E->IgnoreParenImpCasts();
12543 unsigned Pos;
12544 bool IsReferenceType = false;
12545 // Rescaling factor needed to compute the linear parameter
12546 // value in the mangled name.
12547 unsigned PtrRescalingFactor = 1;
12548 if (isa<CXXThisExpr>(Val: E)) {
12549 Pos = ParamPositions[FD];
12550 auto *P = cast<PointerType>(Val: E->getType());
12551 PtrRescalingFactor = CGM.getContext()
12552 .getTypeSizeInChars(T: P->getPointeeType())
12553 .getQuantity();
12554 } else {
12555 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12556 ->getCanonicalDecl();
12557 auto It = ParamPositions.find(Val: PVD);
12558 assert(It != ParamPositions.end() && "Function parameter not found");
12559 Pos = It->second;
12560 if (auto *P = dyn_cast<PointerType>(Val: PVD->getType()))
12561 PtrRescalingFactor = CGM.getContext()
12562 .getTypeSizeInChars(T: P->getPointeeType())
12563 .getQuantity();
12564 else if (PVD->getType()->isReferenceType()) {
12565 IsReferenceType = true;
12566 PtrRescalingFactor =
12567 CGM.getContext()
12568 .getTypeSizeInChars(T: PVD->getType().getNonReferenceType())
12569 .getQuantity();
12570 }
12571 }
12572 llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos];
12573 if (*MI == OMPC_LINEAR_ref)
12574 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef;
12575 else if (*MI == OMPC_LINEAR_uval)
12576 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal;
12577 else if (IsReferenceType)
12578 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal;
12579 else
12580 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear;
12581 // Assuming a stride of 1, for `linear` without modifiers.
12582 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: 1);
12583 if (*SI) {
12584 Expr::EvalResult Result;
12585 if (!(*SI)->EvaluateAsInt(Result, Ctx: C, AllowSideEffects: Expr::SE_AllowSideEffects)) {
12586 if (const auto *DRE =
12587 cast<DeclRefExpr>(Val: (*SI)->IgnoreParenImpCasts())) {
12588 if (const auto *StridePVD =
12589 dyn_cast<ParmVarDecl>(Val: DRE->getDecl())) {
12590 ParamAttr.HasVarStride = true;
12591 auto It = ParamPositions.find(Val: StridePVD->getCanonicalDecl());
12592 assert(It != ParamPositions.end() &&
12593 "Function parameter not found");
12594 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: It->second);
12595 }
12596 }
12597 } else {
12598 ParamAttr.StrideOrArg = Result.Val.getInt();
12599 }
12600 }
12601 // If we are using a linear clause on a pointer, we need to
12602 // rescale the value of linear_step with the byte size of the
12603 // pointee type.
12604 if (!ParamAttr.HasVarStride &&
12605 (ParamAttr.Kind ==
12606 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12607 ParamAttr.Kind ==
12608 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef))
12609 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
12610 ++SI;
12611 ++MI;
12612 }
12613 llvm::APSInt VLENVal;
12614 SourceLocation ExprLoc;
12615 const Expr *VLENExpr = Attr->getSimdlen();
12616 if (VLENExpr) {
12617 VLENVal = VLENExpr->EvaluateKnownConstInt(Ctx: C);
12618 ExprLoc = VLENExpr->getExprLoc();
12619 }
12620 llvm::OpenMPIRBuilder::DeclareSimdBranch State =
12621 convertDeclareSimdBranch(State: Attr->getBranchState());
12622 if (CGM.getTriple().isX86()) {
12623 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
12624 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12625 OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElements: NumElts, VLENVal, ParamAttrs,
12626 Branch: State);
12627 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
12628 unsigned VLEN = VLENVal.getExtValue();
12629 // Get basic data for building the vector signature.
12630 const auto Data = getNDSWDS(FD, ParamAttrs);
12631 const unsigned NDS = std::get<0>(t: Data);
12632 const unsigned WDS = std::get<1>(t: Data);
12633 const bool OutputBecomesInput = std::get<2>(t: Data);
12634 if (CGM.getTarget().hasFeature(Feature: "sve")) {
12635 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 's'))
12636 OMPBuilder.emitAArch64DeclareSimdFunction(
12637 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 's', NarrowestDataSize: NDS, OutputBecomesInput);
12638 } else if (CGM.getTarget().hasFeature(Feature: "neon")) {
12639 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 'n'))
12640 OMPBuilder.emitAArch64DeclareSimdFunction(
12641 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 'n', NarrowestDataSize: NDS, OutputBecomesInput);
12642 }
12643 }
12644 }
12645 FD = FD->getPreviousDecl();
12646 }
12647}
12648
12649namespace {
12650/// Cleanup action for doacross support.
12651class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
12652public:
12653 static const int DoacrossFinArgs = 2;
12654
12655private:
12656 llvm::FunctionCallee RTLFn;
12657 llvm::Value *Args[DoacrossFinArgs];
12658
12659public:
12660 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
12661 ArrayRef<llvm::Value *> CallArgs)
12662 : RTLFn(RTLFn) {
12663 assert(CallArgs.size() == DoacrossFinArgs);
12664 std::copy(first: CallArgs.begin(), last: CallArgs.end(), result: std::begin(arr&: Args));
12665 }
12666 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12667 if (!CGF.HaveInsertPoint())
12668 return;
12669 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12670 }
12671};
12672} // namespace
12673
12674void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
12675 const OMPLoopDirective &D,
12676 ArrayRef<Expr *> NumIterations) {
12677 if (!CGF.HaveInsertPoint())
12678 return;
12679
12680 ASTContext &C = CGM.getContext();
12681 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
12682 RecordDecl *RD;
12683 if (KmpDimTy.isNull()) {
12684 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
12685 // kmp_int64 lo; // lower
12686 // kmp_int64 up; // upper
12687 // kmp_int64 st; // stride
12688 // };
12689 RD = C.buildImplicitRecord(Name: "kmp_dim");
12690 RD->startDefinition();
12691 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12692 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12693 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12694 RD->completeDefinition();
12695 KmpDimTy = C.getCanonicalTagType(TD: RD);
12696 } else {
12697 RD = KmpDimTy->castAsRecordDecl();
12698 }
12699 llvm::APInt Size(/*numBits=*/32, NumIterations.size());
12700 QualType ArrayTy = C.getConstantArrayType(EltTy: KmpDimTy, ArySize: Size, SizeExpr: nullptr,
12701 ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12702
12703 Address DimsAddr = CGF.CreateMemTemp(T: ArrayTy, Name: "dims");
12704 CGF.EmitNullInitialization(DestPtr: DimsAddr, Ty: ArrayTy);
12705 enum { LowerFD = 0, UpperFD, StrideFD };
12706 // Fill dims with data.
12707 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
12708 LValue DimsLVal = CGF.MakeAddrLValue(
12709 Addr: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: I), T: KmpDimTy);
12710 // dims.upper = num_iterations;
12711 LValue UpperLVal = CGF.EmitLValueForField(
12712 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: UpperFD));
12713 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
12714 Src: CGF.EmitScalarExpr(E: NumIterations[I]), SrcTy: NumIterations[I]->getType(),
12715 DstTy: Int64Ty, Loc: NumIterations[I]->getExprLoc());
12716 CGF.EmitStoreOfScalar(value: NumIterVal, lvalue: UpperLVal);
12717 // dims.stride = 1;
12718 LValue StrideLVal = CGF.EmitLValueForField(
12719 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: StrideFD));
12720 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::getSigned(Ty: CGM.Int64Ty, /*V=*/1),
12721 lvalue: StrideLVal);
12722 }
12723
12724 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
12725 // kmp_int32 num_dims, struct kmp_dim * dims);
12726 llvm::Value *Args[] = {
12727 emitUpdateLocation(CGF, Loc: D.getBeginLoc()),
12728 getThreadID(CGF, Loc: D.getBeginLoc()),
12729 llvm::ConstantInt::getSigned(Ty: CGM.Int32Ty, V: NumIterations.size()),
12730 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12731 V: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: 0).emitRawPointer(CGF),
12732 DestTy: CGM.VoidPtrTy)};
12733
12734 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12735 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_init);
12736 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12737 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
12738 emitUpdateLocation(CGF, Loc: D.getEndLoc()), getThreadID(CGF, Loc: D.getEndLoc())};
12739 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12740 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_fini);
12741 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(Kind: NormalAndEHCleanup, A: FiniRTLFn,
12742 A: llvm::ArrayRef(FiniArgs));
12743}
12744
12745template <typename T>
12746static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
12747 const T *C, llvm::Value *ULoc,
12748 llvm::Value *ThreadID) {
12749 QualType Int64Ty =
12750 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
12751 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
12752 QualType ArrayTy = CGM.getContext().getConstantArrayType(
12753 EltTy: Int64Ty, ArySize: Size, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12754 Address CntAddr = CGF.CreateMemTemp(T: ArrayTy, Name: ".cnt.addr");
12755 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
12756 const Expr *CounterVal = C->getLoopData(I);
12757 assert(CounterVal);
12758 llvm::Value *CntVal = CGF.EmitScalarConversion(
12759 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
12760 Loc: CounterVal->getExprLoc());
12761 CGF.EmitStoreOfScalar(Value: CntVal, Addr: CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: I),
12762 /*Volatile=*/false, Ty: Int64Ty);
12763 }
12764 llvm::Value *Args[] = {
12765 ULoc, ThreadID,
12766 CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: 0).emitRawPointer(CGF)};
12767 llvm::FunctionCallee RTLFn;
12768 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
12769 OMPDoacrossKind<T> ODK;
12770 if (ODK.isSource(C)) {
12771 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12772 FnID: OMPRTL___kmpc_doacross_post);
12773 } else {
12774 assert(ODK.isSink(C) && "Expect sink modifier.");
12775 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12776 FnID: OMPRTL___kmpc_doacross_wait);
12777 }
12778 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12779}
12780
12781void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12782 const OMPDependClause *C) {
12783 return EmitDoacrossOrdered<OMPDependClause>(
12784 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12785 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12786}
12787
12788void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12789 const OMPDoacrossClause *C) {
12790 return EmitDoacrossOrdered<OMPDoacrossClause>(
12791 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12792 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12793}
12794
12795void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
12796 llvm::FunctionCallee Callee,
12797 ArrayRef<llvm::Value *> Args) const {
12798 assert(Loc.isValid() && "Outlined function call location must be valid.");
12799 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
12800
12801 if (auto *Fn = dyn_cast<llvm::Function>(Val: Callee.getCallee())) {
12802 if (Fn->doesNotThrow()) {
12803 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
12804 return;
12805 }
12806 }
12807 CGF.EmitRuntimeCall(callee: Callee, args: Args);
12808}
12809
12810void CGOpenMPRuntime::emitOutlinedFunctionCall(
12811 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
12812 ArrayRef<llvm::Value *> Args) const {
12813 emitCall(CGF, Loc, Callee: OutlinedFn, Args);
12814}
12815
12816void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
12817 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D))
12818 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: FD))
12819 HasEmittedDeclareTargetRegion = true;
12820}
12821
12822Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
12823 const VarDecl *NativeParam,
12824 const VarDecl *TargetParam) const {
12825 return CGF.GetAddrOfLocalVar(VD: NativeParam);
12826}
12827
12828/// Return allocator value from expression, or return a null allocator (default
12829/// when no allocator specified).
12830static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
12831 const Expr *Allocator) {
12832 llvm::Value *AllocVal;
12833 if (Allocator) {
12834 AllocVal = CGF.EmitScalarExpr(E: Allocator);
12835 // According to the standard, the original allocator type is a enum
12836 // (integer). Convert to pointer type, if required.
12837 AllocVal = CGF.EmitScalarConversion(Src: AllocVal, SrcTy: Allocator->getType(),
12838 DstTy: CGF.getContext().VoidPtrTy,
12839 Loc: Allocator->getExprLoc());
12840 } else {
12841 // If no allocator specified, it defaults to the null allocator.
12842 AllocVal = llvm::Constant::getNullValue(
12843 Ty: CGF.CGM.getTypes().ConvertType(T: CGF.getContext().VoidPtrTy));
12844 }
12845 return AllocVal;
12846}
12847
12848/// Return the alignment from an allocate directive if present.
12849static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {
12850 std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);
12851
12852 if (!AllocateAlignment)
12853 return nullptr;
12854
12855 return llvm::ConstantInt::get(Ty: CGM.SizeTy, V: AllocateAlignment->getQuantity());
12856}
12857
12858Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
12859 const VarDecl *VD) {
12860 if (!VD)
12861 return Address::invalid();
12862 Address UntiedAddr = Address::invalid();
12863 Address UntiedRealAddr = Address::invalid();
12864 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12865 if (It != FunctionToUntiedTaskStackMap.end()) {
12866 const UntiedLocalVarsAddressesMap &UntiedData =
12867 UntiedLocalVarsStack[It->second];
12868 auto I = UntiedData.find(Key: VD);
12869 if (I != UntiedData.end()) {
12870 UntiedAddr = I->second.first;
12871 UntiedRealAddr = I->second.second;
12872 }
12873 }
12874 // Unreachable declaration: nothing can run the allocation.
12875 if (!CGF.HaveInsertPoint())
12876 return UntiedRealAddr.isValid() ? UntiedRealAddr : UntiedAddr;
12877 const VarDecl *CVD = VD->getCanonicalDecl();
12878 if (CVD->hasAttr<OMPAllocateDeclAttr>()) {
12879 // Use the default allocation.
12880 if (!isAllocatableDecl(VD))
12881 return UntiedAddr;
12882 llvm::Value *Size;
12883 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
12884 if (CVD->getType()->isVariablyModifiedType()) {
12885 Size = CGF.getTypeSize(Ty: CVD->getType());
12886 // Align the size: ((size + align - 1) / align) * align
12887 Size = CGF.Builder.CreateNUWAdd(
12888 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
12889 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
12890 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
12891 } else {
12892 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
12893 Size = CGM.getSize(numChars: Sz.alignTo(Align));
12894 }
12895 llvm::Value *ThreadID = getThreadID(CGF, Loc: CVD->getBeginLoc());
12896 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
12897 const Expr *Allocator = AA->getAllocator();
12898 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
12899 llvm::Value *Alignment = getAlignmentValue(CGM, VD: CVD);
12900 SmallVector<llvm::Value *, 4> Args;
12901 Args.push_back(Elt: ThreadID);
12902 if (Alignment)
12903 Args.push_back(Elt: Alignment);
12904 Args.push_back(Elt: Size);
12905 Args.push_back(Elt: AllocVal);
12906 llvm::omp::RuntimeFunction FnID =
12907 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
12908 llvm::Value *Addr = CGF.EmitRuntimeCall(
12909 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args,
12910 name: getName(Parts: {CVD->getName(), ".void.addr"}));
12911 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12912 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free);
12913 QualType Ty = CGM.getContext().getPointerType(T: CVD->getType());
12914 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12915 V: Addr, DestTy: CGF.ConvertTypeForMem(T: Ty), Name: getName(Parts: {CVD->getName(), ".addr"}));
12916 if (UntiedAddr.isValid())
12917 CGF.EmitStoreOfScalar(Value: Addr, Addr: UntiedAddr, /*Volatile=*/false, Ty);
12918
12919 // Cleanup action for allocate support.
12920 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
12921 llvm::FunctionCallee RTLFn;
12922 SourceLocation::UIntTy LocEncoding;
12923 Address Addr;
12924 const Expr *AllocExpr;
12925
12926 public:
12927 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
12928 SourceLocation::UIntTy LocEncoding, Address Addr,
12929 const Expr *AllocExpr)
12930 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
12931 AllocExpr(AllocExpr) {}
12932 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12933 if (!CGF.HaveInsertPoint())
12934 return;
12935 llvm::Value *Args[3];
12936 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
12937 CGF, Loc: SourceLocation::getFromRawEncoding(Encoding: LocEncoding));
12938 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12939 V: Addr.emitRawPointer(CGF), DestTy: CGF.VoidPtrTy);
12940 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator: AllocExpr);
12941 Args[2] = AllocVal;
12942 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12943 }
12944 };
12945 Address VDAddr =
12946 UntiedRealAddr.isValid()
12947 ? UntiedRealAddr
12948 : Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
12949 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
12950 Kind: NormalAndEHCleanup, A: FiniRTLFn, A: CVD->getLocation().getRawEncoding(),
12951 A: VDAddr, A: Allocator);
12952 if (UntiedRealAddr.isValid())
12953 if (auto *Region =
12954 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
12955 Region->emitUntiedSwitch(CGF);
12956 return VDAddr;
12957 }
12958 return UntiedAddr;
12959}
12960
12961bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,
12962 const VarDecl *VD) const {
12963 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12964 if (It == FunctionToUntiedTaskStackMap.end())
12965 return false;
12966 return UntiedLocalVarsStack[It->second].count(Key: VD) > 0;
12967}
12968
12969CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII(
12970 CodeGenModule &CGM, const OMPLoopDirective &S)
12971 : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {
12972 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12973 if (!NeedToPush)
12974 return;
12975 NontemporalDeclsSet &DS =
12976 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12977 for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) {
12978 for (const Stmt *Ref : C->private_refs()) {
12979 const auto *SimpleRefExpr = cast<Expr>(Val: Ref)->IgnoreParenImpCasts();
12980 const ValueDecl *VD;
12981 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: SimpleRefExpr)) {
12982 VD = DRE->getDecl();
12983 } else {
12984 const auto *ME = cast<MemberExpr>(Val: SimpleRefExpr);
12985 assert((ME->isImplicitCXXThis() ||
12986 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
12987 "Expected member of current class.");
12988 VD = ME->getMemberDecl();
12989 }
12990 DS.insert(V: VD);
12991 }
12992 }
12993}
12994
12995CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() {
12996 if (!NeedToPush)
12997 return;
12998 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
12999}
13000
13001CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII(
13002 CodeGenFunction &CGF,
13003 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
13004 std::pair<Address, Address>> &LocalVars)
13005 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
13006 if (!NeedToPush)
13007 return;
13008 CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace(
13009 Key: CGF.CurFn, Args: CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
13010 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(Elt: LocalVars);
13011}
13012
13013CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() {
13014 if (!NeedToPush)
13015 return;
13016 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
13017}
13018
13019bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const {
13020 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13021
13022 return llvm::any_of(
13023 Range&: CGM.getOpenMPRuntime().NontemporalDeclsStack,
13024 P: [VD](const NontemporalDeclsSet &Set) { return Set.contains(V: VD); });
13025}
13026
13027void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
13028 const OMPExecutableDirective &S,
13029 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled)
13030 const {
13031 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
13032 // Vars in target/task regions must be excluded completely.
13033 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()) ||
13034 isOpenMPTaskingDirective(Kind: S.getDirectiveKind())) {
13035 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
13036 getOpenMPCaptureRegions(CaptureRegions, DKind: S.getDirectiveKind());
13037 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CaptureRegions.front());
13038 for (const CapturedStmt::Capture &Cap : CS->captures()) {
13039 if (Cap.capturesVariable() || Cap.capturesVariableByCopy())
13040 NeedToCheckForLPCs.insert(V: Cap.getCapturedVar());
13041 }
13042 }
13043 // Exclude vars in private clauses.
13044 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
13045 for (const Expr *Ref : C->varlist()) {
13046 if (!Ref->getType()->isScalarType())
13047 continue;
13048 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
13049 if (!DRE)
13050 continue;
13051 NeedToCheckForLPCs.insert(V: DRE->getDecl());
13052 }
13053 }
13054 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
13055 for (const Expr *Ref : C->varlist()) {
13056 if (!Ref->getType()->isScalarType())
13057 continue;
13058 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
13059 if (!DRE)
13060 continue;
13061 NeedToCheckForLPCs.insert(V: DRE->getDecl());
13062 }
13063 }
13064 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
13065 for (const Expr *Ref : C->varlist()) {
13066 if (!Ref->getType()->isScalarType())
13067 continue;
13068 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
13069 if (!DRE)
13070 continue;
13071 NeedToCheckForLPCs.insert(V: DRE->getDecl());
13072 }
13073 }
13074 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
13075 for (const Expr *Ref : C->varlist()) {
13076 if (!Ref->getType()->isScalarType())
13077 continue;
13078 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
13079 if (!DRE)
13080 continue;
13081 NeedToCheckForLPCs.insert(V: DRE->getDecl());
13082 }
13083 }
13084 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
13085 for (const Expr *Ref : C->varlist()) {
13086 if (!Ref->getType()->isScalarType())
13087 continue;
13088 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
13089 if (!DRE)
13090 continue;
13091 NeedToCheckForLPCs.insert(V: DRE->getDecl());
13092 }
13093 }
13094 for (const Decl *VD : NeedToCheckForLPCs) {
13095 for (const LastprivateConditionalData &Data :
13096 llvm::reverse(C&: CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
13097 if (Data.DeclToUniqueName.count(Key: VD) > 0) {
13098 if (!Data.Disabled)
13099 NeedToAddForLPCsAsDisabled.insert(V: VD);
13100 break;
13101 }
13102 }
13103 }
13104}
13105
13106CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13107 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
13108 : CGM(CGF.CGM),
13109 Action((CGM.getLangOpts().OpenMP >= 50 &&
13110 llvm::any_of(Range: S.getClausesOfKind<OMPLastprivateClause>(),
13111 P: [](const OMPLastprivateClause *C) {
13112 return C->getKind() ==
13113 OMPC_LASTPRIVATE_conditional;
13114 }))
13115 ? ActionToDo::PushAsLastprivateConditional
13116 : ActionToDo::DoNotPush) {
13117 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13118 if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
13119 return;
13120 assert(Action == ActionToDo::PushAsLastprivateConditional &&
13121 "Expected a push action.");
13122 LastprivateConditionalData &Data =
13123 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
13124 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
13125 if (C->getKind() != OMPC_LASTPRIVATE_conditional)
13126 continue;
13127
13128 for (const Expr *Ref : C->varlist()) {
13129 Data.DeclToUniqueName.insert(KV: std::make_pair(
13130 x: cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts())->getDecl(),
13131 y: SmallString<16>(generateUniqueName(CGM, Prefix: "pl_cond", Ref))));
13132 }
13133 }
13134 Data.IVLVal = IVLVal;
13135 Data.Fn = CGF.CurFn;
13136}
13137
13138CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
13139 CodeGenFunction &CGF, const OMPExecutableDirective &S)
13140 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
13141 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
13142 if (CGM.getLangOpts().OpenMP < 50)
13143 return;
13144 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
13145 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
13146 if (!NeedToAddForLPCsAsDisabled.empty()) {
13147 Action = ActionToDo::DisableLastprivateConditional;
13148 LastprivateConditionalData &Data =
13149 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
13150 for (const Decl *VD : NeedToAddForLPCsAsDisabled)
13151 Data.DeclToUniqueName.try_emplace(Key: VD);
13152 Data.Fn = CGF.CurFn;
13153 Data.Disabled = true;
13154 }
13155}
13156
13157CGOpenMPRuntime::LastprivateConditionalRAII
13158CGOpenMPRuntime::LastprivateConditionalRAII::disable(
13159 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
13160 return LastprivateConditionalRAII(CGF, S);
13161}
13162
13163CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() {
13164 if (CGM.getLangOpts().OpenMP < 50)
13165 return;
13166 if (Action == ActionToDo::DisableLastprivateConditional) {
13167 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13168 "Expected list of disabled private vars.");
13169 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13170 }
13171 if (Action == ActionToDo::PushAsLastprivateConditional) {
13172 assert(
13173 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
13174 "Expected list of lastprivate conditional vars.");
13175 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
13176 }
13177}
13178
13179Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
13180 const VarDecl *VD) {
13181 ASTContext &C = CGM.getContext();
13182 auto I = LastprivateConditionalToTypes.try_emplace(Key: CGF.CurFn).first;
13183 QualType NewType;
13184 const FieldDecl *VDField;
13185 const FieldDecl *FiredField;
13186 LValue BaseLVal;
13187 auto VI = I->getSecond().find(Val: VD);
13188 if (VI == I->getSecond().end()) {
13189 RecordDecl *RD = C.buildImplicitRecord(Name: "lasprivate.conditional");
13190 RD->startDefinition();
13191 VDField = addFieldToRecordDecl(C, DC: RD, FieldTy: VD->getType().getNonReferenceType());
13192 FiredField = addFieldToRecordDecl(C, DC: RD, FieldTy: C.CharTy);
13193 RD->completeDefinition();
13194 NewType = C.getCanonicalTagType(TD: RD);
13195 Address Addr = CGF.CreateMemTemp(T: NewType, Align: C.getDeclAlign(D: VD), Name: VD->getName());
13196 BaseLVal = CGF.MakeAddrLValue(Addr, T: NewType, Source: AlignmentSource::Decl);
13197 I->getSecond().try_emplace(Key: VD, Args&: NewType, Args&: VDField, Args&: FiredField, Args&: BaseLVal);
13198 } else {
13199 NewType = std::get<0>(t&: VI->getSecond());
13200 VDField = std::get<1>(t&: VI->getSecond());
13201 FiredField = std::get<2>(t&: VI->getSecond());
13202 BaseLVal = std::get<3>(t&: VI->getSecond());
13203 }
13204 LValue FiredLVal =
13205 CGF.EmitLValueForField(Base: BaseLVal, Field: FiredField);
13206 CGF.EmitStoreOfScalar(
13207 value: llvm::ConstantInt::getNullValue(Ty: CGF.ConvertTypeForMem(T: C.CharTy)),
13208 lvalue: FiredLVal);
13209 return CGF.EmitLValueForField(Base: BaseLVal, Field: VDField).getAddress();
13210}
13211
13212namespace {
13213/// Checks if the lastprivate conditional variable is referenced in LHS.
13214class LastprivateConditionalRefChecker final
13215 : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {
13216 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM;
13217 const Expr *FoundE = nullptr;
13218 const Decl *FoundD = nullptr;
13219 StringRef UniqueDeclName;
13220 LValue IVLVal;
13221 llvm::Function *FoundFn = nullptr;
13222 SourceLocation Loc;
13223
13224public:
13225 bool VisitDeclRefExpr(const DeclRefExpr *E) {
13226 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
13227 llvm::reverse(C&: LPM)) {
13228 auto It = D.DeclToUniqueName.find(Key: E->getDecl());
13229 if (It == D.DeclToUniqueName.end())
13230 continue;
13231 if (D.Disabled)
13232 return false;
13233 FoundE = E;
13234 FoundD = E->getDecl()->getCanonicalDecl();
13235 UniqueDeclName = It->second;
13236 IVLVal = D.IVLVal;
13237 FoundFn = D.Fn;
13238 break;
13239 }
13240 return FoundE == E;
13241 }
13242 bool VisitMemberExpr(const MemberExpr *E) {
13243 if (!CodeGenFunction::IsWrappedCXXThis(E: E->getBase()))
13244 return false;
13245 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
13246 llvm::reverse(C&: LPM)) {
13247 auto It = D.DeclToUniqueName.find(Key: E->getMemberDecl());
13248 if (It == D.DeclToUniqueName.end())
13249 continue;
13250 if (D.Disabled)
13251 return false;
13252 FoundE = E;
13253 FoundD = E->getMemberDecl()->getCanonicalDecl();
13254 UniqueDeclName = It->second;
13255 IVLVal = D.IVLVal;
13256 FoundFn = D.Fn;
13257 break;
13258 }
13259 return FoundE == E;
13260 }
13261 bool VisitStmt(const Stmt *S) {
13262 for (const Stmt *Child : S->children()) {
13263 if (!Child)
13264 continue;
13265 if (const auto *E = dyn_cast<Expr>(Val: Child))
13266 if (!E->isGLValue())
13267 continue;
13268 if (Visit(S: Child))
13269 return true;
13270 }
13271 return false;
13272 }
13273 explicit LastprivateConditionalRefChecker(
13274 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
13275 : LPM(LPM) {}
13276 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
13277 getFoundData() const {
13278 return std::make_tuple(args: FoundE, args: FoundD, args: UniqueDeclName, args: IVLVal, args: FoundFn);
13279 }
13280};
13281} // namespace
13282
13283void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
13284 LValue IVLVal,
13285 StringRef UniqueDeclName,
13286 LValue LVal,
13287 SourceLocation Loc) {
13288 // Last updated loop counter for the lastprivate conditional var.
13289 // int<xx> last_iv = 0;
13290 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(T: IVLVal.getType());
13291 llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable(
13292 Ty: LLIVTy, Name: getName(Parts: {UniqueDeclName, "iv"}));
13293 cast<llvm::GlobalVariable>(Val: LastIV)->setAlignment(
13294 IVLVal.getAlignment().getAsAlign());
13295 LValue LastIVLVal =
13296 CGF.MakeNaturalAlignRawAddrLValue(V: LastIV, T: IVLVal.getType());
13297
13298 // Last value of the lastprivate conditional.
13299 // decltype(priv_a) last_a;
13300 llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
13301 Ty: CGF.ConvertTypeForMem(T: LVal.getType()), Name: UniqueDeclName);
13302 cast<llvm::GlobalVariable>(Val: Last)->setAlignment(
13303 LVal.getAlignment().getAsAlign());
13304 LValue LastLVal =
13305 CGF.MakeRawAddrLValue(V: Last, T: LVal.getType(), Alignment: LVal.getAlignment());
13306
13307 // Global loop counter. Required to handle inner parallel-for regions.
13308 // iv
13309 llvm::Value *IVVal = CGF.EmitLoadOfScalar(lvalue: IVLVal, Loc);
13310
13311 // #pragma omp critical(a)
13312 // if (last_iv <= iv) {
13313 // last_iv = iv;
13314 // last_a = priv_a;
13315 // }
13316 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
13317 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
13318 Action.Enter(CGF);
13319 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(lvalue: LastIVLVal, Loc);
13320 // (last_iv <= iv) ? Check if the variable is updated and store new
13321 // value in global var.
13322 llvm::Value *CmpRes;
13323 if (IVLVal.getType()->isSignedIntegerType()) {
13324 CmpRes = CGF.Builder.CreateICmpSLE(LHS: LastIVVal, RHS: IVVal);
13325 } else {
13326 assert(IVLVal.getType()->isUnsignedIntegerType() &&
13327 "Loop iteration variable must be integer.");
13328 CmpRes = CGF.Builder.CreateICmpULE(LHS: LastIVVal, RHS: IVVal);
13329 }
13330 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lp_cond_then");
13331 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "lp_cond_exit");
13332 CGF.Builder.CreateCondBr(Cond: CmpRes, True: ThenBB, False: ExitBB);
13333 // {
13334 CGF.EmitBlock(BB: ThenBB);
13335
13336 // last_iv = iv;
13337 CGF.EmitStoreOfScalar(value: IVVal, lvalue: LastIVLVal);
13338
13339 // last_a = priv_a;
13340 switch (CGF.getEvaluationKind(T: LVal.getType())) {
13341 case TEK_Scalar: {
13342 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
13343 CGF.EmitStoreOfScalar(value: PrivVal, lvalue: LastLVal);
13344 break;
13345 }
13346 case TEK_Complex: {
13347 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(src: LVal, loc: Loc);
13348 CGF.EmitStoreOfComplex(V: PrivVal, dest: LastLVal, /*isInit=*/false);
13349 break;
13350 }
13351 case TEK_Aggregate:
13352 llvm_unreachable(
13353 "Aggregates are not supported in lastprivate conditional.");
13354 }
13355 // }
13356 CGF.EmitBranch(Block: ExitBB);
13357 // There is no need to emit line number for unconditional branch.
13358 (void)ApplyDebugLocation::CreateEmpty(CGF);
13359 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
13360 };
13361
13362 if (CGM.getLangOpts().OpenMPSimd) {
13363 // Do not emit as a critical region as no parallel region could be emitted.
13364 RegionCodeGenTy ThenRCG(CodeGen);
13365 ThenRCG(CGF);
13366 } else {
13367 emitCriticalRegion(CGF, CriticalName: UniqueDeclName, CriticalOpGen: CodeGen, Loc);
13368 }
13369}
13370
13371void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
13372 const Expr *LHS) {
13373 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13374 return;
13375 LastprivateConditionalRefChecker Checker(LastprivateConditionalStack);
13376 if (!Checker.Visit(S: LHS))
13377 return;
13378 const Expr *FoundE;
13379 const Decl *FoundD;
13380 StringRef UniqueDeclName;
13381 LValue IVLVal;
13382 llvm::Function *FoundFn;
13383 std::tie(args&: FoundE, args&: FoundD, args&: UniqueDeclName, args&: IVLVal, args&: FoundFn) =
13384 Checker.getFoundData();
13385 if (FoundFn != CGF.CurFn) {
13386 // Special codegen for inner parallel regions.
13387 // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1;
13388 auto It = LastprivateConditionalToTypes[FoundFn].find(Val: FoundD);
13389 assert(It != LastprivateConditionalToTypes[FoundFn].end() &&
13390 "Lastprivate conditional is not found in outer region.");
13391 QualType StructTy = std::get<0>(t&: It->getSecond());
13392 const FieldDecl* FiredDecl = std::get<2>(t&: It->getSecond());
13393 LValue PrivLVal = CGF.EmitLValue(E: FoundE);
13394 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
13395 Addr: PrivLVal.getAddress(),
13396 Ty: CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: StructTy)),
13397 ElementTy: CGF.ConvertTypeForMem(T: StructTy));
13398 LValue BaseLVal =
13399 CGF.MakeAddrLValue(Addr: StructAddr, T: StructTy, Source: AlignmentSource::Decl);
13400 LValue FiredLVal = CGF.EmitLValueForField(Base: BaseLVal, Field: FiredDecl);
13401 CGF.EmitAtomicStore(rvalue: RValue::get(V: llvm::ConstantInt::get(
13402 Ty: CGF.ConvertTypeForMem(T: FiredDecl->getType()), V: 1)),
13403 lvalue: FiredLVal, AO: llvm::AtomicOrdering::Unordered,
13404 /*IsVolatile=*/true, /*isInit=*/false);
13405 return;
13406 }
13407
13408 // Private address of the lastprivate conditional in the current context.
13409 // priv_a
13410 LValue LVal = CGF.EmitLValue(E: FoundE);
13411 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
13412 Loc: FoundE->getExprLoc());
13413}
13414
13415void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional(
13416 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13417 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) {
13418 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13419 return;
13420 auto Range = llvm::reverse(C&: LastprivateConditionalStack);
13421 auto It = llvm::find_if(
13422 Range, P: [](const LastprivateConditionalData &D) { return !D.Disabled; });
13423 if (It == Range.end() || It->Fn != CGF.CurFn)
13424 return;
13425 auto LPCI = LastprivateConditionalToTypes.find(Val: It->Fn);
13426 assert(LPCI != LastprivateConditionalToTypes.end() &&
13427 "Lastprivates must be registered already.");
13428 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
13429 getOpenMPCaptureRegions(CaptureRegions, DKind: D.getDirectiveKind());
13430 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: CaptureRegions.back());
13431 for (const auto &Pair : It->DeclToUniqueName) {
13432 const auto *VD = cast<VarDecl>(Val: Pair.first->getCanonicalDecl());
13433 if (!CS->capturesVariable(Var: VD) || IgnoredDecls.contains(V: VD))
13434 continue;
13435 auto I = LPCI->getSecond().find(Val: Pair.first);
13436 assert(I != LPCI->getSecond().end() &&
13437 "Lastprivate must be rehistered already.");
13438 // bool Cmp = priv_a.Fired != 0;
13439 LValue BaseLVal = std::get<3>(t&: I->getSecond());
13440 LValue FiredLVal =
13441 CGF.EmitLValueForField(Base: BaseLVal, Field: std::get<2>(t&: I->getSecond()));
13442 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: FiredLVal, Loc: D.getBeginLoc());
13443 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Res);
13444 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lpc.then");
13445 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "lpc.done");
13446 // if (Cmp) {
13447 CGF.Builder.CreateCondBr(Cond: Cmp, True: ThenBB, False: DoneBB);
13448 CGF.EmitBlock(BB: ThenBB);
13449 Address Addr = CGF.GetAddrOfLocalVar(VD);
13450 LValue LVal;
13451 if (VD->getType()->isReferenceType())
13452 LVal = CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
13453 Source: AlignmentSource::Decl);
13454 else
13455 LVal = CGF.MakeAddrLValue(Addr, T: VD->getType().getNonReferenceType(),
13456 Source: AlignmentSource::Decl);
13457 emitLastprivateConditionalUpdate(CGF, IVLVal: It->IVLVal, UniqueDeclName: Pair.second, LVal,
13458 Loc: D.getBeginLoc());
13459 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
13460 CGF.EmitBlock(BB: DoneBB, /*IsFinal=*/IsFinished: true);
13461 // }
13462 }
13463}
13464
13465void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
13466 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
13467 SourceLocation Loc) {
13468 if (CGF.getLangOpts().OpenMP < 50)
13469 return;
13470 auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(Key: VD);
13471 assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() &&
13472 "Unknown lastprivate conditional variable.");
13473 StringRef UniqueName = It->second;
13474 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: UniqueName);
13475 // The variable was not updated in the region - exit.
13476 if (!GV)
13477 return;
13478 LValue LPLVal = CGF.MakeRawAddrLValue(
13479 V: GV, T: PrivLVal.getType().getNonReferenceType(), Alignment: PrivLVal.getAlignment());
13480 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: LPLVal, Loc);
13481 CGF.EmitStoreOfScalar(value: Res, lvalue: PrivLVal);
13482}
13483
13484llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
13485 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13486 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13487 const RegionCodeGenTy &CodeGen) {
13488 llvm_unreachable("Not supported in SIMD-only mode");
13489}
13490
13491llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
13492 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13493 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13494 const RegionCodeGenTy &CodeGen) {
13495 llvm_unreachable("Not supported in SIMD-only mode");
13496}
13497
13498llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
13499 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
13500 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
13501 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
13502 bool Tied, unsigned &NumberOfParts) {
13503 llvm_unreachable("Not supported in SIMD-only mode");
13504}
13505
13506void CGOpenMPSIMDRuntime::emitParallelCall(
13507 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
13508 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
13509 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
13510 OpenMPSeverityClauseKind Severity, const Expr *Message) {
13511 llvm_unreachable("Not supported in SIMD-only mode");
13512}
13513
13514void CGOpenMPSIMDRuntime::emitCriticalRegion(
13515 CodeGenFunction &CGF, StringRef CriticalName,
13516 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
13517 const Expr *Hint) {
13518 llvm_unreachable("Not supported in SIMD-only mode");
13519}
13520
13521void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
13522 const RegionCodeGenTy &MasterOpGen,
13523 SourceLocation Loc) {
13524 llvm_unreachable("Not supported in SIMD-only mode");
13525}
13526
13527void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,
13528 const RegionCodeGenTy &MasterOpGen,
13529 SourceLocation Loc,
13530 const Expr *Filter) {
13531 llvm_unreachable("Not supported in SIMD-only mode");
13532}
13533
13534void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
13535 SourceLocation Loc) {
13536 llvm_unreachable("Not supported in SIMD-only mode");
13537}
13538
13539void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
13540 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
13541 SourceLocation Loc) {
13542 llvm_unreachable("Not supported in SIMD-only mode");
13543}
13544
13545void CGOpenMPSIMDRuntime::emitSingleRegion(
13546 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
13547 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
13548 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
13549 ArrayRef<const Expr *> AssignmentOps) {
13550 llvm_unreachable("Not supported in SIMD-only mode");
13551}
13552
13553void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
13554 const RegionCodeGenTy &OrderedOpGen,
13555 SourceLocation Loc,
13556 bool IsThreads) {
13557 llvm_unreachable("Not supported in SIMD-only mode");
13558}
13559
13560void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
13561 SourceLocation Loc,
13562 OpenMPDirectiveKind Kind,
13563 bool EmitChecks,
13564 bool ForceSimpleCall) {
13565 llvm_unreachable("Not supported in SIMD-only mode");
13566}
13567
13568void CGOpenMPSIMDRuntime::emitForDispatchInit(
13569 CodeGenFunction &CGF, SourceLocation Loc,
13570 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
13571 bool Ordered, const DispatchRTInput &DispatchValues) {
13572 llvm_unreachable("Not supported in SIMD-only mode");
13573}
13574
13575void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
13576 SourceLocation Loc) {
13577 llvm_unreachable("Not supported in SIMD-only mode");
13578}
13579
13580void CGOpenMPSIMDRuntime::emitForStaticInit(
13581 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
13582 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
13583 llvm_unreachable("Not supported in SIMD-only mode");
13584}
13585
13586void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
13587 CodeGenFunction &CGF, SourceLocation Loc,
13588 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
13589 llvm_unreachable("Not supported in SIMD-only mode");
13590}
13591
13592void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
13593 SourceLocation Loc,
13594 unsigned IVSize,
13595 bool IVSigned) {
13596 llvm_unreachable("Not supported in SIMD-only mode");
13597}
13598
13599void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
13600 SourceLocation Loc,
13601 OpenMPDirectiveKind DKind) {
13602 llvm_unreachable("Not supported in SIMD-only mode");
13603}
13604
13605llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
13606 SourceLocation Loc,
13607 unsigned IVSize, bool IVSigned,
13608 Address IL, Address LB,
13609 Address UB, Address ST) {
13610 llvm_unreachable("Not supported in SIMD-only mode");
13611}
13612
13613void CGOpenMPSIMDRuntime::emitNumThreadsClause(
13614 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
13615 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
13616 SourceLocation SeverityLoc, const Expr *Message,
13617 SourceLocation MessageLoc) {
13618 llvm_unreachable("Not supported in SIMD-only mode");
13619}
13620
13621void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
13622 ProcBindKind ProcBind,
13623 SourceLocation Loc) {
13624 llvm_unreachable("Not supported in SIMD-only mode");
13625}
13626
13627Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
13628 const VarDecl *VD,
13629 Address VDAddr,
13630 SourceLocation Loc) {
13631 llvm_unreachable("Not supported in SIMD-only mode");
13632}
13633
13634llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
13635 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
13636 CodeGenFunction *CGF) {
13637 llvm_unreachable("Not supported in SIMD-only mode");
13638}
13639
13640Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
13641 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
13642 llvm_unreachable("Not supported in SIMD-only mode");
13643}
13644
13645void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
13646 ArrayRef<const Expr *> Vars,
13647 SourceLocation Loc,
13648 llvm::AtomicOrdering AO) {
13649 llvm_unreachable("Not supported in SIMD-only mode");
13650}
13651
13652void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
13653 const OMPExecutableDirective &D,
13654 llvm::Function *TaskFunction,
13655 QualType SharedsTy, Address Shareds,
13656 const Expr *IfCond,
13657 const OMPTaskDataTy &Data) {
13658 llvm_unreachable("Not supported in SIMD-only mode");
13659}
13660
13661void CGOpenMPSIMDRuntime::emitTaskLoopCall(
13662 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
13663 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
13664 const Expr *IfCond, const OMPTaskDataTy &Data) {
13665 llvm_unreachable("Not supported in SIMD-only mode");
13666}
13667
13668void CGOpenMPSIMDRuntime::emitReduction(
13669 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
13670 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
13671 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
13672 assert(Options.SimpleReduction && "Only simple reduction is expected.");
13673 CGOpenMPRuntime::emitReduction(CGF, Loc, OrgPrivates: Privates, OrgLHSExprs: LHSExprs, OrgRHSExprs: RHSExprs,
13674 OrgReductionOps: ReductionOps, Options);
13675}
13676
13677llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
13678 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
13679 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
13680 llvm_unreachable("Not supported in SIMD-only mode");
13681}
13682
13683void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
13684 SourceLocation Loc,
13685 bool IsWorksharingReduction) {
13686 llvm_unreachable("Not supported in SIMD-only mode");
13687}
13688
13689void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
13690 SourceLocation Loc,
13691 ReductionCodeGen &RCG,
13692 unsigned N) {
13693 llvm_unreachable("Not supported in SIMD-only mode");
13694}
13695
13696Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
13697 SourceLocation Loc,
13698 llvm::Value *ReductionsPtr,
13699 LValue SharedLVal) {
13700 llvm_unreachable("Not supported in SIMD-only mode");
13701}
13702
13703void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
13704 SourceLocation Loc,
13705 const OMPTaskDataTy &Data) {
13706 llvm_unreachable("Not supported in SIMD-only mode");
13707}
13708
13709void CGOpenMPSIMDRuntime::emitCancellationPointCall(
13710 CodeGenFunction &CGF, SourceLocation Loc,
13711 OpenMPDirectiveKind CancelRegion) {
13712 llvm_unreachable("Not supported in SIMD-only mode");
13713}
13714
13715void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
13716 SourceLocation Loc, const Expr *IfCond,
13717 OpenMPDirectiveKind CancelRegion) {
13718 llvm_unreachable("Not supported in SIMD-only mode");
13719}
13720
13721void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
13722 const OMPExecutableDirective &D, StringRef ParentName,
13723 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
13724 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
13725 llvm_unreachable("Not supported in SIMD-only mode");
13726}
13727
13728void CGOpenMPSIMDRuntime::emitTargetCall(
13729 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13730 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
13731 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
13732 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
13733 const OMPLoopDirective &D)>
13734 SizeEmitter) {
13735 llvm_unreachable("Not supported in SIMD-only mode");
13736}
13737
13738bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
13739 llvm_unreachable("Not supported in SIMD-only mode");
13740}
13741
13742bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
13743 llvm_unreachable("Not supported in SIMD-only mode");
13744}
13745
13746bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
13747 return false;
13748}
13749
13750void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
13751 const OMPExecutableDirective &D,
13752 SourceLocation Loc,
13753 llvm::Function *OutlinedFn,
13754 ArrayRef<llvm::Value *> CapturedVars) {
13755 llvm_unreachable("Not supported in SIMD-only mode");
13756}
13757
13758void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
13759 const Expr *NumTeams,
13760 const Expr *ThreadLimit,
13761 SourceLocation Loc) {
13762 llvm_unreachable("Not supported in SIMD-only mode");
13763}
13764
13765void CGOpenMPSIMDRuntime::emitTargetDataCalls(
13766 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13767 const Expr *Device, const RegionCodeGenTy &CodeGen,
13768 CGOpenMPRuntime::TargetDataInfo &Info) {
13769 llvm_unreachable("Not supported in SIMD-only mode");
13770}
13771
13772void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
13773 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13774 const Expr *Device) {
13775 llvm_unreachable("Not supported in SIMD-only mode");
13776}
13777
13778void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
13779 const OMPLoopDirective &D,
13780 ArrayRef<Expr *> NumIterations) {
13781 llvm_unreachable("Not supported in SIMD-only mode");
13782}
13783
13784void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13785 const OMPDependClause *C) {
13786 llvm_unreachable("Not supported in SIMD-only mode");
13787}
13788
13789void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13790 const OMPDoacrossClause *C) {
13791 llvm_unreachable("Not supported in SIMD-only mode");
13792}
13793
13794const VarDecl *
13795CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
13796 const VarDecl *NativeParam) const {
13797 llvm_unreachable("Not supported in SIMD-only mode");
13798}
13799
13800Address
13801CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
13802 const VarDecl *NativeParam,
13803 const VarDecl *TargetParam) const {
13804 llvm_unreachable("Not supported in SIMD-only mode");
13805}
13806