1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "ABIInfoImpl.h"
15#include "CGCXXABI.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGRecordLayout.h"
19#include "CodeGenFunction.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/Attr.h"
23#include "clang/AST/Decl.h"
24#include "clang/AST/OpenMPClause.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/SourceManager.h"
30#include "clang/CodeGen/ConstantInitBuilder.h"
31#include "llvm/ADT/ArrayRef.h"
32#include "llvm/ADT/SmallSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Bitcode/BitcodeReader.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/GlobalValue.h"
39#include "llvm/IR/InstrTypes.h"
40#include "llvm/IR/Value.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/raw_ostream.h"
43#include <cassert>
44#include <cstdint>
45#include <numeric>
46#include <optional>
47
48using namespace clang;
49using namespace CodeGen;
50using namespace llvm::omp;
51
52namespace {
53/// Base class for handling code generation inside OpenMP regions.
54class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
55public:
56 /// Kinds of OpenMP regions used in codegen.
57 enum CGOpenMPRegionKind {
58 /// Region with outlined function for standalone 'parallel'
59 /// directive.
60 ParallelOutlinedRegion,
61 /// Region with outlined function for standalone 'task' directive.
62 TaskOutlinedRegion,
63 /// Region for constructs that do not require function outlining,
64 /// like 'for', 'sections', 'atomic' etc. directives.
65 InlinedRegion,
66 /// Region with outlined function for standalone 'target' directive.
67 TargetRegion,
68 };
69
70 CGOpenMPRegionInfo(const CapturedStmt &CS,
71 const CGOpenMPRegionKind RegionKind,
72 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
73 bool HasCancel)
74 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
75 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
76
77 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
78 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
79 bool HasCancel)
80 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
81 Kind(Kind), HasCancel(HasCancel) {}
82
83 /// Get a variable or parameter for storing global thread id
84 /// inside OpenMP construct.
85 virtual const VarDecl *getThreadIDVariable() const = 0;
86
87 /// Emit the captured statement body.
88 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
89
90 /// Get an LValue for the current ThreadID variable.
91 /// \return LValue for thread id variable. This LValue always has type int32*.
92 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
93
94 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
95
96 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
97
98 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
99
100 bool hasCancel() const { return HasCancel; }
101
102 static bool classof(const CGCapturedStmtInfo *Info) {
103 return Info->getKind() == CR_OpenMP;
104 }
105
106 ~CGOpenMPRegionInfo() override = default;
107
108protected:
109 CGOpenMPRegionKind RegionKind;
110 RegionCodeGenTy CodeGen;
111 OpenMPDirectiveKind Kind;
112 bool HasCancel;
113};
114
115/// API for captured statement code generation in OpenMP constructs.
116class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
117public:
118 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
119 const RegionCodeGenTy &CodeGen,
120 OpenMPDirectiveKind Kind, bool HasCancel,
121 StringRef HelperName)
122 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
123 HasCancel),
124 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
125 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
126 }
127
128 /// Get a variable or parameter for storing global thread id
129 /// inside OpenMP construct.
130 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
131
132 /// Get the name of the capture helper.
133 StringRef getHelperName() const override { return HelperName; }
134
135 static bool classof(const CGCapturedStmtInfo *Info) {
136 return CGOpenMPRegionInfo::classof(Info) &&
137 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
138 ParallelOutlinedRegion;
139 }
140
141private:
142 /// A variable or parameter storing global thread id for OpenMP
143 /// constructs.
144 const VarDecl *ThreadIDVar;
145 StringRef HelperName;
146};
147
148/// API for captured statement code generation in OpenMP constructs.
149class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
150public:
151 class UntiedTaskActionTy final : public PrePostActionTy {
152 bool Untied;
153 const VarDecl *PartIDVar;
154 const RegionCodeGenTy UntiedCodeGen;
155 llvm::SwitchInst *UntiedSwitch = nullptr;
156
157 public:
158 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
159 const RegionCodeGenTy &UntiedCodeGen)
160 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
161 void Enter(CodeGenFunction &CGF) override {
162 if (Untied) {
163 // Emit task switching point.
164 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
165 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
166 PtrTy: PartIDVar->getType()->castAs<PointerType>());
167 llvm::Value *Res =
168 CGF.EmitLoadOfScalar(lvalue: PartIdLVal, Loc: PartIDVar->getLocation());
169 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: ".untied.done.");
170 UntiedSwitch = CGF.Builder.CreateSwitch(V: Res, Dest: DoneBB);
171 CGF.EmitBlock(BB: DoneBB);
172 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
173 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
174 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: 0),
175 Dest: CGF.Builder.GetInsertBlock());
176 emitUntiedSwitch(CGF);
177 }
178 }
179 void emitUntiedSwitch(CodeGenFunction &CGF) const {
180 if (Untied) {
181 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
182 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
183 PtrTy: PartIDVar->getType()->castAs<PointerType>());
184 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
185 lvalue: PartIdLVal);
186 UntiedCodeGen(CGF);
187 CodeGenFunction::JumpDest CurPoint =
188 CGF.getJumpDestInCurrentScope(Name: ".untied.next.");
189 CGF.EmitBranch(Block: CGF.ReturnBlock.getBlock());
190 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
191 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
192 Dest: CGF.Builder.GetInsertBlock());
193 CGF.EmitBranchThroughCleanup(Dest: CurPoint);
194 CGF.EmitBlock(BB: CurPoint.getBlock());
195 }
196 }
197 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
198 };
199 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
200 const VarDecl *ThreadIDVar,
201 const RegionCodeGenTy &CodeGen,
202 OpenMPDirectiveKind Kind, bool HasCancel,
203 const UntiedTaskActionTy &Action)
204 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
205 ThreadIDVar(ThreadIDVar), Action(Action) {
206 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
207 }
208
209 /// Get a variable or parameter for storing global thread id
210 /// inside OpenMP construct.
211 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
212
213 /// Get an LValue for the current ThreadID variable.
214 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
215
216 /// Get the name of the capture helper.
217 StringRef getHelperName() const override { return ".omp_outlined."; }
218
219 void emitUntiedSwitch(CodeGenFunction &CGF) override {
220 Action.emitUntiedSwitch(CGF);
221 }
222
223 static bool classof(const CGCapturedStmtInfo *Info) {
224 return CGOpenMPRegionInfo::classof(Info) &&
225 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
226 TaskOutlinedRegion;
227 }
228
229private:
230 /// A variable or parameter storing global thread id for OpenMP
231 /// constructs.
232 const VarDecl *ThreadIDVar;
233 /// Action for emitting code for untied tasks.
234 const UntiedTaskActionTy &Action;
235};
236
237/// API for inlined captured statement code generation in OpenMP
238/// constructs.
239class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
240public:
241 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
242 const RegionCodeGenTy &CodeGen,
243 OpenMPDirectiveKind Kind, bool HasCancel)
244 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
245 OldCSI(OldCSI),
246 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(Val: OldCSI)) {}
247
248 // Retrieve the value of the context parameter.
249 llvm::Value *getContextValue() const override {
250 if (OuterRegionInfo)
251 return OuterRegionInfo->getContextValue();
252 llvm_unreachable("No context value for inlined OpenMP region");
253 }
254
255 void setContextValue(llvm::Value *V) override {
256 if (OuterRegionInfo) {
257 OuterRegionInfo->setContextValue(V);
258 return;
259 }
260 llvm_unreachable("No context value for inlined OpenMP region");
261 }
262
263 /// Lookup the captured field decl for a variable.
264 const FieldDecl *lookup(const VarDecl *VD) const override {
265 if (OuterRegionInfo)
266 return OuterRegionInfo->lookup(VD);
267 // If there is no outer outlined region,no need to lookup in a list of
268 // captured variables, we can use the original one.
269 return nullptr;
270 }
271
272 FieldDecl *getThisFieldDecl() const override {
273 if (OuterRegionInfo)
274 return OuterRegionInfo->getThisFieldDecl();
275 return nullptr;
276 }
277
278 /// Get a variable or parameter for storing global thread id
279 /// inside OpenMP construct.
280 const VarDecl *getThreadIDVariable() const override {
281 if (OuterRegionInfo)
282 return OuterRegionInfo->getThreadIDVariable();
283 return nullptr;
284 }
285
286 /// Get an LValue for the current ThreadID variable.
287 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
288 if (OuterRegionInfo)
289 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
290 llvm_unreachable("No LValue for inlined OpenMP construct");
291 }
292
293 /// Get the name of the capture helper.
294 StringRef getHelperName() const override {
295 if (auto *OuterRegionInfo = getOldCSI())
296 return OuterRegionInfo->getHelperName();
297 llvm_unreachable("No helper name for inlined OpenMP construct");
298 }
299
300 void emitUntiedSwitch(CodeGenFunction &CGF) override {
301 if (OuterRegionInfo)
302 OuterRegionInfo->emitUntiedSwitch(CGF);
303 }
304
305 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
306
307 static bool classof(const CGCapturedStmtInfo *Info) {
308 return CGOpenMPRegionInfo::classof(Info) &&
309 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == InlinedRegion;
310 }
311
312 ~CGOpenMPInlinedRegionInfo() override = default;
313
314private:
315 /// CodeGen info about outer OpenMP region.
316 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
317 CGOpenMPRegionInfo *OuterRegionInfo;
318};
319
320/// API for captured statement code generation in OpenMP target
321/// constructs. For this captures, implicit parameters are used instead of the
322/// captured fields. The name of the target region has to be unique in a given
323/// application so it is provided by the client, because only the client has
324/// the information to generate that.
325class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
326public:
327 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
328 const RegionCodeGenTy &CodeGen, StringRef HelperName)
329 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
330 /*HasCancel=*/false),
331 HelperName(HelperName) {}
332
333 /// This is unused for target regions because each starts executing
334 /// with a single thread.
335 const VarDecl *getThreadIDVariable() const override { return nullptr; }
336
337 /// Get the name of the capture helper.
338 StringRef getHelperName() const override { return HelperName; }
339
340 static bool classof(const CGCapturedStmtInfo *Info) {
341 return CGOpenMPRegionInfo::classof(Info) &&
342 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == TargetRegion;
343 }
344
345private:
346 StringRef HelperName;
347};
348
349static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
350 llvm_unreachable("No codegen for expressions");
351}
352/// API for generation of expressions captured in a innermost OpenMP
353/// region.
354class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
355public:
356 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
357 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
358 OMPD_unknown,
359 /*HasCancel=*/false),
360 PrivScope(CGF) {
361 // Make sure the globals captured in the provided statement are local by
362 // using the privatization logic. We assume the same variable is not
363 // captured more than once.
364 for (const auto &C : CS.captures()) {
365 if (!C.capturesVariable() && !C.capturesVariableByCopy())
366 continue;
367
368 const VarDecl *VD = C.getCapturedVar();
369 if (VD->isLocalVarDeclOrParm())
370 continue;
371
372 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
373 /*RefersToEnclosingVariableOrCapture=*/false,
374 VD->getType().getNonReferenceType(), VK_LValue,
375 C.getLocation());
376 PrivScope.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
377 }
378 (void)PrivScope.Privatize();
379 }
380
381 /// Lookup the captured field decl for a variable.
382 const FieldDecl *lookup(const VarDecl *VD) const override {
383 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
384 return FD;
385 return nullptr;
386 }
387
388 /// Emit the captured statement body.
389 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
390 llvm_unreachable("No body for expressions");
391 }
392
393 /// Get a variable or parameter for storing global thread id
394 /// inside OpenMP construct.
395 const VarDecl *getThreadIDVariable() const override {
396 llvm_unreachable("No thread id for expressions");
397 }
398
399 /// Get the name of the capture helper.
400 StringRef getHelperName() const override {
401 llvm_unreachable("No helper name for expressions");
402 }
403
404 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
405
406private:
407 /// Private scope to capture global variables.
408 CodeGenFunction::OMPPrivateScope PrivScope;
409};
410
411/// RAII for emitting code of OpenMP constructs.
412class InlinedOpenMPRegionRAII {
413 CodeGenFunction &CGF;
414 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
415 FieldDecl *LambdaThisCaptureField = nullptr;
416 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
417 bool NoInheritance = false;
418
419public:
420 /// Constructs region for combined constructs.
421 /// \param CodeGen Code generation sequence for combined directives. Includes
422 /// a list of functions used for code generation of implicitly inlined
423 /// regions.
424 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
425 OpenMPDirectiveKind Kind, bool HasCancel,
426 bool NoInheritance = true)
427 : CGF(CGF), NoInheritance(NoInheritance) {
428 // Start emission for the construct.
429 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
430 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
431 if (NoInheritance) {
432 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
433 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
434 CGF.LambdaThisCaptureField = nullptr;
435 BlockInfo = CGF.BlockInfo;
436 CGF.BlockInfo = nullptr;
437 }
438 }
439
440 ~InlinedOpenMPRegionRAII() {
441 // Restore original CapturedStmtInfo only if we're done with code emission.
442 auto *OldCSI =
443 cast<CGOpenMPInlinedRegionInfo>(Val: CGF.CapturedStmtInfo)->getOldCSI();
444 delete CGF.CapturedStmtInfo;
445 CGF.CapturedStmtInfo = OldCSI;
446 if (NoInheritance) {
447 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
448 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
449 CGF.BlockInfo = BlockInfo;
450 }
451 }
452};
453
454/// Values for bit flags used in the ident_t to describe the fields.
455/// All enumeric elements are named and described in accordance with the code
456/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
457enum OpenMPLocationFlags : unsigned {
458 /// Use trampoline for internal microtask.
459 OMP_IDENT_IMD = 0x01,
460 /// Use c-style ident structure.
461 OMP_IDENT_KMPC = 0x02,
462 /// Atomic reduction option for kmpc_reduce.
463 OMP_ATOMIC_REDUCE = 0x10,
464 /// Explicit 'barrier' directive.
465 OMP_IDENT_BARRIER_EXPL = 0x20,
466 /// Implicit barrier in code.
467 OMP_IDENT_BARRIER_IMPL = 0x40,
468 /// Implicit barrier in 'for' directive.
469 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
470 /// Implicit barrier in 'sections' directive.
471 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
472 /// Implicit barrier in 'single' directive.
473 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
474 /// Call of __kmp_for_static_init for static loop.
475 OMP_IDENT_WORK_LOOP = 0x200,
476 /// Call of __kmp_for_static_init for sections.
477 OMP_IDENT_WORK_SECTIONS = 0x400,
478 /// Call of __kmp_for_static_init for distribute.
479 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
480 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
481};
482
483/// Describes ident structure that describes a source location.
484/// All descriptions are taken from
485/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
486/// Original structure:
487/// typedef struct ident {
488/// kmp_int32 reserved_1; /**< might be used in Fortran;
489/// see above */
490/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
491/// KMP_IDENT_KMPC identifies this union
492/// member */
493/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
494/// see above */
495///#if USE_ITT_BUILD
496/// /* but currently used for storing
497/// region-specific ITT */
498/// /* contextual information. */
499///#endif /* USE_ITT_BUILD */
500/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
501/// C++ */
502/// char const *psource; /**< String describing the source location.
503/// The string is composed of semi-colon separated
504// fields which describe the source file,
505/// the function and a pair of line numbers that
506/// delimit the construct.
507/// */
508/// } ident_t;
509enum IdentFieldIndex {
510 /// might be used in Fortran
511 IdentField_Reserved_1,
512 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
513 IdentField_Flags,
514 /// Not really used in Fortran any more
515 IdentField_Reserved_2,
516 /// Source[4] in Fortran, do not use for C++
517 IdentField_Reserved_3,
518 /// String describing the source location. The string is composed of
519 /// semi-colon separated fields which describe the source file, the function
520 /// and a pair of line numbers that delimit the construct.
521 IdentField_PSource
522};
523
524/// Schedule types for 'omp for' loops (these enumerators are taken from
525/// the enum sched_type in kmp.h).
526enum OpenMPSchedType {
527 /// Lower bound for default (unordered) versions.
528 OMP_sch_lower = 32,
529 OMP_sch_static_chunked = 33,
530 OMP_sch_static = 34,
531 OMP_sch_dynamic_chunked = 35,
532 OMP_sch_guided_chunked = 36,
533 OMP_sch_runtime = 37,
534 OMP_sch_auto = 38,
535 /// static with chunk adjustment (e.g., simd)
536 OMP_sch_static_balanced_chunked = 45,
537 /// Lower bound for 'ordered' versions.
538 OMP_ord_lower = 64,
539 OMP_ord_static_chunked = 65,
540 OMP_ord_static = 66,
541 OMP_ord_dynamic_chunked = 67,
542 OMP_ord_guided_chunked = 68,
543 OMP_ord_runtime = 69,
544 OMP_ord_auto = 70,
545 OMP_sch_default = OMP_sch_static,
546 /// dist_schedule types
547 OMP_dist_sch_static_chunked = 91,
548 OMP_dist_sch_static = 92,
549 /// Fused distribute+for static schedule (entityId = team*nthreads + tid,
550 /// num_entities = nteams*nthreads). One for_static_init call, no
551 /// surrounding distribute_static_init. Matches
552 /// kmp_sched_distr_static_chunk_sched_static_chunkone in the device RTL
553 /// (openmp/device/include/DeviceTypes.h).
554 OMP_dist_sch_static_chunked_sch_static_chunkone = 93,
555 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
556 /// Set if the monotonic schedule modifier was present.
557 OMP_sch_modifier_monotonic = (1 << 29),
558 /// Set if the nonmonotonic schedule modifier was present.
559 OMP_sch_modifier_nonmonotonic = (1 << 30),
560};
561
562/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
563/// region.
564class CleanupTy final : public EHScopeStack::Cleanup {
565 PrePostActionTy *Action;
566
567public:
568 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
569 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
570 if (!CGF.HaveInsertPoint())
571 return;
572 Action->Exit(CGF);
573 }
574};
575
576} // anonymous namespace
577
578void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
579 CodeGenFunction::RunCleanupsScope Scope(CGF);
580 if (PrePostAction) {
581 CGF.EHStack.pushCleanup<CleanupTy>(Kind: NormalAndEHCleanup, A: PrePostAction);
582 Callback(CodeGen, CGF, *PrePostAction);
583 } else {
584 PrePostActionTy Action;
585 Callback(CodeGen, CGF, Action);
586 }
587}
588
589/// Check if the combiner is a call to UDR combiner and if it is so return the
590/// UDR decl used for reduction.
591static const OMPDeclareReductionDecl *
592getReductionInit(const Expr *ReductionOp) {
593 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
594 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
595 if (const auto *DRE =
596 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
597 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl()))
598 return DRD;
599 return nullptr;
600}
601
602static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
603 const OMPDeclareReductionDecl *DRD,
604 const Expr *InitOp,
605 Address Private, Address Original,
606 QualType Ty) {
607 if (DRD->getInitializer()) {
608 std::pair<llvm::Function *, llvm::Function *> Reduction =
609 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
610 const auto *CE = cast<CallExpr>(Val: InitOp);
611 const auto *OVE = cast<OpaqueValueExpr>(Val: CE->getCallee());
612 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
613 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
614 const auto *LHSDRE =
615 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: LHS)->getSubExpr());
616 const auto *RHSDRE =
617 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: RHS)->getSubExpr());
618 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
619 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: LHSDRE->getDecl()), Addr: Private);
620 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: RHSDRE->getDecl()), Addr: Original);
621 (void)PrivateScope.Privatize();
622 RValue Func = RValue::get(V: Reduction.second);
623 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
624 CGF.EmitIgnoredExpr(E: InitOp);
625 } else {
626 llvm::Constant *Init = CGF.CGM.EmitNullConstant(T: Ty);
627 std::string Name = CGF.CGM.getOpenMPRuntime().getName(Parts: {"init"});
628 auto *GV = new llvm::GlobalVariable(
629 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
630 llvm::GlobalValue::PrivateLinkage, Init, Name);
631 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V: GV, T: Ty);
632 RValue InitRVal;
633 switch (CGF.getEvaluationKind(T: Ty)) {
634 case TEK_Scalar:
635 InitRVal = CGF.EmitLoadOfLValue(V: LV, Loc: DRD->getLocation());
636 break;
637 case TEK_Complex:
638 InitRVal =
639 RValue::getComplex(C: CGF.EmitLoadOfComplex(src: LV, loc: DRD->getLocation()));
640 break;
641 case TEK_Aggregate: {
642 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);
643 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
644 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
645 /*IsInitializer=*/false);
646 return;
647 }
648 }
649 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue);
650 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
651 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
652 /*IsInitializer=*/false);
653 }
654}
655
656/// Emit initialization of arrays of complex types.
657/// \param DestAddr Address of the array.
658/// \param Type Type of array.
659/// \param Init Initial expression of array.
660/// \param SrcAddr Address of the original array.
661static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
662 QualType Type, bool EmitDeclareReductionInit,
663 const Expr *Init,
664 const OMPDeclareReductionDecl *DRD,
665 Address SrcAddr = Address::invalid()) {
666 // Perform element-by-element initialization.
667 QualType ElementTy;
668
669 // Drill down to the base element type on both arrays.
670 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
671 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
672 if (DRD)
673 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
674
675 llvm::Value *SrcBegin = nullptr;
676 if (DRD)
677 SrcBegin = SrcAddr.emitRawPointer(CGF);
678 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
679 // Cast from pointer to array type to pointer to single element.
680 llvm::Value *DestEnd =
681 CGF.Builder.CreateGEP(Ty: DestAddr.getElementType(), Ptr: DestBegin, IdxList: NumElements);
682 // The basic structure here is a while-do loop.
683 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arrayinit.body");
684 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arrayinit.done");
685 llvm::Value *IsEmpty =
686 CGF.Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arrayinit.isempty");
687 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
688
689 // Enter the loop body, making that address the current address.
690 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
691 CGF.EmitBlock(BB: BodyBB);
692
693 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
694
695 llvm::PHINode *SrcElementPHI = nullptr;
696 Address SrcElementCurrent = Address::invalid();
697 if (DRD) {
698 SrcElementPHI = CGF.Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2,
699 Name: "omp.arraycpy.srcElementPast");
700 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
701 SrcElementCurrent =
702 Address(SrcElementPHI, SrcAddr.getElementType(),
703 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
704 }
705 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
706 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
707 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
708 Address DestElementCurrent =
709 Address(DestElementPHI, DestAddr.getElementType(),
710 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
711
712 // Emit copy.
713 {
714 CodeGenFunction::RunCleanupsScope InitScope(CGF);
715 if (EmitDeclareReductionInit) {
716 emitInitWithReductionInitializer(CGF, DRD, InitOp: Init, Private: DestElementCurrent,
717 Original: SrcElementCurrent, Ty: ElementTy);
718 } else
719 CGF.EmitAnyExprToMem(E: Init, Location: DestElementCurrent, Quals: ElementTy.getQualifiers(),
720 /*IsInitializer=*/false);
721 }
722
723 if (DRD) {
724 // Shift the address forward by one element.
725 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
726 Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI, /*Idx0=*/1,
727 Name: "omp.arraycpy.dest.element");
728 SrcElementPHI->addIncoming(V: SrcElementNext, BB: CGF.Builder.GetInsertBlock());
729 }
730
731 // Shift the address forward by one element.
732 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
733 Ty: DestAddr.getElementType(), Ptr: DestElementPHI, /*Idx0=*/1,
734 Name: "omp.arraycpy.dest.element");
735 // Check whether we've reached the end.
736 llvm::Value *Done =
737 CGF.Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
738 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
739 DestElementPHI->addIncoming(V: DestElementNext, BB: CGF.Builder.GetInsertBlock());
740
741 // Done.
742 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
743}
744
745LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
746 return CGF.EmitOMPSharedLValue(E);
747}
748
749LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
750 const Expr *E) {
751 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E))
752 return CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false);
753 return LValue();
754}
755
756void ReductionCodeGen::emitAggregateInitialization(
757 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
758 const OMPDeclareReductionDecl *DRD) {
759 // Emit VarDecl with copy init for arrays.
760 // Get the address of the original variable captured in current
761 // captured region.
762 const auto *PrivateVD =
763 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
764 bool EmitDeclareReductionInit =
765 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
766 EmitOMPAggregateInit(CGF, DestAddr: PrivateAddr, Type: PrivateVD->getType(),
767 EmitDeclareReductionInit,
768 Init: EmitDeclareReductionInit ? ClausesData[N].ReductionOp
769 : PrivateVD->getInit(),
770 DRD, SrcAddr: SharedAddr);
771}
772
773ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
774 ArrayRef<const Expr *> Origs,
775 ArrayRef<const Expr *> Privates,
776 ArrayRef<const Expr *> ReductionOps) {
777 ClausesData.reserve(N: Shareds.size());
778 SharedAddresses.reserve(N: Shareds.size());
779 Sizes.reserve(N: Shareds.size());
780 BaseDecls.reserve(N: Shareds.size());
781 const auto *IOrig = Origs.begin();
782 const auto *IPriv = Privates.begin();
783 const auto *IRed = ReductionOps.begin();
784 for (const Expr *Ref : Shareds) {
785 ClausesData.emplace_back(Args&: Ref, Args: *IOrig, Args: *IPriv, Args: *IRed);
786 std::advance(i&: IOrig, n: 1);
787 std::advance(i&: IPriv, n: 1);
788 std::advance(i&: IRed, n: 1);
789 }
790}
791
792void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
793 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
794 "Number of generated lvalues must be exactly N.");
795 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Shared);
796 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Shared);
797 SharedAddresses.emplace_back(Args&: First, Args&: Second);
798 if (ClausesData[N].Shared == ClausesData[N].Ref) {
799 OrigAddresses.emplace_back(Args&: First, Args&: Second);
800 } else {
801 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Ref);
802 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Ref);
803 OrigAddresses.emplace_back(Args&: First, Args&: Second);
804 }
805}
806
807void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
808 QualType PrivateType = getPrivateType(N);
809 bool AsArraySection = isa<ArraySectionExpr>(Val: ClausesData[N].Ref);
810 if (!PrivateType->isVariablyModifiedType()) {
811 Sizes.emplace_back(
812 Args: CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType()),
813 Args: nullptr);
814 return;
815 }
816 llvm::Value *Size;
817 llvm::Value *SizeInChars;
818 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
819 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(Ty: ElemType);
820 if (AsArraySection) {
821 Size = CGF.Builder.CreatePtrDiff(ElemTy: ElemType,
822 LHS: OrigAddresses[N].second.getPointer(CGF),
823 RHS: OrigAddresses[N].first.getPointer(CGF));
824 Size = CGF.Builder.CreateZExtOrTrunc(V: Size, DestTy: ElemSizeOf->getType());
825 Size = CGF.Builder.CreateNUWAdd(
826 LHS: Size, RHS: llvm::ConstantInt::get(Ty: Size->getType(), /*V=*/1));
827 SizeInChars = CGF.Builder.CreateNUWMul(LHS: Size, RHS: ElemSizeOf);
828 } else {
829 SizeInChars =
830 CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType());
831 Size = CGF.Builder.CreateExactUDiv(LHS: SizeInChars, RHS: ElemSizeOf);
832 }
833 Sizes.emplace_back(Args&: SizeInChars, Args&: Size);
834 CodeGenFunction::OpaqueValueMapping OpaqueMap(
835 CGF,
836 cast<OpaqueValueExpr>(
837 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
838 RValue::get(V: Size));
839 CGF.EmitVariablyModifiedType(Ty: PrivateType);
840}
841
842void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
843 llvm::Value *Size) {
844 QualType PrivateType = getPrivateType(N);
845 if (!PrivateType->isVariablyModifiedType()) {
846 assert(!Size && !Sizes[N].second &&
847 "Size should be nullptr for non-variably modified reduction "
848 "items.");
849 return;
850 }
851 CodeGenFunction::OpaqueValueMapping OpaqueMap(
852 CGF,
853 cast<OpaqueValueExpr>(
854 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
855 RValue::get(V: Size));
856 CGF.EmitVariablyModifiedType(Ty: PrivateType);
857}
858
859void ReductionCodeGen::emitInitialization(
860 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
861 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
862 assert(SharedAddresses.size() > N && "No variable was generated");
863 const auto *PrivateVD =
864 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
865 const OMPDeclareReductionDecl *DRD =
866 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
867 if (CGF.getContext().getAsArrayType(T: PrivateVD->getType())) {
868 if (DRD && DRD->getInitializer())
869 (void)DefaultInit(CGF);
870 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
871 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
872 (void)DefaultInit(CGF);
873 QualType SharedType = SharedAddresses[N].first.getType();
874 emitInitWithReductionInitializer(CGF, DRD, InitOp: ClausesData[N].ReductionOp,
875 Private: PrivateAddr, Original: SharedAddr, Ty: SharedType);
876 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
877 !CGF.isTrivialInitializer(Init: PrivateVD->getInit())) {
878 CGF.EmitAnyExprToMem(E: PrivateVD->getInit(), Location: PrivateAddr,
879 Quals: PrivateVD->getType().getQualifiers(),
880 /*IsInitializer=*/false);
881 }
882}
883
884bool ReductionCodeGen::needCleanups(unsigned N) {
885 QualType PrivateType = getPrivateType(N);
886 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
887 return DTorKind != QualType::DK_none;
888}
889
890void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
891 Address PrivateAddr) {
892 QualType PrivateType = getPrivateType(N);
893 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
894 if (needCleanups(N)) {
895 PrivateAddr =
896 PrivateAddr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: PrivateType));
897 CGF.pushDestroy(dtorKind: DTorKind, addr: PrivateAddr, type: PrivateType);
898 }
899}
900
901static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
902 LValue BaseLV) {
903 BaseTy = BaseTy.getNonReferenceType();
904 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
905 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
906 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
907 BaseLV = CGF.EmitLoadOfPointerLValue(Ptr: BaseLV.getAddress(), PtrTy);
908 } else {
909 LValue RefLVal = CGF.MakeAddrLValue(Addr: BaseLV.getAddress(), T: BaseTy);
910 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
911 }
912 BaseTy = BaseTy->getPointeeType();
913 }
914 return CGF.MakeAddrLValue(
915 Addr: BaseLV.getAddress().withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy)),
916 T: BaseLV.getType(), BaseInfo: BaseLV.getBaseInfo(),
917 TBAAInfo: CGF.CGM.getTBAAInfoForSubobject(Base: BaseLV, AccessType: BaseLV.getType()));
918}
919
920static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
921 Address OriginalBaseAddress, llvm::Value *Addr) {
922 RawAddress Tmp = RawAddress::invalid();
923 Address TopTmp = Address::invalid();
924 Address MostTopTmp = Address::invalid();
925 BaseTy = BaseTy.getNonReferenceType();
926 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
927 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
928 Tmp = CGF.CreateMemTempWithoutCast(T: BaseTy);
929 if (TopTmp.isValid())
930 CGF.Builder.CreateStore(Val: Tmp.getPointer(), Addr: TopTmp);
931 else
932 MostTopTmp = Tmp;
933 TopTmp = Tmp;
934 BaseTy = BaseTy->getPointeeType();
935 }
936
937 if (Tmp.isValid()) {
938 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
939 V: Addr, DestTy: Tmp.getElementType());
940 CGF.Builder.CreateStore(Val: Addr, Addr: Tmp);
941 return MostTopTmp;
942 }
943
944 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
945 V: Addr, DestTy: OriginalBaseAddress.getType());
946 return OriginalBaseAddress.withPointer(NewPointer: Addr, IsKnownNonNull: NotKnownNonNull);
947}
948
949static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
950 const VarDecl *OrigVD = nullptr;
951 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Ref)) {
952 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
953 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Val: Base))
954 Base = TempOASE->getBase()->IgnoreParenImpCasts();
955 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
956 Base = TempASE->getBase()->IgnoreParenImpCasts();
957 DE = cast<DeclRefExpr>(Val: Base);
958 OrigVD = cast<VarDecl>(Val: DE->getDecl());
959 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Ref)) {
960 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
961 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
962 Base = TempASE->getBase()->IgnoreParenImpCasts();
963 DE = cast<DeclRefExpr>(Val: Base);
964 OrigVD = cast<VarDecl>(Val: DE->getDecl());
965 }
966 return OrigVD;
967}
968
969Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
970 Address PrivateAddr) {
971 const DeclRefExpr *DE;
972 if (const VarDecl *OrigVD = ::getBaseDecl(Ref: ClausesData[N].Ref, DE)) {
973 BaseDecls.emplace_back(Args&: OrigVD);
974 LValue OriginalBaseLValue = CGF.EmitLValue(E: DE);
975 LValue BaseLValue =
976 loadToBegin(CGF, BaseTy: OrigVD->getType(), ElTy: SharedAddresses[N].first.getType(),
977 BaseLV: OriginalBaseLValue);
978 Address SharedAddr = SharedAddresses[N].first.getAddress();
979 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
980 ElemTy: SharedAddr.getElementType(), LHS: BaseLValue.getPointer(CGF),
981 RHS: SharedAddr.emitRawPointer(CGF));
982 llvm::Value *PrivatePointer =
983 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
984 V: PrivateAddr.emitRawPointer(CGF), DestTy: SharedAddr.getType());
985 llvm::Value *Ptr = CGF.Builder.CreateGEP(
986 Ty: SharedAddr.getElementType(), Ptr: PrivatePointer, IdxList: Adjustment);
987 return castToBase(CGF, BaseTy: OrigVD->getType(),
988 ElTy: SharedAddresses[N].first.getType(),
989 OriginalBaseAddress: OriginalBaseLValue.getAddress(), Addr: Ptr);
990 }
991 BaseDecls.emplace_back(
992 Args: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Ref)->getDecl()));
993 return PrivateAddr;
994}
995
996bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
997 const OMPDeclareReductionDecl *DRD =
998 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
999 return DRD && DRD->getInitializer();
1000}
1001
1002LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1003 return CGF.EmitLoadOfPointerLValue(
1004 Ptr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1005 PtrTy: getThreadIDVariable()->getType()->castAs<PointerType>());
1006}
1007
1008void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1009 if (!CGF.HaveInsertPoint())
1010 return;
1011 // 1.2.2 OpenMP Language Terminology
1012 // Structured block - An executable statement with a single entry at the
1013 // top and a single exit at the bottom.
1014 // The point of exit cannot be a branch out of the structured block.
1015 // longjmp() and throw() must not violate the entry/exit criteria.
1016 CGF.EHStack.pushTerminate();
1017 if (S)
1018 CGF.incrementProfileCounter(S);
1019 CodeGen(CGF);
1020 CGF.EHStack.popTerminate();
1021}
1022
1023LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1024 CodeGenFunction &CGF) {
1025 return CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1026 T: getThreadIDVariable()->getType(),
1027 Source: AlignmentSource::Decl);
1028}
1029
1030static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1031 QualType FieldTy) {
1032 auto *Field = FieldDecl::Create(
1033 C, DC, StartLoc: SourceLocation(), IdLoc: SourceLocation(), /*Id=*/nullptr, T: FieldTy,
1034 TInfo: C.getTrivialTypeSourceInfo(T: FieldTy, Loc: SourceLocation()),
1035 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1036 Field->setAccess(AS_public);
1037 DC->addDecl(D: Field);
1038 return Field;
1039}
1040
1041CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
1042 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1043 KmpCriticalNameTy = llvm::ArrayType::get(ElementType: CGM.Int32Ty, /*NumElements*/ 8);
1044 llvm::OpenMPIRBuilderConfig Config(
1045 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),
1046 CGM.getLangOpts().OpenMPOffloadMandatory,
1047 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,
1048 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);
1049 Config.setDefaultTargetAS(
1050 CGM.getContext().getTargetInfo().getTargetAddressSpace(AS: LangAS::Default));
1051 Config.setRuntimeCC(CGM.getRuntimeCC());
1052
1053 OMPBuilder.setConfig(Config);
1054 OMPBuilder.initialize();
1055 OMPBuilder.loadOffloadInfoMetadata(VFS&: *CGM.getFileSystem(),
1056 HostFilePath: CGM.getLangOpts().OpenMPIsTargetDevice
1057 ? CGM.getLangOpts().OMPHostIRFile
1058 : StringRef{});
1059
1060 // The user forces the compiler to behave as if omp requires
1061 // unified_shared_memory was given.
1062 if (CGM.getLangOpts().OpenMPForceUSM) {
1063 HasRequiresUnifiedSharedMemory = true;
1064 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
1065 }
1066}
1067
1068void CGOpenMPRuntime::clear() {
1069 InternalVars.clear();
1070 // Clean non-target variable declarations possibly used only in debug info.
1071 for (const auto &Data : EmittedNonTargetVariables) {
1072 if (!Data.getValue().pointsToAliveValue())
1073 continue;
1074 auto *GV = dyn_cast<llvm::GlobalVariable>(Val: Data.getValue());
1075 if (!GV)
1076 continue;
1077 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1078 continue;
1079 GV->eraseFromParent();
1080 }
1081}
1082
1083std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1084 return OMPBuilder.createPlatformSpecificName(Parts);
1085}
1086
1087static llvm::Function *
1088emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1089 const Expr *CombinerInitializer, const VarDecl *In,
1090 const VarDecl *Out, bool IsCombiner) {
1091 // void .omp_combiner.(Ty *in, Ty *out);
1092 ASTContext &C = CGM.getContext();
1093 QualType PtrTy = C.getPointerType(T: Ty).withRestrict();
1094 auto *OmpOutParm = ImplicitParamDecl::Create(
1095 C, /*DC=*/nullptr, IdLoc: Out->getLocation(),
1096 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1097 auto *OmpInParm = ImplicitParamDecl::Create(
1098 C, /*DC=*/nullptr, IdLoc: In->getLocation(),
1099 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1100 FunctionArgList Args{OmpOutParm, OmpInParm};
1101 const CGFunctionInfo &FnInfo =
1102 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
1103 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1104 std::string Name = CGM.getOpenMPRuntime().getName(
1105 Parts: {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1106 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
1107 N: Name, M: &CGM.getModule());
1108 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
1109 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1110 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
1111 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
1112 Fn->removeFnAttr(Kind: llvm::Attribute::NoInline);
1113 Fn->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
1114 Fn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1115 }
1116 CodeGenFunction CGF(CGM);
1117 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1118 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1119 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc: In->getLocation(),
1120 StartLoc: Out->getLocation());
1121 CodeGenFunction::OMPPrivateScope Scope(CGF);
1122 Address AddrIn = CGF.GetAddrOfLocalVar(VD: OmpInParm);
1123 Scope.addPrivate(
1124 LocalVD: In, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrIn, PtrTy: PtrTy->castAs<PointerType>())
1125 .getAddress());
1126 Address AddrOut = CGF.GetAddrOfLocalVar(VD: OmpOutParm);
1127 Scope.addPrivate(
1128 LocalVD: Out, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrOut, PtrTy: PtrTy->castAs<PointerType>())
1129 .getAddress());
1130 (void)Scope.Privatize();
1131 if (!IsCombiner && Out->hasInit() &&
1132 !CGF.isTrivialInitializer(Init: Out->getInit())) {
1133 CGF.EmitAnyExprToMem(E: Out->getInit(), Location: CGF.GetAddrOfLocalVar(VD: Out),
1134 Quals: Out->getType().getQualifiers(),
1135 /*IsInitializer=*/true);
1136 }
1137 if (CombinerInitializer)
1138 CGF.EmitIgnoredExpr(E: CombinerInitializer);
1139 Scope.ForceCleanup();
1140 CGF.FinishFunction();
1141 return Fn;
1142}
1143
1144void CGOpenMPRuntime::emitUserDefinedReduction(
1145 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1146 if (UDRMap.count(Val: D) > 0)
1147 return;
1148 llvm::Function *Combiner = emitCombinerOrInitializer(
1149 CGM, Ty: D->getType(), CombinerInitializer: D->getCombiner(),
1150 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerIn())->getDecl()),
1151 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerOut())->getDecl()),
1152 /*IsCombiner=*/true);
1153 llvm::Function *Initializer = nullptr;
1154 if (const Expr *Init = D->getInitializer()) {
1155 Initializer = emitCombinerOrInitializer(
1156 CGM, Ty: D->getType(),
1157 CombinerInitializer: D->getInitializerKind() == OMPDeclareReductionInitKind::Call ? Init
1158 : nullptr,
1159 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitOrig())->getDecl()),
1160 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitPriv())->getDecl()),
1161 /*IsCombiner=*/false);
1162 }
1163 UDRMap.try_emplace(Key: D, Args&: Combiner, Args&: Initializer);
1164 if (CGF)
1165 FunctionUDRMap[CGF->CurFn].push_back(Elt: D);
1166}
1167
1168std::pair<llvm::Function *, llvm::Function *>
1169CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1170 auto I = UDRMap.find(Val: D);
1171 if (I != UDRMap.end())
1172 return I->second;
1173 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1174 return UDRMap.lookup(Val: D);
1175}
1176
1177namespace {
1178// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1179// Builder if one is present.
1180struct PushAndPopStackRAII {
1181 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1182 bool HasCancel, llvm::omp::Directive Kind)
1183 : OMPBuilder(OMPBuilder) {
1184 if (!OMPBuilder)
1185 return;
1186
1187 // The following callback is the crucial part of clangs cleanup process.
1188 //
1189 // NOTE:
1190 // Once the OpenMPIRBuilder is used to create parallel regions (and
1191 // similar), the cancellation destination (Dest below) is determined via
1192 // IP. That means if we have variables to finalize we split the block at IP,
1193 // use the new block (=BB) as destination to build a JumpDest (via
1194 // getJumpDestInCurrentScope(BB)) which then is fed to
1195 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1196 // to push & pop an FinalizationInfo object.
1197 // The FiniCB will still be needed but at the point where the
1198 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1199 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1200 assert(IP.getBlock()->end() == IP.getPoint() &&
1201 "Clang CG should cause non-terminated block!");
1202 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1203 CGF.Builder.restoreIP(IP);
1204 CodeGenFunction::JumpDest Dest =
1205 CGF.getOMPCancelDestination(Kind: OMPD_parallel);
1206 CGF.EmitBranchThroughCleanup(Dest);
1207 return llvm::Error::success();
1208 };
1209
1210 // TODO: Remove this once we emit parallel regions through the
1211 // OpenMPIRBuilder as it can do this setup internally.
1212 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1213 OMPBuilder->pushFinalizationCB(FI: std::move(FI));
1214 }
1215 ~PushAndPopStackRAII() {
1216 if (OMPBuilder)
1217 OMPBuilder->popFinalizationCB();
1218 }
1219 llvm::OpenMPIRBuilder *OMPBuilder;
1220};
1221} // namespace
1222
1223static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1224 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1225 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1226 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1227 assert(ThreadIDVar->getType()->isPointerType() &&
1228 "thread id variable must be of type kmp_int32 *");
1229 CodeGenFunction CGF(CGM, true);
1230 bool HasCancel = false;
1231 if (const auto *OPD = dyn_cast<OMPParallelDirective>(Val: &D))
1232 HasCancel = OPD->hasCancel();
1233 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(Val: &D))
1234 HasCancel = OPD->hasCancel();
1235 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &D))
1236 HasCancel = OPSD->hasCancel();
1237 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(Val: &D))
1238 HasCancel = OPFD->hasCancel();
1239 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(Val: &D))
1240 HasCancel = OPFD->hasCancel();
1241 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(Val: &D))
1242 HasCancel = OPFD->hasCancel();
1243 else if (const auto *OPFD =
1244 dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &D))
1245 HasCancel = OPFD->hasCancel();
1246 else if (const auto *OPFD =
1247 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &D))
1248 HasCancel = OPFD->hasCancel();
1249
1250 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1251 // parallel region to make cancellation barriers work properly.
1252 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1253 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1254 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1255 HasCancel, OutlinedHelperName);
1256 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1257 return CGF.GenerateOpenMPCapturedStmtFunction(S: *CS, D);
1258}
1259
1260std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const {
1261 std::string Suffix = getName(Parts: {"omp_outlined"});
1262 return (Name + Suffix).str();
1263}
1264
1265std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {
1266 return getOutlinedHelperName(Name: CGF.CurFn->getName());
1267}
1268
1269std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const {
1270 std::string Suffix = getName(Parts: {"omp", "reduction", "reduction_func"});
1271 return (Name + Suffix).str();
1272}
1273
1274llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1275 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1276 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1277 const RegionCodeGenTy &CodeGen) {
1278 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_parallel);
1279 return emitParallelOrTeamsOutlinedFunction(
1280 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1281 CodeGen);
1282}
1283
1284llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1285 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1286 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1287 const RegionCodeGenTy &CodeGen) {
1288 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_teams);
1289 return emitParallelOrTeamsOutlinedFunction(
1290 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1291 CodeGen);
1292}
1293
1294llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1295 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1296 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1297 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1298 bool Tied, unsigned &NumberOfParts) {
1299 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1300 PrePostActionTy &) {
1301 llvm::Value *ThreadID = getThreadID(CGF, Loc: D.getBeginLoc());
1302 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
1303 llvm::Value *TaskArgs[] = {
1304 UpLoc, ThreadID,
1305 CGF.EmitLoadOfPointerLValue(Ptr: CGF.GetAddrOfLocalVar(VD: TaskTVar),
1306 PtrTy: TaskTVar->getType()->castAs<PointerType>())
1307 .getPointer(CGF)};
1308 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1309 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
1310 args: TaskArgs);
1311 };
1312 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1313 UntiedCodeGen);
1314 CodeGen.setAction(Action);
1315 assert(!ThreadIDVar->getType()->isPointerType() &&
1316 "thread id variable must be of type kmp_int32 for tasks");
1317 const OpenMPDirectiveKind Region =
1318 isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) ? OMPD_taskloop
1319 : OMPD_task;
1320 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: Region);
1321 bool HasCancel = false;
1322 if (const auto *TD = dyn_cast<OMPTaskDirective>(Val: &D))
1323 HasCancel = TD->hasCancel();
1324 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(Val: &D))
1325 HasCancel = TD->hasCancel();
1326 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(Val: &D))
1327 HasCancel = TD->hasCancel();
1328 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(Val: &D))
1329 HasCancel = TD->hasCancel();
1330
1331 CodeGenFunction CGF(CGM, true);
1332 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1333 InnermostKind, HasCancel, Action);
1334 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1335 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(S: *CS);
1336 if (!Tied)
1337 NumberOfParts = Action.getNumberOfParts();
1338 return Res;
1339}
1340
1341void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1342 bool AtCurrentPoint) {
1343 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1344 assert(!Elem.ServiceInsertPt && "Insert point is set already.");
1345
1346 llvm::Value *Undef = llvm::UndefValue::get(T: CGF.Int32Ty);
1347 if (AtCurrentPoint) {
1348 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",
1349 CGF.Builder.GetInsertBlock());
1350 } else {
1351 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1352 Elem.ServiceInsertPt->insertAfter(InsertPos: CGF.AllocaInsertPt->getIterator());
1353 }
1354}
1355
1356void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1357 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1358 if (Elem.ServiceInsertPt) {
1359 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1360 Elem.ServiceInsertPt = nullptr;
1361 Ptr->eraseFromParent();
1362 }
1363}
1364
1365static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1366 SourceLocation Loc,
1367 SmallString<128> &Buffer) {
1368 llvm::raw_svector_ostream OS(Buffer);
1369 // Build debug location
1370 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1371 OS << ";";
1372 if (auto *DbgInfo = CGF.getDebugInfo())
1373 OS << DbgInfo->remapDIPath(PLoc.getFilename());
1374 else
1375 OS << PLoc.getFilename();
1376 OS << ";";
1377 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1378 OS << FD->getQualifiedNameAsString();
1379 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1380 return OS.str();
1381}
1382
1383llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1384 SourceLocation Loc,
1385 unsigned Flags, bool EmitLoc) {
1386 uint32_t SrcLocStrSize;
1387 llvm::Constant *SrcLocStr;
1388 if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() ==
1389 llvm::codegenoptions::NoDebugInfo) ||
1390 Loc.isInvalid()) {
1391 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1392 } else {
1393 std::string FunctionName;
1394 std::string FileName;
1395 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1396 FunctionName = FD->getQualifiedNameAsString();
1397 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1398 if (auto *DbgInfo = CGF.getDebugInfo())
1399 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
1400 else
1401 FileName = PLoc.getFilename();
1402 unsigned Line = PLoc.getLine();
1403 unsigned Column = PLoc.getColumn();
1404 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1405 Column, SrcLocStrSize);
1406 }
1407 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1408 return OMPBuilder.getOrCreateIdent(
1409 SrcLocStr, SrcLocStrSize, Flags: llvm::omp::IdentFlag(Flags), Reserve2Flags: Reserved2Flags);
1410}
1411
1412llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1413 SourceLocation Loc) {
1414 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1415 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1416 // the clang invariants used below might be broken.
1417 if (CGM.getLangOpts().OpenMPIRBuilder) {
1418 SmallString<128> Buffer;
1419 OMPBuilder.updateToLocation(Loc: CGF.Builder.saveIP());
1420 uint32_t SrcLocStrSize;
1421 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1422 LocStr: getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1423 return OMPBuilder.getOrCreateThreadID(
1424 Ident: OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1425 }
1426
1427 llvm::Value *ThreadID = nullptr;
1428 // Check whether we've already cached a load of the thread id in this
1429 // function.
1430 auto I = OpenMPLocThreadIDMap.find(Val: CGF.CurFn);
1431 if (I != OpenMPLocThreadIDMap.end()) {
1432 ThreadID = I->second.ThreadID;
1433 if (ThreadID != nullptr)
1434 return ThreadID;
1435 }
1436 // If exceptions are enabled, do not use parameter to avoid possible crash.
1437 if (auto *OMPRegionInfo =
1438 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
1439 if (OMPRegionInfo->getThreadIDVariable()) {
1440 // Check if this an outlined function with thread id passed as argument.
1441 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1442 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1443 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1444 !CGF.getLangOpts().CXXExceptions ||
1445 CGF.Builder.GetInsertBlock() == TopBlock ||
1446 !isa<llvm::Instruction>(Val: LVal.getPointer(CGF)) ||
1447 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1448 TopBlock ||
1449 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1450 CGF.Builder.GetInsertBlock()) {
1451 ThreadID = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
1452 // If value loaded in entry block, cache it and use it everywhere in
1453 // function.
1454 if (CGF.Builder.GetInsertBlock() == TopBlock)
1455 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;
1456 return ThreadID;
1457 }
1458 }
1459 }
1460
1461 // This is not an outlined function region - need to call __kmpc_int32
1462 // kmpc_global_thread_num(ident_t *loc).
1463 // Generate thread id value and cache this value for use across the
1464 // function.
1465 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1466 if (!Elem.ServiceInsertPt)
1467 setLocThreadIdInsertPt(CGF);
1468 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1469 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);
1470 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
1471 llvm::CallInst *Call = CGF.Builder.CreateCall(
1472 Callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
1473 FnID: OMPRTL___kmpc_global_thread_num),
1474 Args: emitUpdateLocation(CGF, Loc));
1475 Call->setCallingConv(CGF.getRuntimeCC());
1476 Elem.ThreadID = Call;
1477 return Call;
1478}
1479
1480void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1481 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1482 if (OpenMPLocThreadIDMap.count(Val: CGF.CurFn)) {
1483 clearLocThreadIdInsertPt(CGF);
1484 OpenMPLocThreadIDMap.erase(Val: CGF.CurFn);
1485 }
1486 if (auto I = FunctionUDRMap.find(Val: CGF.CurFn); I != FunctionUDRMap.end()) {
1487 for (const auto *D : I->second)
1488 UDRMap.erase(Val: D);
1489 FunctionUDRMap.erase(I);
1490 }
1491 if (auto I = FunctionUDMMap.find(Val: CGF.CurFn); I != FunctionUDMMap.end()) {
1492 for (const auto *D : I->second)
1493 UDMMap.erase(Val: D);
1494 FunctionUDMMap.erase(I);
1495 }
1496 LastprivateConditionalToTypes.erase(Val: CGF.CurFn);
1497 FunctionToUntiedTaskStackMap.erase(Val: CGF.CurFn);
1498}
1499
1500llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1501 return OMPBuilder.IdentPtr;
1502}
1503
1504static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1505convertDeviceClause(const VarDecl *VD) {
1506 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1507 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1508 if (!DevTy)
1509 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1510
1511 switch ((int)*DevTy) { // Avoid -Wcovered-switch-default
1512 case OMPDeclareTargetDeclAttr::DT_Host:
1513 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1514 break;
1515 case OMPDeclareTargetDeclAttr::DT_NoHost:
1516 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1517 break;
1518 case OMPDeclareTargetDeclAttr::DT_Any:
1519 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1520 break;
1521 default:
1522 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1523 break;
1524 }
1525}
1526
1527static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1528convertCaptureClause(const VarDecl *VD) {
1529 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1530 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1531 if (!MapType)
1532 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1533 switch ((int)*MapType) { // Avoid -Wcovered-switch-default
1534 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1535 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1536 break;
1537 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1538 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1539 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1540 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1541 break;
1542 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local:
1543 // MT_Local variables don't need offload entry (device-local).
1544 llvm_unreachable("MT_Local should not reach convertCaptureClause");
1545 break;
1546 default:
1547 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1548 break;
1549 }
1550}
1551
1552static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
1553 CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder,
1554 SourceLocation BeginLoc, llvm::StringRef ParentName = "") {
1555
1556 auto FileInfoCallBack = [&]() {
1557 SourceManager &SM = CGM.getContext().getSourceManager();
1558 PresumedLoc PLoc = SM.getPresumedLoc(Loc: BeginLoc);
1559
1560 if (!CGM.getFileSystem()->exists(Path: PLoc.getFilename()))
1561 PLoc = SM.getPresumedLoc(Loc: BeginLoc, /*UseLineDirectives=*/false);
1562
1563 return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine());
1564 };
1565
1566 return OMPBuilder.getTargetEntryUniqueInfo(CallBack: FileInfoCallBack,
1567 VFS&: *CGM.getFileSystem(), ParentName);
1568}
1569
1570ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1571 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
1572
1573 auto LinkageForVariable = [&VD, this]() {
1574 return CGM.getLLVMLinkageVarDefinition(VD);
1575 };
1576
1577 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1578
1579 llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(
1580 T: CGM.getContext().getPointerType(T: VD->getType()));
1581 llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar(
1582 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
1583 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
1584 IsExternallyVisible: VD->isExternallyVisible(),
1585 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
1586 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
1587 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
1588 TargetTriple: CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, GlobalInitializer: AddrOfGlobal,
1589 VariableLinkage: LinkageForVariable);
1590
1591 if (!addr)
1592 return ConstantAddress::invalid();
1593 return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(D: VD));
1594}
1595
1596llvm::Constant *
1597CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1598 assert(!CGM.getLangOpts().OpenMPUseTLS ||
1599 !CGM.getContext().getTargetInfo().isTLSSupported());
1600 // Lookup the entry, lazily creating it if necessary.
1601 std::string Suffix = getName(Parts: {"cache", ""});
1602 return OMPBuilder.getOrCreateInternalVariable(
1603 Ty: CGM.Int8PtrPtrTy, Name: Twine(CGM.getMangledName(GD: VD)).concat(Suffix).str());
1604}
1605
1606Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1607 const VarDecl *VD,
1608 Address VDAddr,
1609 SourceLocation Loc) {
1610 if (CGM.getLangOpts().OpenMPUseTLS &&
1611 CGM.getContext().getTargetInfo().isTLSSupported())
1612 return VDAddr;
1613
1614 llvm::Type *VarTy = VDAddr.getElementType();
1615 llvm::Value *Args[] = {
1616 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1617 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy),
1618 CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy)),
1619 getOrCreateThreadPrivateCache(VD)};
1620 return Address(
1621 CGF.EmitRuntimeCall(
1622 callee: OMPBuilder.getOrCreateRuntimeFunction(
1623 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1624 args: Args),
1625 CGF.Int8Ty, VDAddr.getAlignment());
1626}
1627
1628void CGOpenMPRuntime::emitThreadPrivateVarInit(
1629 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1630 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1631 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1632 // library.
1633 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1634 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1635 M&: CGM.getModule(), FnID: OMPRTL___kmpc_global_thread_num),
1636 args: OMPLoc);
1637 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1638 // to register constructor/destructor for variable.
1639 llvm::Value *Args[] = {
1640 OMPLoc,
1641 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy),
1642 Ctor, CopyCtor, Dtor};
1643 CGF.EmitRuntimeCall(
1644 callee: OMPBuilder.getOrCreateRuntimeFunction(
1645 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_register),
1646 args: Args);
1647}
1648
1649llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1650 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1651 bool PerformInit, CodeGenFunction *CGF) {
1652 if (CGM.getLangOpts().OpenMPUseTLS &&
1653 CGM.getContext().getTargetInfo().isTLSSupported())
1654 return nullptr;
1655
1656 VD = VD->getDefinition(C&: CGM.getContext());
1657 if (VD && ThreadPrivateWithDefinition.insert(key: CGM.getMangledName(GD: VD)).second) {
1658 QualType ASTTy = VD->getType();
1659
1660 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1661 const Expr *Init = VD->getAnyInitializer();
1662 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1663 // Generate function that re-emits the declaration's initializer into the
1664 // threadprivate copy of the variable VD
1665 CodeGenFunction CtorCGF(CGM);
1666 auto *Dst = ImplicitParamDecl::Create(
1667 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1668 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1669
1670 FunctionArgList Args{Dst};
1671 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1672 resultType: CGM.getContext().VoidPtrTy, args: Args);
1673 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1674 std::string Name = getName(Parts: {"__kmpc_global_ctor_", ""});
1675 llvm::Function *Fn =
1676 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1677 CtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidPtrTy, Fn, FnInfo: FI,
1678 Args, Loc, StartLoc: Loc);
1679 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1680 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1681 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1682 Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(T: ASTTy),
1683 VDAddr.getAlignment());
1684 CtorCGF.EmitAnyExprToMem(E: Init, Location: Arg, Quals: Init->getType().getQualifiers(),
1685 /*IsInitializer=*/true);
1686 ArgVal = CtorCGF.EmitLoadOfScalar(
1687 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1688 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1689 CtorCGF.Builder.CreateStore(Val: ArgVal, Addr: CtorCGF.ReturnValue);
1690 CtorCGF.FinishFunction();
1691 Ctor = Fn;
1692 }
1693 if (VD->getType().isDestructedType() != QualType::DK_none) {
1694 // Generate function that emits destructor call for the threadprivate copy
1695 // of the variable VD
1696 CodeGenFunction DtorCGF(CGM);
1697 auto *Dst = ImplicitParamDecl::Create(
1698 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1699 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1700
1701 FunctionArgList Args{Dst};
1702 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1703 resultType: CGM.getContext().VoidTy, args: Args);
1704 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1705 std::string Name = getName(Parts: {"__kmpc_global_dtor_", ""});
1706 llvm::Function *Fn =
1707 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1708 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: DtorCGF);
1709 DtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn, FnInfo: FI, Args,
1710 Loc, StartLoc: Loc);
1711 // Create a scope with an artificial location for the body of this function.
1712 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: DtorCGF);
1713 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1714 Addr: DtorCGF.GetAddrOfLocalVar(VD: Dst),
1715 /*Volatile=*/false, Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1716 DtorCGF.emitDestroy(
1717 addr: Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), type: ASTTy,
1718 destroyer: DtorCGF.getDestroyer(destructionKind: ASTTy.isDestructedType()),
1719 useEHCleanupForArray: DtorCGF.needsEHCleanup(kind: ASTTy.isDestructedType()));
1720 DtorCGF.FinishFunction();
1721 Dtor = Fn;
1722 }
1723 // Do not emit init function if it is not required.
1724 if (!Ctor && !Dtor)
1725 return nullptr;
1726
1727 // Copying constructor for the threadprivate variable.
1728 // Must be NULL - reserved by runtime, but currently it requires that this
1729 // parameter is always NULL. Otherwise it fires assertion.
1730 CopyCtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1731 if (Ctor == nullptr) {
1732 Ctor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1733 }
1734 if (Dtor == nullptr) {
1735 Dtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1736 }
1737 if (!CGF) {
1738 auto *InitFunctionTy =
1739 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg*/ false);
1740 std::string Name = getName(Parts: {"__omp_threadprivate_init_", ""});
1741 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1742 ty: InitFunctionTy, name: Name, FI: CGM.getTypes().arrangeNullaryFunction());
1743 CodeGenFunction InitCGF(CGM);
1744 FunctionArgList ArgList;
1745 InitCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: InitFunction,
1746 FnInfo: CGM.getTypes().arrangeNullaryFunction(), Args: ArgList,
1747 Loc, StartLoc: Loc);
1748 emitThreadPrivateVarInit(CGF&: InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1749 InitCGF.FinishFunction();
1750 return InitFunction;
1751 }
1752 emitThreadPrivateVarInit(CGF&: *CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1753 }
1754 return nullptr;
1755}
1756
1757void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD,
1758 llvm::GlobalValue *GV) {
1759 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1760 OMPDeclareTargetDeclAttr::getActiveAttr(VD: FD);
1761
1762 // We only need to handle active 'indirect' declare target functions.
1763 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1764 return;
1765
1766 // Get a mangled name to store the new device global in.
1767 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1768 CGM, OMPBuilder, BeginLoc: FD->getCanonicalDecl()->getBeginLoc(), ParentName: FD->getName());
1769 SmallString<128> Name;
1770 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1771
1772 // We need to generate a new global to hold the address of the indirectly
1773 // called device function. Doing this allows us to keep the visibility and
1774 // linkage of the associated function unchanged while allowing the runtime to
1775 // access its value.
1776 llvm::GlobalValue *Addr = GV;
1777 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1778 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1779 C&: CGM.getLLVMContext(),
1780 AddressSpace: CGM.getModule().getDataLayout().getProgramAddressSpace());
1781 Addr = new llvm::GlobalVariable(
1782 CGM.getModule(), FnPtrTy,
1783 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1784 nullptr, llvm::GlobalValue::NotThreadLocal,
1785 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1786 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1787 }
1788
1789 // Register the indirect Vtable:
1790 // This is similar to OMPTargetGlobalVarEntryIndirect, except that the
1791 // size field refers to the size of memory pointed to, not the size of
1792 // the pointer symbol itself (which is implicitly the size of a pointer).
1793 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1794 VarName: Name, Addr, VarSize: CGM.GetTargetTypeStoreSize(Ty: CGM.VoidPtrTy).getQuantity(),
1795 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1796 Linkage: llvm::GlobalValue::WeakODRLinkage);
1797}
1798
1799void CGOpenMPRuntime::registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1800 const VarDecl *VD) {
1801 // TODO: add logic to avoid duplicate vtable registrations per
1802 // translation unit; though for external linkage, this should no
1803 // longer be an issue - or at least we can avoid the issue by
1804 // checking for an existing offloading entry. But, perhaps the
1805 // better approach is to defer emission of the vtables and offload
1806 // entries until later (by tracking a list of items that need to be
1807 // emitted).
1808
1809 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1810
1811 // Generate a new externally visible global to point to the
1812 // internally visible vtable. Doing this allows us to keep the
1813 // visibility and linkage of the associated vtable unchanged while
1814 // allowing the runtime to access its value. The externally
1815 // visible global var needs to be emitted with a unique mangled
1816 // name that won't conflict with similarly named (internal)
1817 // vtables in other translation units.
1818
1819 // Register vtable with source location of dynamic object in map
1820 // clause.
1821 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1822 CGM, OMPBuilder, BeginLoc: VD->getCanonicalDecl()->getBeginLoc(),
1823 ParentName: VTable->getName());
1824
1825 llvm::GlobalVariable *Addr = VTable;
1826 SmallString<128> AddrName;
1827 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name&: AddrName, EntryInfo);
1828 AddrName.append(RHS: "addr");
1829
1830 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1831 Addr = new llvm::GlobalVariable(
1832 CGM.getModule(), VTable->getType(),
1833 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, VTable,
1834 AddrName,
1835 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1836 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1837 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1838 }
1839 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1840 VarName: AddrName, Addr: VTable,
1841 VarSize: CGM.getDataLayout().getTypeAllocSize(Ty: VTable->getInitializer()->getType()),
1842 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable,
1843 Linkage: llvm::GlobalValue::WeakODRLinkage);
1844}
1845
1846void CGOpenMPRuntime::emitAndRegisterVTable(CodeGenModule &CGM,
1847 CXXRecordDecl *CXXRecord,
1848 const VarDecl *VD) {
1849 // Register C++ VTable to OpenMP Offload Entry if it's a new
1850 // CXXRecordDecl.
1851 if (CXXRecord && CXXRecord->isDynamicClass() &&
1852 !CGM.getOpenMPRuntime().VTableDeclMap.contains(Val: CXXRecord)) {
1853 auto Res = CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(Key: CXXRecord, Args&: VD);
1854 if (Res.second) {
1855 CGM.EmitVTable(Class: CXXRecord);
1856 CodeGenVTables VTables = CGM.getVTables();
1857 llvm::GlobalVariable *VTablesAddr = VTables.GetAddrOfVTable(RD: CXXRecord);
1858 assert(VTablesAddr && "Expected non-null VTable address");
1859 // Must set VTables to weak since we're emitting them in multiple TUs now
1860 if (VTablesAddr->hasExternalLinkage())
1861 VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1862 CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTable: VTablesAddr, VD);
1863 // Emit VTable for all the fields containing dynamic CXXRecord
1864 for (const FieldDecl *Field : CXXRecord->fields()) {
1865 if (CXXRecordDecl *RecordDecl = Field->getType()->getAsCXXRecordDecl())
1866 emitAndRegisterVTable(CGM, CXXRecord: RecordDecl, VD);
1867 }
1868 // Emit VTable for all dynamic parent class
1869 for (CXXBaseSpecifier &Base : CXXRecord->bases()) {
1870 if (CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl())
1871 emitAndRegisterVTable(CGM, CXXRecord: BaseDecl, VD);
1872 }
1873 }
1874 }
1875}
1876
1877void CGOpenMPRuntime::registerVTable(const OMPExecutableDirective &D) {
1878 // Register VTable by scanning through the map clause of OpenMP target region.
1879 // Get CXXRecordDecl and VarDecl from Expr.
1880 auto GetVTableDecl = [](const Expr *E) {
1881 QualType VDTy = E->getType();
1882 CXXRecordDecl *CXXRecord = nullptr;
1883 if (const auto *RefType = VDTy->getAs<LValueReferenceType>())
1884 VDTy = RefType->getPointeeType();
1885 if (VDTy->isPointerType())
1886 CXXRecord = VDTy->getPointeeType()->getAsCXXRecordDecl();
1887 else
1888 CXXRecord = VDTy->getAsCXXRecordDecl();
1889
1890 const VarDecl *VD = nullptr;
1891 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1892 VD = cast<VarDecl>(Val: DRE->getDecl());
1893 } else if (auto *MRE = dyn_cast<MemberExpr>(Val: E)) {
1894 if (auto *BaseDRE = dyn_cast<DeclRefExpr>(Val: MRE->getBase())) {
1895 if (auto *BaseVD = dyn_cast<VarDecl>(Val: BaseDRE->getDecl()))
1896 VD = BaseVD;
1897 }
1898 }
1899 return std::pair<CXXRecordDecl *, const VarDecl *>(CXXRecord, VD);
1900 };
1901 // Collect VTable from OpenMP map clause.
1902 for (const auto *C : D.getClausesOfKind<OMPMapClause>()) {
1903 for (const auto *E : C->varlist()) {
1904 auto DeclPair = GetVTableDecl(E);
1905 // Ensure VD is not null
1906 if (DeclPair.second)
1907 emitAndRegisterVTable(CGM, CXXRecord: DeclPair.first, VD: DeclPair.second);
1908 }
1909 }
1910}
1911
1912Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1913 QualType VarType,
1914 StringRef Name) {
1915 std::string Suffix = getName(Parts: {"artificial", ""});
1916 llvm::Type *VarLVType = CGF.ConvertTypeForMem(T: VarType);
1917 llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable(
1918 Ty: VarLVType, Name: Twine(Name).concat(Suffix).str());
1919 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1920 CGM.getTarget().isTLSSupported()) {
1921 GAddr->setThreadLocal(/*Val=*/true);
1922 return Address(GAddr, GAddr->getValueType(),
1923 CGM.getContext().getTypeAlignInChars(T: VarType));
1924 }
1925 std::string CacheSuffix = getName(Parts: {"cache", ""});
1926 llvm::Value *Args[] = {
1927 emitUpdateLocation(CGF, Loc: SourceLocation()),
1928 getThreadID(CGF, Loc: SourceLocation()),
1929 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: GAddr, DestTy: CGM.VoidPtrTy),
1930 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: VarType), DestTy: CGM.SizeTy,
1931 /*isSigned=*/false),
1932 OMPBuilder.getOrCreateInternalVariable(
1933 Ty: CGM.VoidPtrPtrTy,
1934 Name: Twine(Name).concat(Suffix).concat(Suffix: CacheSuffix).str())};
1935 return Address(
1936 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1937 V: CGF.EmitRuntimeCall(
1938 callee: OMPBuilder.getOrCreateRuntimeFunction(
1939 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1940 args: Args),
1941 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
1942 VarLVType, CGM.getContext().getTypeAlignInChars(T: VarType));
1943}
1944
1945void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
1946 const RegionCodeGenTy &ThenGen,
1947 const RegionCodeGenTy &ElseGen) {
1948 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1949
1950 // If the condition constant folds and can be elided, try to avoid emitting
1951 // the condition and the dead arm of the if/else.
1952 bool CondConstant;
1953 if (CGF.ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant)) {
1954 if (CondConstant)
1955 ThenGen(CGF);
1956 else
1957 ElseGen(CGF);
1958 return;
1959 }
1960
1961 // Otherwise, the condition did not fold, or we couldn't elide it. Just
1962 // emit the conditional branch.
1963 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
1964 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock(name: "omp_if.else");
1965 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "omp_if.end");
1966 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, /*TrueCount=*/0);
1967
1968 // Emit the 'then' code.
1969 CGF.EmitBlock(BB: ThenBlock);
1970 ThenGen(CGF);
1971 CGF.EmitBranch(Block: ContBlock);
1972 // Emit the 'else' code if present.
1973 // There is no need to emit line number for unconditional branch.
1974 (void)ApplyDebugLocation::CreateEmpty(CGF);
1975 CGF.EmitBlock(BB: ElseBlock);
1976 ElseGen(CGF);
1977 // There is no need to emit line number for unconditional branch.
1978 (void)ApplyDebugLocation::CreateEmpty(CGF);
1979 CGF.EmitBranch(Block: ContBlock);
1980 // Emit the continuation block for code after the if.
1981 CGF.EmitBlock(BB: ContBlock, /*IsFinished=*/true);
1982}
1983
1984void CGOpenMPRuntime::emitParallelCall(
1985 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
1986 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
1987 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
1988 OpenMPSeverityClauseKind Severity, const Expr *Message) {
1989 if (!CGF.HaveInsertPoint())
1990 return;
1991 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1992 auto &M = CGM.getModule();
1993 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1994 this](CodeGenFunction &CGF, PrePostActionTy &) {
1995 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
1996 llvm::Value *Args[] = {
1997 RTLoc,
1998 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
1999 OutlinedFn};
2000 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2001 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
2002 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2003
2004 llvm::FunctionCallee RTLFn =
2005 OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_fork_call);
2006 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
2007 };
2008 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2009 this](CodeGenFunction &CGF, PrePostActionTy &) {
2010 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2011 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2012 // Build calls:
2013 // __kmpc_serialized_parallel(&Loc, GTid);
2014 llvm::Value *Args[] = {RTLoc, ThreadID};
2015 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2016 M, FnID: OMPRTL___kmpc_serialized_parallel),
2017 args: Args);
2018
2019 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2020 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2021 RawAddress ZeroAddrBound =
2022 CGF.CreateDefaultAlignTempAlloca(Ty: CGF.Int32Ty,
2023 /*Name=*/".bound.zero.addr");
2024 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(/*C*/ 0), Addr: ZeroAddrBound);
2025 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2026 // ThreadId for serialized parallels is 0.
2027 OutlinedFnArgs.push_back(Elt: ThreadIDAddr.emitRawPointer(CGF));
2028 OutlinedFnArgs.push_back(Elt: ZeroAddrBound.getPointer());
2029 OutlinedFnArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2030
2031 // Ensure we do not inline the function. This is trivially true for the ones
2032 // passed to __kmpc_fork_call but the ones called in serialized regions
2033 // could be inlined. This is not a perfect but it is closer to the invariant
2034 // we want, namely, every data environment starts with a new function.
2035 // TODO: We should pass the if condition to the runtime function and do the
2036 // handling there. Much cleaner code.
2037 OutlinedFn->removeFnAttr(Kind: llvm::Attribute::AlwaysInline);
2038 OutlinedFn->addFnAttr(Kind: llvm::Attribute::NoInline);
2039 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, Args: OutlinedFnArgs);
2040
2041 // __kmpc_end_serialized_parallel(&Loc, GTid);
2042 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2043 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2044 M, FnID: OMPRTL___kmpc_end_serialized_parallel),
2045 args: EndArgs);
2046 };
2047 if (IfCond) {
2048 emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
2049 } else {
2050 RegionCodeGenTy ThenRCG(ThenGen);
2051 ThenRCG(CGF);
2052 }
2053}
2054
2055// If we're inside an (outlined) parallel region, use the region info's
2056// thread-ID variable (it is passed in a first argument of the outlined function
2057// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2058// regular serial code region, get thread ID by calling kmp_int32
2059// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2060// return the address of that temp.
2061Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2062 SourceLocation Loc) {
2063 if (auto *OMPRegionInfo =
2064 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2065 if (OMPRegionInfo->getThreadIDVariable())
2066 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2067
2068 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2069 QualType Int32Ty =
2070 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2071 Address ThreadIDTemp =
2072 CGF.CreateMemTempWithoutCast(T: Int32Ty, /*Name*/ ".threadid_temp.");
2073 CGF.EmitStoreOfScalar(value: ThreadID,
2074 lvalue: CGF.MakeAddrLValue(Addr: ThreadIDTemp, T: Int32Ty));
2075
2076 return ThreadIDTemp;
2077}
2078
2079llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2080 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2081 std::string Name = getName(Parts: {Prefix, "var"});
2082 llvm::GlobalVariable *GV =
2083 OMPBuilder.getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
2084 CGM.setDSOLocal(GV);
2085 return GV;
2086}
2087
2088namespace {
2089/// Common pre(post)-action for different OpenMP constructs.
2090class CommonActionTy final : public PrePostActionTy {
2091 llvm::FunctionCallee EnterCallee;
2092 ArrayRef<llvm::Value *> EnterArgs;
2093 llvm::FunctionCallee ExitCallee;
2094 ArrayRef<llvm::Value *> ExitArgs;
2095 bool Conditional;
2096 llvm::BasicBlock *ContBlock = nullptr;
2097
2098public:
2099 CommonActionTy(llvm::FunctionCallee EnterCallee,
2100 ArrayRef<llvm::Value *> EnterArgs,
2101 llvm::FunctionCallee ExitCallee,
2102 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2103 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2104 ExitArgs(ExitArgs), Conditional(Conditional) {}
2105 void Enter(CodeGenFunction &CGF) override {
2106 llvm::Value *EnterRes = CGF.EmitRuntimeCall(callee: EnterCallee, args: EnterArgs);
2107 if (Conditional) {
2108 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(Arg: EnterRes);
2109 auto *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
2110 ContBlock = CGF.createBasicBlock(name: "omp_if.end");
2111 // Generate the branch (If-stmt)
2112 CGF.Builder.CreateCondBr(Cond: CallBool, True: ThenBlock, False: ContBlock);
2113 CGF.EmitBlock(BB: ThenBlock);
2114 }
2115 }
2116 void Done(CodeGenFunction &CGF) {
2117 // Emit the rest of blocks/branches
2118 CGF.EmitBranch(Block: ContBlock);
2119 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
2120 }
2121 void Exit(CodeGenFunction &CGF) override {
2122 CGF.EmitRuntimeCall(callee: ExitCallee, args: ExitArgs);
2123 }
2124};
2125} // anonymous namespace
2126
2127void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2128 StringRef CriticalName,
2129 const RegionCodeGenTy &CriticalOpGen,
2130 SourceLocation Loc, const Expr *Hint) {
2131 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2132 // CriticalOpGen();
2133 // __kmpc_end_critical(ident_t *, gtid, Lock);
2134 // Prepare arguments and build a call to __kmpc_critical
2135 if (!CGF.HaveInsertPoint())
2136 return;
2137 llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction(
2138 M&: CGM.getModule(),
2139 FnID: Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);
2140 llvm::Value *LockVar = getCriticalRegionLock(CriticalName);
2141 unsigned LockVarArgIdx = 2;
2142 if (cast<llvm::GlobalVariable>(Val: LockVar)->getAddressSpace() !=
2143 RuntimeFcn.getFunctionType()
2144 ->getParamType(i: LockVarArgIdx)
2145 ->getPointerAddressSpace())
2146 LockVar = CGF.Builder.CreateAddrSpaceCast(
2147 V: LockVar, DestTy: RuntimeFcn.getFunctionType()->getParamType(i: LockVarArgIdx));
2148 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2149 LockVar};
2150 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args),
2151 std::end(arr&: Args));
2152 if (Hint) {
2153 EnterArgs.push_back(Elt: CGF.Builder.CreateIntCast(
2154 V: CGF.EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, /*isSigned=*/false));
2155 }
2156 CommonActionTy Action(RuntimeFcn, EnterArgs,
2157 OMPBuilder.getOrCreateRuntimeFunction(
2158 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_critical),
2159 Args);
2160 CriticalOpGen.setAction(Action);
2161 emitInlinedDirective(CGF, InnermostKind: OMPD_critical, CodeGen: CriticalOpGen);
2162}
2163
2164void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2165 const RegionCodeGenTy &MasterOpGen,
2166 SourceLocation Loc) {
2167 if (!CGF.HaveInsertPoint())
2168 return;
2169 // if(__kmpc_master(ident_t *, gtid)) {
2170 // MasterOpGen();
2171 // __kmpc_end_master(ident_t *, gtid);
2172 // }
2173 // Prepare arguments and build a call to __kmpc_master
2174 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2175 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2176 M&: CGM.getModule(), FnID: OMPRTL___kmpc_master),
2177 Args,
2178 OMPBuilder.getOrCreateRuntimeFunction(
2179 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_master),
2180 Args,
2181 /*Conditional=*/true);
2182 MasterOpGen.setAction(Action);
2183 emitInlinedDirective(CGF, InnermostKind: OMPD_master, CodeGen: MasterOpGen);
2184 Action.Done(CGF);
2185}
2186
2187void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,
2188 const RegionCodeGenTy &MaskedOpGen,
2189 SourceLocation Loc, const Expr *Filter) {
2190 if (!CGF.HaveInsertPoint())
2191 return;
2192 // if(__kmpc_masked(ident_t *, gtid, filter)) {
2193 // MaskedOpGen();
2194 // __kmpc_end_masked(iden_t *, gtid);
2195 // }
2196 // Prepare arguments and build a call to __kmpc_masked
2197 llvm::Value *FilterVal = Filter
2198 ? CGF.EmitScalarExpr(E: Filter, IgnoreResultAssign: CGF.Int32Ty)
2199 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
2200 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2201 FilterVal};
2202 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2203 getThreadID(CGF, Loc)};
2204 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2205 M&: CGM.getModule(), FnID: OMPRTL___kmpc_masked),
2206 Args,
2207 OMPBuilder.getOrCreateRuntimeFunction(
2208 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_masked),
2209 ArgsEnd,
2210 /*Conditional=*/true);
2211 MaskedOpGen.setAction(Action);
2212 emitInlinedDirective(CGF, InnermostKind: OMPD_masked, CodeGen: MaskedOpGen);
2213 Action.Done(CGF);
2214}
2215
2216void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2217 SourceLocation Loc) {
2218 if (!CGF.HaveInsertPoint())
2219 return;
2220 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2221 OMPBuilder.createTaskyield(Loc: CGF.Builder);
2222 } else {
2223 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2224 llvm::Value *Args[] = {
2225 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2226 llvm::ConstantInt::get(Ty: CGM.IntTy, /*V=*/0, /*isSigned=*/IsSigned: true)};
2227 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2228 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_taskyield),
2229 args: Args);
2230 }
2231
2232 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2233 Region->emitUntiedSwitch(CGF);
2234}
2235
2236void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2237 const RegionCodeGenTy &TaskgroupOpGen,
2238 SourceLocation Loc) {
2239 if (!CGF.HaveInsertPoint())
2240 return;
2241 // __kmpc_taskgroup(ident_t *, gtid);
2242 // TaskgroupOpGen();
2243 // __kmpc_end_taskgroup(ident_t *, gtid);
2244 // Prepare arguments and build a call to __kmpc_taskgroup
2245 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2246 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2247 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskgroup),
2248 Args,
2249 OMPBuilder.getOrCreateRuntimeFunction(
2250 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_taskgroup),
2251 Args);
2252 TaskgroupOpGen.setAction(Action);
2253 emitInlinedDirective(CGF, InnermostKind: OMPD_taskgroup, CodeGen: TaskgroupOpGen);
2254}
2255
2256/// Given an array of pointers to variables, project the address of a
2257/// given variable.
2258static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2259 unsigned Index, const VarDecl *Var) {
2260 // Pull out the pointer to the variable.
2261 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Addr: Array, Index);
2262 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: PtrAddr);
2263
2264 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: Var->getType());
2265 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(D: Var));
2266}
2267
2268static llvm::Value *emitCopyprivateCopyFunction(
2269 CodeGenModule &CGM, llvm::Type *ArgsElemType,
2270 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2271 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2272 SourceLocation Loc) {
2273 ASTContext &C = CGM.getContext();
2274 // void copy_func(void *LHSArg, void *RHSArg);
2275
2276 auto *LHSArg =
2277 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2278 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2279 auto *RHSArg =
2280 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2281 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2282 FunctionArgList Args{LHSArg, RHSArg};
2283 const auto &CGFI =
2284 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
2285 std::string Name =
2286 CGM.getOpenMPRuntime().getName(Parts: {"omp", "copyprivate", "copy_func"});
2287 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
2288 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
2289 M: &CGM.getModule());
2290 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
2291 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
2292 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
2293 Fn->setDoesNotRecurse();
2294 CodeGenFunction CGF(CGM);
2295 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
2296 // Dest = (void*[n])(LHSArg);
2297 // Src = (void*[n])(RHSArg);
2298 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2299 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
2300 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2301 ArgsElemType, CGF.getPointerAlign());
2302 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2303 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
2304 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2305 ArgsElemType, CGF.getPointerAlign());
2306 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2307 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2308 // ...
2309 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2310 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2311 const auto *DestVar =
2312 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: DestExprs[I])->getDecl());
2313 Address DestAddr = emitAddrOfVarFromArray(CGF, Array: LHS, Index: I, Var: DestVar);
2314
2315 const auto *SrcVar =
2316 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: SrcExprs[I])->getDecl());
2317 Address SrcAddr = emitAddrOfVarFromArray(CGF, Array: RHS, Index: I, Var: SrcVar);
2318
2319 const auto *VD = cast<DeclRefExpr>(Val: CopyprivateVars[I])->getDecl();
2320 QualType Type = VD->getType();
2321 CGF.EmitOMPCopy(OriginalType: Type, DestAddr, SrcAddr, DestVD: DestVar, SrcVD: SrcVar, Copy: AssignmentOps[I]);
2322 }
2323 CGF.FinishFunction();
2324 return Fn;
2325}
2326
2327void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2328 const RegionCodeGenTy &SingleOpGen,
2329 SourceLocation Loc,
2330 ArrayRef<const Expr *> CopyprivateVars,
2331 ArrayRef<const Expr *> SrcExprs,
2332 ArrayRef<const Expr *> DstExprs,
2333 ArrayRef<const Expr *> AssignmentOps) {
2334 if (!CGF.HaveInsertPoint())
2335 return;
2336 assert(CopyprivateVars.size() == SrcExprs.size() &&
2337 CopyprivateVars.size() == DstExprs.size() &&
2338 CopyprivateVars.size() == AssignmentOps.size());
2339 ASTContext &C = CGM.getContext();
2340 // int32 did_it = 0;
2341 // if(__kmpc_single(ident_t *, gtid)) {
2342 // SingleOpGen();
2343 // __kmpc_end_single(ident_t *, gtid);
2344 // did_it = 1;
2345 // }
2346 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2347 // <copy_func>, did_it);
2348
2349 Address DidIt = Address::invalid();
2350 if (!CopyprivateVars.empty()) {
2351 // int32 did_it = 0;
2352 QualType KmpInt32Ty =
2353 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2354 DidIt = CGF.CreateMemTempWithoutCast(T: KmpInt32Ty, Name: ".omp.copyprivate.did_it");
2355 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 0), Addr: DidIt);
2356 }
2357 // Prepare arguments and build a call to __kmpc_single
2358 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2359 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2360 M&: CGM.getModule(), FnID: OMPRTL___kmpc_single),
2361 Args,
2362 OMPBuilder.getOrCreateRuntimeFunction(
2363 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_single),
2364 Args,
2365 /*Conditional=*/true);
2366 SingleOpGen.setAction(Action);
2367 emitInlinedDirective(CGF, InnermostKind: OMPD_single, CodeGen: SingleOpGen);
2368 if (DidIt.isValid()) {
2369 // did_it = 1;
2370 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 1), Addr: DidIt);
2371 }
2372 Action.Done(CGF);
2373 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2374 // <copy_func>, did_it);
2375 if (DidIt.isValid()) {
2376 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2377 QualType CopyprivateArrayTy = C.getConstantArrayType(
2378 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
2379 /*IndexTypeQuals=*/0);
2380 // Create a list of all private variables for copyprivate.
2381 Address CopyprivateList = CGF.CreateMemTempWithoutCast(
2382 T: CopyprivateArrayTy, Name: ".omp.copyprivate.cpr_list");
2383 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2384 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: CopyprivateList, Index: I);
2385 CGF.Builder.CreateStore(
2386 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2387 V: CGF.EmitLValue(E: CopyprivateVars[I]).getPointer(CGF),
2388 DestTy: CGF.VoidPtrTy),
2389 Addr: Elem);
2390 }
2391 // Build function that copies private values from single region to all other
2392 // threads in the corresponding parallel region.
2393 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2394 CGM, ArgsElemType: CGF.ConvertTypeForMem(T: CopyprivateArrayTy), CopyprivateVars,
2395 DestExprs: SrcExprs, SrcExprs: DstExprs, AssignmentOps, Loc);
2396 llvm::Value *BufSize = CGF.getTypeSize(Ty: CopyprivateArrayTy);
2397 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2398 Addr: CopyprivateList, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
2399 llvm::Value *DidItVal = CGF.Builder.CreateLoad(Addr: DidIt);
2400 llvm::Value *Args[] = {
2401 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2402 getThreadID(CGF, Loc), // i32 <gtid>
2403 BufSize, // size_t <buf_size>
2404 CL.emitRawPointer(CGF), // void *<copyprivate list>
2405 CpyFn, // void (*) (void *, void *) <copy_func>
2406 DidItVal // i32 did_it
2407 };
2408 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2409 M&: CGM.getModule(), FnID: OMPRTL___kmpc_copyprivate),
2410 args: Args);
2411 }
2412}
2413
2414void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2415 const RegionCodeGenTy &OrderedOpGen,
2416 SourceLocation Loc, bool IsThreads) {
2417 if (!CGF.HaveInsertPoint())
2418 return;
2419 // __kmpc_ordered(ident_t *, gtid);
2420 // OrderedOpGen();
2421 // __kmpc_end_ordered(ident_t *, gtid);
2422 // Prepare arguments and build a call to __kmpc_ordered
2423 if (IsThreads) {
2424 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2425 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2426 M&: CGM.getModule(), FnID: OMPRTL___kmpc_ordered),
2427 Args,
2428 OMPBuilder.getOrCreateRuntimeFunction(
2429 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_ordered),
2430 Args);
2431 OrderedOpGen.setAction(Action);
2432 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered, CodeGen: OrderedOpGen);
2433 return;
2434 }
2435 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered, CodeGen: OrderedOpGen);
2436}
2437
2438unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2439 unsigned Flags;
2440 if (Kind == OMPD_for)
2441 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2442 else if (Kind == OMPD_sections)
2443 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2444 else if (Kind == OMPD_single)
2445 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2446 else if (Kind == OMPD_barrier)
2447 Flags = OMP_IDENT_BARRIER_EXPL;
2448 else
2449 Flags = OMP_IDENT_BARRIER_IMPL;
2450 return Flags;
2451}
2452
2453void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2454 CodeGenFunction &CGF, const OMPLoopDirective &S,
2455 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2456 // Check if the loop directive is actually a doacross loop directive. In this
2457 // case choose static, 1 schedule.
2458 if (llvm::any_of(
2459 Range: S.getClausesOfKind<OMPOrderedClause>(),
2460 P: [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2461 ScheduleKind = OMPC_SCHEDULE_static;
2462 // Chunk size is 1 in this case.
2463 llvm::APInt ChunkSize(32, 1);
2464 ChunkExpr = IntegerLiteral::Create(
2465 C: CGF.getContext(), V: ChunkSize,
2466 type: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
2467 l: SourceLocation());
2468 }
2469}
2470
2471void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2472 OpenMPDirectiveKind Kind, bool EmitChecks,
2473 bool ForceSimpleCall) {
2474 // Check if we should use the OMPBuilder
2475 auto *OMPRegionInfo =
2476 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo);
2477 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2478 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2479 cantFail(ValOrErr: OMPBuilder.createBarrier(Loc: CGF.Builder, Kind, ForceSimpleCall,
2480 CheckCancelFlag: EmitChecks));
2481 CGF.Builder.restoreIP(IP: AfterIP);
2482 return;
2483 }
2484
2485 if (!CGF.HaveInsertPoint())
2486 return;
2487 // Build call __kmpc_cancel_barrier(loc, thread_id);
2488 // Build call __kmpc_barrier(loc, thread_id);
2489 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2490 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2491 // thread_id);
2492 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2493 getThreadID(CGF, Loc)};
2494 if (OMPRegionInfo) {
2495 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2496 llvm::Value *Result = CGF.EmitRuntimeCall(
2497 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
2498 FnID: OMPRTL___kmpc_cancel_barrier),
2499 args: Args);
2500 if (EmitChecks) {
2501 // if (__kmpc_cancel_barrier()) {
2502 // exit from construct;
2503 // }
2504 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
2505 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
2506 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
2507 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
2508 CGF.EmitBlock(BB: ExitBB);
2509 // exit from construct;
2510 CodeGenFunction::JumpDest CancelDestination =
2511 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
2512 CGF.EmitBranchThroughCleanup(Dest: CancelDestination);
2513 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
2514 }
2515 return;
2516 }
2517 }
2518 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2519 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
2520 args: Args);
2521}
2522
2523void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,
2524 Expr *ME, bool IsFatal) {
2525 llvm::Value *MVL = ME ? CGF.EmitScalarExpr(E: ME)
2526 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2527 // Build call void __kmpc_error(ident_t *loc, int severity, const char
2528 // *message)
2529 llvm::Value *Args[] = {
2530 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/EmitLoc: true),
2531 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: IsFatal ? 2 : 1),
2532 CGF.Builder.CreatePointerCast(V: MVL, DestTy: CGM.Int8PtrTy)};
2533 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2534 M&: CGM.getModule(), FnID: OMPRTL___kmpc_error),
2535 args: Args);
2536}
2537
2538/// Map the OpenMP loop schedule to the runtime enumeration.
2539static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2540 bool Chunked, bool Ordered) {
2541 switch (ScheduleKind) {
2542 case OMPC_SCHEDULE_static:
2543 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2544 : (Ordered ? OMP_ord_static : OMP_sch_static);
2545 case OMPC_SCHEDULE_dynamic:
2546 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2547 case OMPC_SCHEDULE_guided:
2548 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2549 case OMPC_SCHEDULE_runtime:
2550 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2551 case OMPC_SCHEDULE_auto:
2552 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2553 case OMPC_SCHEDULE_unknown:
2554 assert(!Chunked && "chunk was specified but schedule kind not known");
2555 return Ordered ? OMP_ord_static : OMP_sch_static;
2556 }
2557 llvm_unreachable("Unexpected runtime schedule");
2558}
2559
2560/// Map the OpenMP distribute schedule to the runtime enumeration.
2561static OpenMPSchedType
2562getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2563 // only static is allowed for dist_schedule
2564 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2565}
2566
2567bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2568 bool Chunked) const {
2569 OpenMPSchedType Schedule =
2570 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2571 return Schedule == OMP_sch_static;
2572}
2573
2574bool CGOpenMPRuntime::isStaticNonchunked(
2575 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2576 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2577 return Schedule == OMP_dist_sch_static;
2578}
2579
2580bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2581 bool Chunked) const {
2582 OpenMPSchedType Schedule =
2583 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2584 return Schedule == OMP_sch_static_chunked;
2585}
2586
2587bool CGOpenMPRuntime::isStaticChunked(
2588 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2589 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2590 return Schedule == OMP_dist_sch_static_chunked;
2591}
2592
2593bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2594 OpenMPSchedType Schedule =
2595 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2596 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
2597 return Schedule != OMP_sch_static;
2598}
2599
2600static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2601 OpenMPScheduleClauseModifier M1,
2602 OpenMPScheduleClauseModifier M2) {
2603 int Modifier = 0;
2604 switch (M1) {
2605 case OMPC_SCHEDULE_MODIFIER_monotonic:
2606 Modifier = OMP_sch_modifier_monotonic;
2607 break;
2608 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2609 Modifier = OMP_sch_modifier_nonmonotonic;
2610 break;
2611 case OMPC_SCHEDULE_MODIFIER_simd:
2612 if (Schedule == OMP_sch_static_chunked)
2613 Schedule = OMP_sch_static_balanced_chunked;
2614 break;
2615 case OMPC_SCHEDULE_MODIFIER_last:
2616 case OMPC_SCHEDULE_MODIFIER_unknown:
2617 break;
2618 }
2619 switch (M2) {
2620 case OMPC_SCHEDULE_MODIFIER_monotonic:
2621 Modifier = OMP_sch_modifier_monotonic;
2622 break;
2623 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2624 Modifier = OMP_sch_modifier_nonmonotonic;
2625 break;
2626 case OMPC_SCHEDULE_MODIFIER_simd:
2627 if (Schedule == OMP_sch_static_chunked)
2628 Schedule = OMP_sch_static_balanced_chunked;
2629 break;
2630 case OMPC_SCHEDULE_MODIFIER_last:
2631 case OMPC_SCHEDULE_MODIFIER_unknown:
2632 break;
2633 }
2634 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2635 // If the static schedule kind is specified or if the ordered clause is
2636 // specified, and if the nonmonotonic modifier is not specified, the effect is
2637 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2638 // modifier is specified, the effect is as if the nonmonotonic modifier is
2639 // specified.
2640 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2641 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2642 Schedule == OMP_sch_static_balanced_chunked ||
2643 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2644 Schedule == OMP_dist_sch_static_chunked ||
2645 Schedule == OMP_dist_sch_static ||
2646 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone))
2647 Modifier = OMP_sch_modifier_nonmonotonic;
2648 }
2649 return Schedule | Modifier;
2650}
2651
2652void CGOpenMPRuntime::emitForDispatchInit(
2653 CodeGenFunction &CGF, SourceLocation Loc,
2654 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2655 bool Ordered, const DispatchRTInput &DispatchValues) {
2656 if (!CGF.HaveInsertPoint())
2657 return;
2658 OpenMPSchedType Schedule = getRuntimeSchedule(
2659 ScheduleKind: ScheduleKind.Schedule, Chunked: DispatchValues.Chunk != nullptr, Ordered);
2660 assert(Ordered ||
2661 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2662 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2663 Schedule != OMP_sch_static_balanced_chunked));
2664 // Call __kmpc_dispatch_init(
2665 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2666 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2667 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2668
2669 // If the Chunk was not specified in the clause - use default value 1.
2670 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2671 : CGF.Builder.getIntN(N: IVSize, C: 1);
2672 llvm::Value *Args[] = {
2673 emitUpdateLocation(CGF, Loc),
2674 getThreadID(CGF, Loc),
2675 CGF.Builder.getInt32(C: addMonoNonMonoModifier(
2676 CGM, Schedule, M1: ScheduleKind.M1, M2: ScheduleKind.M2)), // Schedule type
2677 DispatchValues.LB, // Lower
2678 DispatchValues.UB, // Upper
2679 CGF.Builder.getIntN(N: IVSize, C: 1), // Stride
2680 Chunk // Chunk
2681 };
2682 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2683 args: Args);
2684}
2685
2686void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
2687 SourceLocation Loc) {
2688 if (!CGF.HaveInsertPoint())
2689 return;
2690 // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid);
2691 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2692 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchDeinitFunction(), args: Args);
2693}
2694
2695static void emitForStaticInitCall(
2696 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2697 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2698 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2699 const CGOpenMPRuntime::StaticRTInput &Values) {
2700 if (!CGF.HaveInsertPoint())
2701 return;
2702
2703 assert(!Values.Ordered);
2704 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2705 Schedule == OMP_sch_static_balanced_chunked ||
2706 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2707 Schedule == OMP_dist_sch_static ||
2708 Schedule == OMP_dist_sch_static_chunked ||
2709 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone);
2710
2711 // Call __kmpc_for_static_init(
2712 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2713 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2714 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2715 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2716 llvm::Value *Chunk = Values.Chunk;
2717 if (Chunk == nullptr) {
2718 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2719 Schedule == OMP_dist_sch_static) &&
2720 "expected static non-chunked schedule");
2721 // If the Chunk was not specified in the clause - use default value 1.
2722 Chunk = CGF.Builder.getIntN(N: Values.IVSize, C: 1);
2723 } else {
2724 assert((Schedule == OMP_sch_static_chunked ||
2725 Schedule == OMP_sch_static_balanced_chunked ||
2726 Schedule == OMP_ord_static_chunked ||
2727 Schedule == OMP_dist_sch_static_chunked ||
2728 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) &&
2729 "expected static chunked schedule");
2730 }
2731 llvm::Value *Args[] = {
2732 UpdateLocation,
2733 ThreadId,
2734 CGF.Builder.getInt32(C: addMonoNonMonoModifier(CGM&: CGF.CGM, Schedule, M1,
2735 M2)), // Schedule type
2736 Values.IL.emitRawPointer(CGF), // &isLastIter
2737 Values.LB.emitRawPointer(CGF), // &LB
2738 Values.UB.emitRawPointer(CGF), // &UB
2739 Values.ST.emitRawPointer(CGF), // &Stride
2740 CGF.Builder.getIntN(N: Values.IVSize, C: 1), // Incr
2741 Chunk // Chunk
2742 };
2743 CGF.EmitRuntimeCall(callee: ForStaticInitFunction, args: Args);
2744}
2745
2746void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2747 SourceLocation Loc,
2748 OpenMPDirectiveKind DKind,
2749 const OpenMPScheduleTy &ScheduleKind,
2750 const StaticRTInput &Values) {
2751 OpenMPSchedType ScheduleNum =
2752 ScheduleKind.UseFusedDistChunkSchedule
2753 ? OMP_dist_sch_static_chunked_sch_static_chunkone
2754 : getRuntimeSchedule(ScheduleKind: ScheduleKind.Schedule, Chunked: Values.Chunk != nullptr,
2755 Ordered: Values.Ordered);
2756 assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&
2757 "Expected loop-based or sections-based directive.");
2758 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2759 Flags: isOpenMPLoopDirective(DKind)
2760 ? OMP_IDENT_WORK_LOOP
2761 : OMP_IDENT_WORK_SECTIONS);
2762 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2763 llvm::FunctionCallee StaticInitFunction =
2764 OMPBuilder.createForStaticInitFunction(IVSize: Values.IVSize, IVSigned: Values.IVSigned,
2765 IsGPUDistribute: false);
2766 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2767 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2768 Schedule: ScheduleNum, M1: ScheduleKind.M1, M2: ScheduleKind.M2, Values);
2769}
2770
2771void CGOpenMPRuntime::emitDistributeStaticInit(
2772 CodeGenFunction &CGF, SourceLocation Loc,
2773 OpenMPDistScheduleClauseKind SchedKind,
2774 const CGOpenMPRuntime::StaticRTInput &Values) {
2775 OpenMPSchedType ScheduleNum =
2776 getRuntimeSchedule(ScheduleKind: SchedKind, Chunked: Values.Chunk != nullptr);
2777 llvm::Value *UpdatedLocation =
2778 emitUpdateLocation(CGF, Loc, Flags: OMP_IDENT_WORK_DISTRIBUTE);
2779 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2780 llvm::FunctionCallee StaticInitFunction;
2781 bool isGPUDistribute =
2782 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU();
2783 StaticInitFunction = OMPBuilder.createForStaticInitFunction(
2784 IVSize: Values.IVSize, IVSigned: Values.IVSigned, IsGPUDistribute: isGPUDistribute);
2785
2786 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2787 Schedule: ScheduleNum, M1: OMPC_SCHEDULE_MODIFIER_unknown,
2788 M2: OMPC_SCHEDULE_MODIFIER_unknown, Values);
2789}
2790
2791void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2792 SourceLocation Loc,
2793 OpenMPDirectiveKind DKind) {
2794 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2795 DKind == OMPD_sections) &&
2796 "Expected distribute, for, or sections directive kind");
2797 if (!CGF.HaveInsertPoint())
2798 return;
2799 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2800 llvm::Value *Args[] = {
2801 emitUpdateLocation(CGF, Loc,
2802 Flags: isOpenMPDistributeDirective(DKind) ||
2803 (DKind == OMPD_target_teams_loop)
2804 ? OMP_IDENT_WORK_DISTRIBUTE
2805 : isOpenMPLoopDirective(DKind)
2806 ? OMP_IDENT_WORK_LOOP
2807 : OMP_IDENT_WORK_SECTIONS),
2808 getThreadID(CGF, Loc)};
2809 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2810 if (isOpenMPDistributeDirective(DKind) &&
2811 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU())
2812 CGF.EmitRuntimeCall(
2813 callee: OMPBuilder.getOrCreateRuntimeFunction(
2814 M&: CGM.getModule(), FnID: OMPRTL___kmpc_distribute_static_fini),
2815 args: Args);
2816 else
2817 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2818 M&: CGM.getModule(), FnID: OMPRTL___kmpc_for_static_fini),
2819 args: Args);
2820}
2821
2822void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2823 SourceLocation Loc,
2824 unsigned IVSize,
2825 bool IVSigned) {
2826 if (!CGF.HaveInsertPoint())
2827 return;
2828 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2829 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2830 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2831 args: Args);
2832}
2833
2834llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2835 SourceLocation Loc, unsigned IVSize,
2836 bool IVSigned, Address IL,
2837 Address LB, Address UB,
2838 Address ST) {
2839 // Call __kmpc_dispatch_next(
2840 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2841 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2842 // kmp_int[32|64] *p_stride);
2843 llvm::Value *Args[] = {
2844 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2845 IL.emitRawPointer(CGF), // &isLastIter
2846 LB.emitRawPointer(CGF), // &Lower
2847 UB.emitRawPointer(CGF), // &Upper
2848 ST.emitRawPointer(CGF) // &Stride
2849 };
2850 llvm::Value *Call = CGF.EmitRuntimeCall(
2851 callee: OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), args: Args);
2852 return CGF.EmitScalarConversion(
2853 Src: Call, SrcTy: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/1),
2854 DstTy: CGF.getContext().BoolTy, Loc);
2855}
2856
2857llvm::Value *CGOpenMPRuntime::emitMessageClause(CodeGenFunction &CGF,
2858 const Expr *Message,
2859 SourceLocation Loc) {
2860 if (!Message)
2861 return llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2862 return CGF.EmitScalarExpr(E: Message);
2863}
2864
2865llvm::Value *
2866CGOpenMPRuntime::emitSeverityClause(OpenMPSeverityClauseKind Severity,
2867 SourceLocation Loc) {
2868 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is
2869 // as if sev-level is fatal."
2870 return llvm::ConstantInt::get(Ty: CGM.Int32Ty,
2871 V: Severity == OMPC_SEVERITY_warning ? 1 : 2);
2872}
2873
2874void CGOpenMPRuntime::emitNumThreadsClause(
2875 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
2876 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
2877 SourceLocation SeverityLoc, const Expr *Message,
2878 SourceLocation MessageLoc) {
2879 if (!CGF.HaveInsertPoint())
2880 return;
2881 llvm::SmallVector<llvm::Value *, 4> Args(
2882 {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2883 CGF.Builder.CreateIntCast(V: NumThreads, DestTy: CGF.Int32Ty, /*isSigned*/ true)});
2884 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2885 // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity,
2886 // messsage) if strict modifier is used.
2887 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2888 if (Modifier == OMPC_NUMTHREADS_strict) {
2889 FnID = OMPRTL___kmpc_push_num_threads_strict;
2890 Args.push_back(Elt: emitSeverityClause(Severity, Loc: SeverityLoc));
2891 Args.push_back(Elt: emitMessageClause(CGF, Message, Loc: MessageLoc));
2892 }
2893 CGF.EmitRuntimeCall(
2894 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args);
2895}
2896
2897void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2898 ProcBindKind ProcBind,
2899 SourceLocation Loc) {
2900 if (!CGF.HaveInsertPoint())
2901 return;
2902 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");
2903 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2904 llvm::Value *Args[] = {
2905 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2906 llvm::ConstantInt::get(Ty: CGM.IntTy, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
2907 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2908 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_proc_bind),
2909 args: Args);
2910}
2911
2912void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2913 SourceLocation Loc, llvm::AtomicOrdering AO) {
2914 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2915 OMPBuilder.createFlush(Loc: CGF.Builder);
2916 } else {
2917 if (!CGF.HaveInsertPoint())
2918 return;
2919 // Build call void __kmpc_flush(ident_t *loc)
2920 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2921 M&: CGM.getModule(), FnID: OMPRTL___kmpc_flush),
2922 args: emitUpdateLocation(CGF, Loc));
2923 }
2924}
2925
2926namespace {
2927/// Indexes of fields for type kmp_task_t.
2928enum KmpTaskTFields {
2929 /// List of shared variables.
2930 KmpTaskTShareds,
2931 /// Task routine.
2932 KmpTaskTRoutine,
2933 /// Partition id for the untied tasks.
2934 KmpTaskTPartId,
2935 /// Function with call of destructors for private variables.
2936 Data1,
2937 /// Task priority.
2938 Data2,
2939 /// (Taskloops only) Lower bound.
2940 KmpTaskTLowerBound,
2941 /// (Taskloops only) Upper bound.
2942 KmpTaskTUpperBound,
2943 /// (Taskloops only) Stride.
2944 KmpTaskTStride,
2945 /// (Taskloops only) Is last iteration flag.
2946 KmpTaskTLastIter,
2947 /// (Taskloops only) Reduction data.
2948 KmpTaskTReductions,
2949};
2950} // anonymous namespace
2951
2952void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
2953 // If we are in simd mode or there are no entries, we don't need to do
2954 // anything.
2955 if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty())
2956 return;
2957
2958 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2959 [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2960 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
2961 SourceLocation Loc;
2962 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2963 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
2964 E = CGM.getContext().getSourceManager().fileinfo_end();
2965 I != E; ++I) {
2966 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2967 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2968 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
2969 SourceFile: I->getFirst(), Line: EntryInfo.Line, Col: 1);
2970 break;
2971 }
2972 }
2973 }
2974 switch (Kind) {
2975 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2976 CGM.getDiags().Report(Loc,
2977 DiagID: diag::err_target_region_offloading_entry_incorrect)
2978 << EntryInfo.ParentName;
2979 } break;
2980 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2981 CGM.getDiags().Report(
2982 Loc, DiagID: diag::err_target_var_offloading_entry_incorrect_with_parent)
2983 << EntryInfo.ParentName;
2984 } break;
2985 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2986 CGM.getDiags().Report(DiagID: diag::err_target_var_offloading_entry_incorrect);
2987 } break;
2988 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: {
2989 unsigned DiagID = CGM.getDiags().getCustomDiagID(
2990 L: DiagnosticsEngine::Error, FormatString: "Offloading entry for indirect declare "
2991 "target variable is incorrect: the "
2992 "address is invalid.");
2993 CGM.getDiags().Report(DiagID);
2994 } break;
2995 }
2996 };
2997
2998 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
2999}
3000
3001void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3002 if (!KmpRoutineEntryPtrTy) {
3003 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3004 ASTContext &C = CGM.getContext();
3005 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3006 FunctionProtoType::ExtProtoInfo EPI;
3007 KmpRoutineEntryPtrQTy = C.getPointerType(
3008 T: C.getFunctionType(ResultTy: KmpInt32Ty, Args: KmpRoutineEntryTyArgs, EPI));
3009 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(T: KmpRoutineEntryPtrQTy);
3010 }
3011}
3012
3013namespace {
3014struct PrivateHelpersTy {
3015 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3016 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3017 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3018 PrivateElemInit(PrivateElemInit) {}
3019 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3020 const Expr *OriginalRef = nullptr;
3021 const VarDecl *Original = nullptr;
3022 const VarDecl *PrivateCopy = nullptr;
3023 const VarDecl *PrivateElemInit = nullptr;
3024 bool isLocalPrivate() const {
3025 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3026 }
3027};
3028typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3029} // anonymous namespace
3030
3031static bool isAllocatableDecl(const VarDecl *VD) {
3032 const VarDecl *CVD = VD->getCanonicalDecl();
3033 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3034 return false;
3035 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3036 // Use the default allocation.
3037 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3038 !AA->getAllocator());
3039}
3040
3041static RecordDecl *
3042createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3043 if (!Privates.empty()) {
3044 ASTContext &C = CGM.getContext();
3045 // Build struct .kmp_privates_t. {
3046 // /* private vars */
3047 // };
3048 RecordDecl *RD = C.buildImplicitRecord(Name: ".kmp_privates.t");
3049 RD->startDefinition();
3050 for (const auto &Pair : Privates) {
3051 const VarDecl *VD = Pair.second.Original;
3052 QualType Type = VD->getType().getNonReferenceType();
3053 // If the private variable is a local variable with lvalue ref type,
3054 // allocate the pointer instead of the pointee type.
3055 if (Pair.second.isLocalPrivate()) {
3056 if (VD->getType()->isLValueReferenceType())
3057 Type = C.getPointerType(T: Type);
3058 if (isAllocatableDecl(VD))
3059 Type = C.getPointerType(T: Type);
3060 }
3061 FieldDecl *FD = addFieldToRecordDecl(C, DC: RD, FieldTy: Type);
3062 if (VD->hasAttrs()) {
3063 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3064 E(VD->getAttrs().end());
3065 I != E; ++I)
3066 FD->addAttr(A: *I);
3067 }
3068 }
3069 RD->completeDefinition();
3070 return RD;
3071 }
3072 return nullptr;
3073}
3074
3075static RecordDecl *
3076createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3077 QualType KmpInt32Ty,
3078 QualType KmpRoutineEntryPointerQTy) {
3079 ASTContext &C = CGM.getContext();
3080 // Build struct kmp_task_t {
3081 // void * shareds;
3082 // kmp_routine_entry_t routine;
3083 // kmp_int32 part_id;
3084 // kmp_cmplrdata_t data1;
3085 // kmp_cmplrdata_t data2;
3086 // For taskloops additional fields:
3087 // kmp_uint64 lb;
3088 // kmp_uint64 ub;
3089 // kmp_int64 st;
3090 // kmp_int32 liter;
3091 // void * reductions;
3092 // };
3093 RecordDecl *UD = C.buildImplicitRecord(Name: "kmp_cmplrdata_t", TK: TagTypeKind::Union);
3094 UD->startDefinition();
3095 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpInt32Ty);
3096 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpRoutineEntryPointerQTy);
3097 UD->completeDefinition();
3098 CanQualType KmpCmplrdataTy = C.getCanonicalTagType(TD: UD);
3099 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t");
3100 RD->startDefinition();
3101 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3102 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpRoutineEntryPointerQTy);
3103 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3104 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3105 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3106 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3107 QualType KmpUInt64Ty =
3108 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3109 QualType KmpInt64Ty =
3110 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3111 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3112 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3113 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt64Ty);
3114 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3115 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3116 }
3117 RD->completeDefinition();
3118 return RD;
3119}
3120
3121static RecordDecl *
3122createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3123 ArrayRef<PrivateDataTy> Privates) {
3124 ASTContext &C = CGM.getContext();
3125 // Build struct kmp_task_t_with_privates {
3126 // kmp_task_t task_data;
3127 // .kmp_privates_t. privates;
3128 // };
3129 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t_with_privates");
3130 RD->startDefinition();
3131 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpTaskTQTy);
3132 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3133 addFieldToRecordDecl(C, DC: RD, FieldTy: C.getCanonicalTagType(TD: PrivateRD));
3134 RD->completeDefinition();
3135 return RD;
3136}
3137
3138/// Emit a proxy function which accepts kmp_task_t as the second
3139/// argument.
3140/// \code
3141/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3142/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3143/// For taskloops:
3144/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3145/// tt->reductions, tt->shareds);
3146/// return 0;
3147/// }
3148/// \endcode
3149static llvm::Function *
3150emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3151 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3152 QualType KmpTaskTWithPrivatesPtrQTy,
3153 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3154 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3155 llvm::Value *TaskPrivatesMap) {
3156 ASTContext &C = CGM.getContext();
3157 auto *GtidArg =
3158 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3159 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3160 auto *TaskTypeArg = ImplicitParamDecl::Create(
3161 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3162 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3163 FunctionArgList Args{GtidArg, TaskTypeArg};
3164 const auto &TaskEntryFnInfo =
3165 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3166 llvm::FunctionType *TaskEntryTy =
3167 CGM.getTypes().GetFunctionType(Info: TaskEntryFnInfo);
3168 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_entry", ""});
3169 auto *TaskEntry = llvm::Function::Create(
3170 Ty: TaskEntryTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3171 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskEntry, FI: TaskEntryFnInfo);
3172 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3173 TaskEntry->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3174 TaskEntry->setDoesNotRecurse();
3175 CodeGenFunction CGF(CGM);
3176 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: TaskEntry, FnInfo: TaskEntryFnInfo, Args,
3177 Loc, StartLoc: Loc);
3178
3179 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3180 // tt,
3181 // For taskloops:
3182 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3183 // tt->task_data.shareds);
3184 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3185 Addr: CGF.GetAddrOfLocalVar(VD: GtidArg), /*Volatile=*/false, Ty: KmpInt32Ty, Loc);
3186 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3187 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3188 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3189 const auto *KmpTaskTWithPrivatesQTyRD =
3190 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3191 LValue Base =
3192 CGF.EmitLValueForField(Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3193 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3194 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
3195 LValue PartIdLVal = CGF.EmitLValueForField(Base, Field: *PartIdFI);
3196 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3197
3198 auto SharedsFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds);
3199 LValue SharedsLVal = CGF.EmitLValueForField(Base, Field: *SharedsFI);
3200 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3201 V: CGF.EmitLoadOfScalar(lvalue: SharedsLVal, Loc),
3202 DestTy: CGF.ConvertTypeForMem(T: SharedsPtrTy));
3203
3204 auto PrivatesFI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin(), n: 1);
3205 llvm::Value *PrivatesParam;
3206 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3207 LValue PrivatesLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PrivatesFI);
3208 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3209 V: PrivatesLVal.getPointer(CGF), DestTy: CGF.VoidPtrTy);
3210 } else {
3211 PrivatesParam = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
3212 }
3213
3214 llvm::Value *CommonArgs[] = {
3215 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3216 CGF.Builder
3217 .CreatePointerBitCastOrAddrSpaceCast(Addr: TDBase.getAddress(),
3218 Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty)
3219 .emitRawPointer(CGF)};
3220 SmallVector<llvm::Value *, 16> CallArgs(std::begin(arr&: CommonArgs),
3221 std::end(arr&: CommonArgs));
3222 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3223 auto LBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound);
3224 LValue LBLVal = CGF.EmitLValueForField(Base, Field: *LBFI);
3225 llvm::Value *LBParam = CGF.EmitLoadOfScalar(lvalue: LBLVal, Loc);
3226 auto UBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound);
3227 LValue UBLVal = CGF.EmitLValueForField(Base, Field: *UBFI);
3228 llvm::Value *UBParam = CGF.EmitLoadOfScalar(lvalue: UBLVal, Loc);
3229 auto StFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride);
3230 LValue StLVal = CGF.EmitLValueForField(Base, Field: *StFI);
3231 llvm::Value *StParam = CGF.EmitLoadOfScalar(lvalue: StLVal, Loc);
3232 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3233 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3234 llvm::Value *LIParam = CGF.EmitLoadOfScalar(lvalue: LILVal, Loc);
3235 auto RFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions);
3236 LValue RLVal = CGF.EmitLValueForField(Base, Field: *RFI);
3237 llvm::Value *RParam = CGF.EmitLoadOfScalar(lvalue: RLVal, Loc);
3238 CallArgs.push_back(Elt: LBParam);
3239 CallArgs.push_back(Elt: UBParam);
3240 CallArgs.push_back(Elt: StParam);
3241 CallArgs.push_back(Elt: LIParam);
3242 CallArgs.push_back(Elt: RParam);
3243 }
3244 CallArgs.push_back(Elt: SharedsParam);
3245
3246 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskFunction,
3247 Args: CallArgs);
3248 CGF.EmitStoreThroughLValue(Src: RValue::get(V: CGF.Builder.getInt32(/*C=*/0)),
3249 Dst: CGF.MakeAddrLValue(Addr: CGF.ReturnValue, T: KmpInt32Ty));
3250 CGF.FinishFunction();
3251 return TaskEntry;
3252}
3253
3254static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3255 SourceLocation Loc,
3256 QualType KmpInt32Ty,
3257 QualType KmpTaskTWithPrivatesPtrQTy,
3258 QualType KmpTaskTWithPrivatesQTy) {
3259 ASTContext &C = CGM.getContext();
3260 auto *GtidArg =
3261 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3262 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3263 auto *TaskTypeArg = ImplicitParamDecl::Create(
3264 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3265 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3266 FunctionArgList Args{GtidArg, TaskTypeArg};
3267 const auto &DestructorFnInfo =
3268 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3269 llvm::FunctionType *DestructorFnTy =
3270 CGM.getTypes().GetFunctionType(Info: DestructorFnInfo);
3271 std::string Name =
3272 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_destructor", ""});
3273 auto *DestructorFn =
3274 llvm::Function::Create(Ty: DestructorFnTy, Linkage: llvm::GlobalValue::InternalLinkage,
3275 N: Name, M: &CGM.getModule());
3276 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: DestructorFn,
3277 FI: DestructorFnInfo);
3278 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3279 DestructorFn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3280 DestructorFn->setDoesNotRecurse();
3281 CodeGenFunction CGF(CGM);
3282 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: DestructorFn, FnInfo: DestructorFnInfo,
3283 Args, Loc, StartLoc: Loc);
3284
3285 LValue Base = CGF.EmitLoadOfPointerLValue(
3286 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3287 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3288 const auto *KmpTaskTWithPrivatesQTyRD =
3289 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3290 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3291 Base = CGF.EmitLValueForField(Base, Field: *FI);
3292 for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3293 if (QualType::DestructionKind DtorKind =
3294 Field->getType().isDestructedType()) {
3295 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3296 CGF.pushDestroy(dtorKind: DtorKind, addr: FieldLValue.getAddress(), type: Field->getType());
3297 }
3298 }
3299 CGF.FinishFunction();
3300 return DestructorFn;
3301}
3302
3303/// Emit a privates mapping function for correct handling of private and
3304/// firstprivate variables.
3305/// \code
3306/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3307/// **noalias priv1,..., <tyn> **noalias privn) {
3308/// *priv1 = &.privates.priv1;
3309/// ...;
3310/// *privn = &.privates.privn;
3311/// }
3312/// \endcode
3313static llvm::Value *
3314emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3315 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3316 ArrayRef<PrivateDataTy> Privates) {
3317 ASTContext &C = CGM.getContext();
3318 FunctionArgList Args;
3319 auto *TaskPrivatesArg = ImplicitParamDecl::Create(
3320 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3321 T: C.getPointerType(T: PrivatesQTy).withConst().withRestrict(),
3322 ParamKind: ImplicitParamKind::Other);
3323 Args.push_back(Elt: TaskPrivatesArg);
3324 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3325 unsigned Counter = 1;
3326 for (const Expr *E : Data.PrivateVars) {
3327 Args.push_back(Elt: ImplicitParamDecl::Create(
3328 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3329 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3330 .withConst()
3331 .withRestrict(),
3332 ParamKind: ImplicitParamKind::Other));
3333 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3334 PrivateVarsPos[VD] = Counter;
3335 ++Counter;
3336 }
3337 for (const Expr *E : Data.FirstprivateVars) {
3338 Args.push_back(Elt: ImplicitParamDecl::Create(
3339 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3340 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3341 .withConst()
3342 .withRestrict(),
3343 ParamKind: ImplicitParamKind::Other));
3344 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3345 PrivateVarsPos[VD] = Counter;
3346 ++Counter;
3347 }
3348 for (const Expr *E : Data.LastprivateVars) {
3349 Args.push_back(Elt: ImplicitParamDecl::Create(
3350 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3351 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3352 .withConst()
3353 .withRestrict(),
3354 ParamKind: ImplicitParamKind::Other));
3355 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3356 PrivateVarsPos[VD] = Counter;
3357 ++Counter;
3358 }
3359 for (const VarDecl *VD : Data.PrivateLocals) {
3360 QualType Ty = VD->getType().getNonReferenceType();
3361 if (VD->getType()->isLValueReferenceType())
3362 Ty = C.getPointerType(T: Ty);
3363 if (isAllocatableDecl(VD))
3364 Ty = C.getPointerType(T: Ty);
3365 Args.push_back(Elt: ImplicitParamDecl::Create(
3366 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3367 T: C.getPointerType(T: C.getPointerType(T: Ty)).withConst().withRestrict(),
3368 ParamKind: ImplicitParamKind::Other));
3369 PrivateVarsPos[VD] = Counter;
3370 ++Counter;
3371 }
3372 const auto &TaskPrivatesMapFnInfo =
3373 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3374 llvm::FunctionType *TaskPrivatesMapTy =
3375 CGM.getTypes().GetFunctionType(Info: TaskPrivatesMapFnInfo);
3376 std::string Name =
3377 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_privates_map", ""});
3378 auto *TaskPrivatesMap = llvm::Function::Create(
3379 Ty: TaskPrivatesMapTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
3380 M: &CGM.getModule());
3381 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskPrivatesMap,
3382 FI: TaskPrivatesMapFnInfo);
3383 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3384 TaskPrivatesMap->addFnAttr(Kind: "sample-profile-suffix-elision-policy",
3385 Val: "selected");
3386 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
3387 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::NoInline);
3388 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
3389 TaskPrivatesMap->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
3390 }
3391 CodeGenFunction CGF(CGM);
3392 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskPrivatesMap,
3393 FnInfo: TaskPrivatesMapFnInfo, Args, Loc, StartLoc: Loc);
3394
3395 // *privi = &.privates.privi;
3396 LValue Base = CGF.EmitLoadOfPointerLValue(
3397 Ptr: CGF.GetAddrOfLocalVar(VD: TaskPrivatesArg),
3398 PtrTy: TaskPrivatesArg->getType()->castAs<PointerType>());
3399 const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl();
3400 Counter = 0;
3401 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3402 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3403 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3404 LValue RefLVal =
3405 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD), T: VD->getType());
3406 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3407 Ptr: RefLVal.getAddress(), PtrTy: RefLVal.getType()->castAs<PointerType>());
3408 CGF.EmitStoreOfScalar(value: FieldLVal.getPointer(CGF), lvalue: RefLoadLVal);
3409 ++Counter;
3410 }
3411 CGF.FinishFunction();
3412 return TaskPrivatesMap;
3413}
3414
3415/// Emit initialization for private variables in task-based directives.
3416static void emitPrivatesInit(CodeGenFunction &CGF,
3417 const OMPExecutableDirective &D,
3418 Address KmpTaskSharedsPtr, LValue TDBase,
3419 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3420 QualType SharedsTy, QualType SharedsPtrTy,
3421 const OMPTaskDataTy &Data,
3422 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3423 ASTContext &C = CGF.getContext();
3424 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3425 LValue PrivatesBase = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
3426 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())
3427 ? OMPD_taskloop
3428 : OMPD_task;
3429 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: Kind);
3430 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3431 LValue SrcBase;
3432 bool IsTargetTask =
3433 isOpenMPTargetDataManagementDirective(DKind: D.getDirectiveKind()) ||
3434 isOpenMPTargetExecutionDirective(DKind: D.getDirectiveKind());
3435 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3436 // PointersArray, SizesArray, and MappersArray. The original variables for
3437 // these arrays are not captured and we get their addresses explicitly.
3438 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3439 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3440 SrcBase = CGF.MakeAddrLValue(
3441 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3442 Addr: KmpTaskSharedsPtr, Ty: CGF.ConvertTypeForMem(T: SharedsPtrTy),
3443 ElementTy: CGF.ConvertTypeForMem(T: SharedsTy)),
3444 T: SharedsTy);
3445 }
3446 FI = FI->getType()->castAsRecordDecl()->field_begin();
3447 for (const PrivateDataTy &Pair : Privates) {
3448 // Do not initialize private locals.
3449 if (Pair.second.isLocalPrivate()) {
3450 ++FI;
3451 continue;
3452 }
3453 const VarDecl *VD = Pair.second.PrivateCopy;
3454 const Expr *Init = VD->getAnyInitializer();
3455 if (Init && (!ForDup || (isa<CXXConstructExpr>(Val: Init) &&
3456 !CGF.isTrivialInitializer(Init)))) {
3457 LValue PrivateLValue = CGF.EmitLValueForField(Base: PrivatesBase, Field: *FI);
3458 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3459 const VarDecl *OriginalVD = Pair.second.Original;
3460 // Check if the variable is the target-based BasePointersArray,
3461 // PointersArray, SizesArray, or MappersArray.
3462 LValue SharedRefLValue;
3463 QualType Type = PrivateLValue.getType();
3464 const FieldDecl *SharedField = CapturesInfo.lookup(VD: OriginalVD);
3465 if (IsTargetTask && !SharedField) {
3466 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3467 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
3468 cast<CapturedDecl>(OriginalVD->getDeclContext())
3469 ->getNumParams() == 0 &&
3470 isa<TranslationUnitDecl>(
3471 cast<CapturedDecl>(OriginalVD->getDeclContext())
3472 ->getDeclContext()) &&
3473 "Expected artificial target data variable.");
3474 SharedRefLValue =
3475 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: OriginalVD), T: Type);
3476 } else if (ForDup) {
3477 SharedRefLValue = CGF.EmitLValueForField(Base: SrcBase, Field: SharedField);
3478 SharedRefLValue = CGF.MakeAddrLValue(
3479 Addr: SharedRefLValue.getAddress().withAlignment(
3480 NewAlignment: C.getDeclAlign(D: OriginalVD)),
3481 T: SharedRefLValue.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3482 TBAAInfo: SharedRefLValue.getTBAAInfo());
3483 } else if (CGF.LambdaCaptureFields.count(
3484 Val: Pair.second.Original->getCanonicalDecl()) > 0 ||
3485 isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl)) {
3486 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3487 } else {
3488 // Processing for implicitly captured variables.
3489 InlinedOpenMPRegionRAII Region(
3490 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3491 /*HasCancel=*/false, /*NoInheritance=*/true);
3492 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3493 }
3494 if (Type->isArrayType()) {
3495 // Initialize firstprivate array.
3496 if (!isa<CXXConstructExpr>(Val: Init) || CGF.isTrivialInitializer(Init)) {
3497 // Perform simple memcpy.
3498 CGF.EmitAggregateAssign(Dest: PrivateLValue, Src: SharedRefLValue, EltTy: Type);
3499 } else {
3500 // Initialize firstprivate array using element-by-element
3501 // initialization.
3502 CGF.EmitOMPAggregateAssign(
3503 DestAddr: PrivateLValue.getAddress(), SrcAddr: SharedRefLValue.getAddress(), OriginalType: Type,
3504 CopyGen: [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3505 Address SrcElement) {
3506 // Clean up any temporaries needed by the initialization.
3507 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3508 InitScope.addPrivate(LocalVD: Elem, Addr: SrcElement);
3509 (void)InitScope.Privatize();
3510 // Emit initialization for single element.
3511 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3512 CGF, &CapturesInfo);
3513 CGF.EmitAnyExprToMem(E: Init, Location: DestElement,
3514 Quals: Init->getType().getQualifiers(),
3515 /*IsInitializer=*/false);
3516 });
3517 }
3518 } else {
3519 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3520 InitScope.addPrivate(LocalVD: Elem, Addr: SharedRefLValue.getAddress());
3521 (void)InitScope.Privatize();
3522 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3523 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue,
3524 /*capturedByInit=*/false);
3525 }
3526 } else {
3527 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue, /*capturedByInit=*/false);
3528 }
3529 }
3530 ++FI;
3531 }
3532}
3533
3534/// Check if duplication function is required for taskloops.
3535static bool checkInitIsRequired(CodeGenFunction &CGF,
3536 ArrayRef<PrivateDataTy> Privates) {
3537 bool InitRequired = false;
3538 for (const PrivateDataTy &Pair : Privates) {
3539 if (Pair.second.isLocalPrivate())
3540 continue;
3541 const VarDecl *VD = Pair.second.PrivateCopy;
3542 const Expr *Init = VD->getAnyInitializer();
3543 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Val: Init) &&
3544 !CGF.isTrivialInitializer(Init));
3545 if (InitRequired)
3546 break;
3547 }
3548 return InitRequired;
3549}
3550
3551
3552/// Emit task_dup function (for initialization of
3553/// private/firstprivate/lastprivate vars and last_iter flag)
3554/// \code
3555/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3556/// lastpriv) {
3557/// // setup lastprivate flag
3558/// task_dst->last = lastpriv;
3559/// // could be constructor calls here...
3560/// }
3561/// \endcode
3562static llvm::Value *
3563emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3564 const OMPExecutableDirective &D,
3565 QualType KmpTaskTWithPrivatesPtrQTy,
3566 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3567 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3568 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3569 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3570 ASTContext &C = CGM.getContext();
3571 auto *DstArg = ImplicitParamDecl::Create(
3572 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3573 ParamKind: ImplicitParamKind::Other);
3574 auto *SrcArg = ImplicitParamDecl::Create(
3575 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3576 ParamKind: ImplicitParamKind::Other);
3577 auto *LastprivArg =
3578 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: C.IntTy,
3579 ParamKind: ImplicitParamKind::Other);
3580 FunctionArgList Args{DstArg, SrcArg, LastprivArg};
3581 const auto &TaskDupFnInfo =
3582 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3583 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(Info: TaskDupFnInfo);
3584 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_dup", ""});
3585 auto *TaskDup = llvm::Function::Create(
3586 Ty: TaskDupTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3587 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskDup, FI: TaskDupFnInfo);
3588 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3589 TaskDup->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3590 TaskDup->setDoesNotRecurse();
3591 CodeGenFunction CGF(CGM);
3592 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskDup, FnInfo: TaskDupFnInfo, Args, Loc,
3593 StartLoc: Loc);
3594
3595 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3596 Ptr: CGF.GetAddrOfLocalVar(VD: DstArg),
3597 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3598 // task_dst->liter = lastpriv;
3599 if (WithLastIter) {
3600 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3601 LValue Base = CGF.EmitLValueForField(
3602 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3603 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3604 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3605 Addr: CGF.GetAddrOfLocalVar(VD: LastprivArg), /*Volatile=*/false, Ty: C.IntTy, Loc);
3606 CGF.EmitStoreOfScalar(value: Lastpriv, lvalue: LILVal);
3607 }
3608
3609 // Emit initial values for private copies (if any).
3610 assert(!Privates.empty());
3611 Address KmpTaskSharedsPtr = Address::invalid();
3612 if (!Data.FirstprivateVars.empty()) {
3613 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3614 Ptr: CGF.GetAddrOfLocalVar(VD: SrcArg),
3615 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3616 LValue Base = CGF.EmitLValueForField(
3617 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3618 KmpTaskSharedsPtr = Address(
3619 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValueForField(
3620 Base, Field: *std::next(x: KmpTaskTQTyRD->field_begin(),
3621 n: KmpTaskTShareds)),
3622 Loc),
3623 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
3624 }
3625 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3626 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3627 CGF.FinishFunction();
3628 return TaskDup;
3629}
3630
3631/// Checks if destructor function is required to be generated.
3632/// \return true if cleanups are required, false otherwise.
3633static bool
3634checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3635 ArrayRef<PrivateDataTy> Privates) {
3636 for (const PrivateDataTy &P : Privates) {
3637 if (P.second.isLocalPrivate())
3638 continue;
3639 QualType Ty = P.second.Original->getType().getNonReferenceType();
3640 if (Ty.isDestructedType())
3641 return true;
3642 }
3643 return false;
3644}
3645
3646namespace {
3647/// Loop generator for OpenMP iterator expression.
3648class OMPIteratorGeneratorScope final
3649 : public CodeGenFunction::OMPPrivateScope {
3650 CodeGenFunction &CGF;
3651 const OMPIteratorExpr *E = nullptr;
3652 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3653 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3654 OMPIteratorGeneratorScope() = delete;
3655 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
3656
3657public:
3658 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3659 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3660 if (!E)
3661 return;
3662 SmallVector<llvm::Value *, 4> Uppers;
3663 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3664 Uppers.push_back(Elt: CGF.EmitScalarExpr(E: E->getHelper(I).Upper));
3665 const auto *VD = cast<VarDecl>(Val: E->getIteratorDecl(I));
3666 addPrivate(LocalVD: VD, Addr: CGF.CreateMemTemp(T: VD->getType(), Name: VD->getName()));
3667 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3668 addPrivate(
3669 LocalVD: HelperData.CounterVD,
3670 Addr: CGF.CreateMemTemp(T: HelperData.CounterVD->getType(), Name: "counter.addr"));
3671 }
3672 Privatize();
3673
3674 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3675 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3676 LValue CLVal =
3677 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: HelperData.CounterVD),
3678 T: HelperData.CounterVD->getType());
3679 // Counter = 0;
3680 CGF.EmitStoreOfScalar(
3681 value: llvm::ConstantInt::get(Ty: CLVal.getAddress().getElementType(), V: 0),
3682 lvalue: CLVal);
3683 CodeGenFunction::JumpDest &ContDest =
3684 ContDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.cont"));
3685 CodeGenFunction::JumpDest &ExitDest =
3686 ExitDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.exit"));
3687 // N = <number-of_iterations>;
3688 llvm::Value *N = Uppers[I];
3689 // cont:
3690 // if (Counter < N) goto body; else goto exit;
3691 CGF.EmitBlock(BB: ContDest.getBlock());
3692 auto *CVal =
3693 CGF.EmitLoadOfScalar(lvalue: CLVal, Loc: HelperData.CounterVD->getLocation());
3694 llvm::Value *Cmp =
3695 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3696 ? CGF.Builder.CreateICmpSLT(LHS: CVal, RHS: N)
3697 : CGF.Builder.CreateICmpULT(LHS: CVal, RHS: N);
3698 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "iter.body");
3699 CGF.Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitDest.getBlock());
3700 // body:
3701 CGF.EmitBlock(BB: BodyBB);
3702 // Iteri = Begini + Counter * Stepi;
3703 CGF.EmitIgnoredExpr(E: HelperData.Update);
3704 }
3705 }
3706 ~OMPIteratorGeneratorScope() {
3707 if (!E)
3708 return;
3709 for (unsigned I = E->numOfIterators(); I > 0; --I) {
3710 // Counter = Counter + 1;
3711 const OMPIteratorHelperData &HelperData = E->getHelper(I: I - 1);
3712 CGF.EmitIgnoredExpr(E: HelperData.CounterUpdate);
3713 // goto cont;
3714 CGF.EmitBranchThroughCleanup(Dest: ContDests[I - 1]);
3715 // exit:
3716 CGF.EmitBlock(BB: ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3717 }
3718 }
3719};
3720} // namespace
3721
3722static std::pair<llvm::Value *, llvm::Value *>
3723getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
3724 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(Val: E);
3725 llvm::Value *Addr;
3726 if (OASE) {
3727 const Expr *Base = OASE->getBase();
3728 Addr = CGF.EmitScalarExpr(E: Base);
3729 } else {
3730 Addr = CGF.EmitLValue(E).getPointer(CGF);
3731 }
3732 llvm::Value *SizeVal;
3733 QualType Ty = E->getType();
3734 if (OASE) {
3735 SizeVal = CGF.getTypeSize(Ty: OASE->getBase()->getType()->getPointeeType());
3736 for (const Expr *SE : OASE->getDimensions()) {
3737 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
3738 Sz = CGF.EmitScalarConversion(
3739 Src: Sz, SrcTy: SE->getType(), DstTy: CGF.getContext().getSizeType(), Loc: SE->getExprLoc());
3740 SizeVal = CGF.Builder.CreateNUWMul(LHS: SizeVal, RHS: Sz);
3741 }
3742 } else if (const auto *ASE =
3743 dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenImpCasts())) {
3744 LValue UpAddrLVal = CGF.EmitArraySectionExpr(E: ASE, /*IsLowerBound=*/false);
3745 Address UpAddrAddress = UpAddrLVal.getAddress();
3746 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3747 Ty: UpAddrAddress.getElementType(), Ptr: UpAddrAddress.emitRawPointer(CGF),
3748 /*Idx0=*/1);
3749 SizeVal = CGF.Builder.CreatePtrDiff(LHS: UpAddr, RHS: Addr, Name: "", /*IsNUW=*/true);
3750 } else {
3751 SizeVal = CGF.getTypeSize(Ty);
3752 }
3753 return std::make_pair(x&: Addr, y&: SizeVal);
3754}
3755
3756/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
3757static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
3758 QualType FlagsTy = C.getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/false);
3759 if (KmpTaskAffinityInfoTy.isNull()) {
3760 RecordDecl *KmpAffinityInfoRD =
3761 C.buildImplicitRecord(Name: "kmp_task_affinity_info_t");
3762 KmpAffinityInfoRD->startDefinition();
3763 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getIntPtrType());
3764 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getSizeType());
3765 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: FlagsTy);
3766 KmpAffinityInfoRD->completeDefinition();
3767 KmpTaskAffinityInfoTy = C.getCanonicalTagType(TD: KmpAffinityInfoRD);
3768 }
3769}
3770
3771CGOpenMPRuntime::TaskResultTy
3772CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3773 const OMPExecutableDirective &D,
3774 llvm::Function *TaskFunction, QualType SharedsTy,
3775 Address Shareds, const OMPTaskDataTy &Data) {
3776 ASTContext &C = CGM.getContext();
3777 llvm::SmallVector<PrivateDataTy, 4> Privates;
3778 // Aggregate privates and sort them by the alignment.
3779 const auto *I = Data.PrivateCopies.begin();
3780 for (const Expr *E : Data.PrivateVars) {
3781 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3782 Privates.emplace_back(
3783 Args: C.getDeclAlign(D: VD),
3784 Args: PrivateHelpersTy(E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3785 /*PrivateElemInit=*/nullptr));
3786 ++I;
3787 }
3788 I = Data.FirstprivateCopies.begin();
3789 const auto *IElemInitRef = Data.FirstprivateInits.begin();
3790 for (const Expr *E : Data.FirstprivateVars) {
3791 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3792 Privates.emplace_back(
3793 Args: C.getDeclAlign(D: VD),
3794 Args: PrivateHelpersTy(
3795 E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3796 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IElemInitRef)->getDecl())));
3797 ++I;
3798 ++IElemInitRef;
3799 }
3800 I = Data.LastprivateCopies.begin();
3801 for (const Expr *E : Data.LastprivateVars) {
3802 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3803 Privates.emplace_back(
3804 Args: C.getDeclAlign(D: VD),
3805 Args: PrivateHelpersTy(E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3806 /*PrivateElemInit=*/nullptr));
3807 ++I;
3808 }
3809 for (const VarDecl *VD : Data.PrivateLocals) {
3810 if (isAllocatableDecl(VD))
3811 Privates.emplace_back(Args: CGM.getPointerAlign(), Args: PrivateHelpersTy(VD));
3812 else
3813 Privates.emplace_back(Args: C.getDeclAlign(D: VD), Args: PrivateHelpersTy(VD));
3814 }
3815 llvm::stable_sort(Range&: Privates,
3816 C: [](const PrivateDataTy &L, const PrivateDataTy &R) {
3817 return L.first > R.first;
3818 });
3819 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3820 // Build type kmp_routine_entry_t (if not built yet).
3821 emitKmpRoutineEntryT(KmpInt32Ty);
3822 // Build type kmp_task_t (if not built yet).
3823 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())) {
3824 if (SavedKmpTaskloopTQTy.isNull()) {
3825 SavedKmpTaskloopTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
3826 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
3827 }
3828 KmpTaskTQTy = SavedKmpTaskloopTQTy;
3829 } else {
3830 assert((D.getDirectiveKind() == OMPD_task ||
3831 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
3832 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
3833 "Expected taskloop, task or target directive");
3834 if (SavedKmpTaskTQTy.isNull()) {
3835 SavedKmpTaskTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
3836 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
3837 }
3838 KmpTaskTQTy = SavedKmpTaskTQTy;
3839 }
3840 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3841 // Build particular struct kmp_task_t for the given task.
3842 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3843 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
3844 CanQualType KmpTaskTWithPrivatesQTy =
3845 C.getCanonicalTagType(TD: KmpTaskTWithPrivatesQTyRD);
3846 QualType KmpTaskTWithPrivatesPtrQTy =
3847 C.getPointerType(T: KmpTaskTWithPrivatesQTy);
3848 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(AddrSpace: 0);
3849 llvm::Value *KmpTaskTWithPrivatesTySize =
3850 CGF.getTypeSize(Ty: KmpTaskTWithPrivatesQTy);
3851 QualType SharedsPtrTy = C.getPointerType(T: SharedsTy);
3852
3853 // Emit initial values for private copies (if any).
3854 llvm::Value *TaskPrivatesMap = nullptr;
3855 llvm::Type *TaskPrivatesMapTy =
3856 std::next(x: TaskFunction->arg_begin(), n: 3)->getType();
3857 if (!Privates.empty()) {
3858 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3859 TaskPrivatesMap =
3860 emitTaskPrivateMappingFunction(CGM, Loc, Data, PrivatesQTy: FI->getType(), Privates);
3861 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3862 V: TaskPrivatesMap, DestTy: TaskPrivatesMapTy);
3863 } else {
3864 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3865 T: cast<llvm::PointerType>(Val: TaskPrivatesMapTy));
3866 }
3867 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
3868 // kmp_task_t *tt);
3869 llvm::Function *TaskEntry = emitProxyTaskFunction(
3870 CGM, Loc, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3871 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3872 TaskPrivatesMap);
3873
3874 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
3875 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
3876 // kmp_routine_entry_t *task_entry);
3877 // Task flags. Format is taken from
3878 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
3879 // description of kmp_tasking_flags struct.
3880 enum {
3881 TiedFlag = 0x1,
3882 FinalFlag = 0x2,
3883 DestructorsFlag = 0x8,
3884 PriorityFlag = 0x20,
3885 DetachableFlag = 0x40,
3886 FreeAgentFlag = 0x80,
3887 TransparentFlag = 0x100,
3888 };
3889 unsigned Flags = Data.Tied ? TiedFlag : 0;
3890 bool NeedsCleanup = false;
3891 if (!Privates.empty()) {
3892 NeedsCleanup =
3893 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
3894 if (NeedsCleanup)
3895 Flags = Flags | DestructorsFlag;
3896 }
3897 if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {
3898 OpenMPThreadsetKind Kind = Clause->getThreadsetKind();
3899 if (Kind == OMPC_THREADSET_omp_pool)
3900 Flags = Flags | FreeAgentFlag;
3901 }
3902 if (D.getSingleClause<OMPTransparentClause>())
3903 Flags |= TransparentFlag;
3904
3905 if (Data.Priority.getInt())
3906 Flags = Flags | PriorityFlag;
3907 if (D.hasClausesOfKind<OMPDetachClause>())
3908 Flags = Flags | DetachableFlag;
3909 llvm::Value *TaskFlags =
3910 Data.Final.getPointer()
3911 ? CGF.Builder.CreateSelect(C: Data.Final.getPointer(),
3912 True: CGF.Builder.getInt32(C: FinalFlag),
3913 False: CGF.Builder.getInt32(/*C=*/0))
3914 : CGF.Builder.getInt32(C: Data.Final.getInt() ? FinalFlag : 0);
3915 TaskFlags = CGF.Builder.CreateOr(LHS: TaskFlags, RHS: CGF.Builder.getInt32(C: Flags));
3916 llvm::Value *SharedsSize = CGM.getSize(numChars: C.getTypeSizeInChars(T: SharedsTy));
3917 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
3918 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3919 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3920 V: TaskEntry, DestTy: KmpRoutineEntryPtrTy)};
3921 llvm::Value *NewTask;
3922 if (D.hasClausesOfKind<OMPNowaitClause>()) {
3923 // Check if we have any device clause associated with the directive.
3924 const Expr *Device = nullptr;
3925 if (auto *C = D.getSingleClause<OMPDeviceClause>())
3926 Device = C->getDevice();
3927 // Emit device ID if any otherwise use default value.
3928 llvm::Value *DeviceID;
3929 if (Device)
3930 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
3931 DestTy: CGF.Int64Ty, /*isSigned=*/true);
3932 else
3933 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
3934 AllocArgs.push_back(Elt: DeviceID);
3935 NewTask = CGF.EmitRuntimeCall(
3936 callee: OMPBuilder.getOrCreateRuntimeFunction(
3937 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_target_task_alloc),
3938 args: AllocArgs);
3939 } else {
3940 NewTask =
3941 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
3942 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_alloc),
3943 args: AllocArgs);
3944 }
3945 // Emit detach clause initialization.
3946 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
3947 // task_descriptor);
3948 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
3949 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
3950 LValue EvtLVal = CGF.EmitLValue(E: Evt);
3951
3952 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
3953 // int gtid, kmp_task_t *task);
3954 llvm::Value *Loc = emitUpdateLocation(CGF, Loc: DC->getBeginLoc());
3955 llvm::Value *Tid = getThreadID(CGF, Loc: DC->getBeginLoc());
3956 Tid = CGF.Builder.CreateIntCast(V: Tid, DestTy: CGF.IntTy, /*isSigned=*/false);
3957 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
3958 callee: OMPBuilder.getOrCreateRuntimeFunction(
3959 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_allow_completion_event),
3960 args: {Loc, Tid, NewTask});
3961 EvtVal = CGF.EmitScalarConversion(Src: EvtVal, SrcTy: C.VoidPtrTy, DstTy: Evt->getType(),
3962 Loc: Evt->getExprLoc());
3963 CGF.EmitStoreOfScalar(value: EvtVal, lvalue: EvtLVal);
3964 }
3965 // Process affinity clauses.
3966 if (D.hasClausesOfKind<OMPAffinityClause>()) {
3967 // Process list of affinity data.
3968 ASTContext &C = CGM.getContext();
3969 Address AffinitiesArray = Address::invalid();
3970 // Calculate number of elements to form the array of affinity data.
3971 llvm::Value *NumOfElements = nullptr;
3972 unsigned NumAffinities = 0;
3973 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
3974 if (const Expr *Modifier = C->getModifier()) {
3975 const auto *IE = cast<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts());
3976 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3977 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
3978 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
3979 NumOfElements =
3980 NumOfElements ? CGF.Builder.CreateNUWMul(LHS: NumOfElements, RHS: Sz) : Sz;
3981 }
3982 } else {
3983 NumAffinities += C->varlist_size();
3984 }
3985 }
3986 getKmpAffinityType(C&: CGM.getContext(), KmpTaskAffinityInfoTy);
3987 // Fields ids in kmp_task_affinity_info record.
3988 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3989
3990 QualType KmpTaskAffinityInfoArrayTy;
3991 if (NumOfElements) {
3992 NumOfElements = CGF.Builder.CreateNUWAdd(
3993 LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumAffinities), RHS: NumOfElements);
3994 auto *OVE = new (C) OpaqueValueExpr(
3995 Loc,
3996 C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.getSizeType()), /*Signed=*/0),
3997 VK_PRValue);
3998 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
3999 RValue::get(V: NumOfElements));
4000 KmpTaskAffinityInfoArrayTy = C.getVariableArrayType(
4001 EltTy: KmpTaskAffinityInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4002 /*IndexTypeQuals=*/0);
4003 // Properly emit variable-sized array.
4004 auto *PD = ImplicitParamDecl::Create(C, T: KmpTaskAffinityInfoArrayTy,
4005 ParamKind: ImplicitParamKind::Other);
4006 CGF.EmitVarDecl(D: *PD);
4007 AffinitiesArray = CGF.GetAddrOfLocalVar(VD: PD);
4008 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4009 /*isSigned=*/false);
4010 } else {
4011 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4012 EltTy: KmpTaskAffinityInfoTy,
4013 ArySize: llvm::APInt(C.getTypeSize(T: C.getSizeType()), NumAffinities), SizeExpr: nullptr,
4014 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4015 AffinitiesArray = CGF.CreateMemTempWithoutCast(T: KmpTaskAffinityInfoArrayTy,
4016 Name: ".affs.arr.addr");
4017 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(Addr: AffinitiesArray, Index: 0);
4018 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumAffinities,
4019 /*isSigned=*/IsSigned: false);
4020 }
4021
4022 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4023 // Fill array by elements without iterators.
4024 unsigned Pos = 0;
4025 bool HasIterator = false;
4026 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4027 if (C->getModifier()) {
4028 HasIterator = true;
4029 continue;
4030 }
4031 for (const Expr *E : C->varlist()) {
4032 llvm::Value *Addr;
4033 llvm::Value *Size;
4034 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4035 LValue Base =
4036 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateConstGEP(Addr: AffinitiesArray, Index: Pos),
4037 T: KmpTaskAffinityInfoTy);
4038 // affs[i].base_addr = &<Affinities[i].second>;
4039 LValue BaseAddrLVal = CGF.EmitLValueForField(
4040 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4041 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4042 lvalue: BaseAddrLVal);
4043 // affs[i].len = sizeof(<Affinities[i].second>);
4044 LValue LenLVal = CGF.EmitLValueForField(
4045 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4046 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4047 ++Pos;
4048 }
4049 }
4050 LValue PosLVal;
4051 if (HasIterator) {
4052 PosLVal = CGF.MakeAddrLValue(
4053 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "affs.counter.addr"),
4054 T: C.getSizeType());
4055 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4056 }
4057 // Process elements with iterators.
4058 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4059 const Expr *Modifier = C->getModifier();
4060 if (!Modifier)
4061 continue;
4062 OMPIteratorGeneratorScope IteratorScope(
4063 CGF, cast_or_null<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts()));
4064 for (const Expr *E : C->varlist()) {
4065 llvm::Value *Addr;
4066 llvm::Value *Size;
4067 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4068 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4069 LValue Base =
4070 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateGEP(CGF, Addr: AffinitiesArray, Index: Idx),
4071 T: KmpTaskAffinityInfoTy);
4072 // affs[i].base_addr = &<Affinities[i].second>;
4073 LValue BaseAddrLVal = CGF.EmitLValueForField(
4074 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4075 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4076 lvalue: BaseAddrLVal);
4077 // affs[i].len = sizeof(<Affinities[i].second>);
4078 LValue LenLVal = CGF.EmitLValueForField(
4079 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4080 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4081 Idx = CGF.Builder.CreateNUWAdd(
4082 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4083 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4084 }
4085 }
4086 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4087 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4088 // naffins, kmp_task_affinity_info_t *affin_list);
4089 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4090 llvm::Value *GTid = getThreadID(CGF, Loc);
4091 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4092 V: AffinitiesArray.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy);
4093 // FIXME: Emit the function and ignore its result for now unless the
4094 // runtime function is properly implemented.
4095 (void)CGF.EmitRuntimeCall(
4096 callee: OMPBuilder.getOrCreateRuntimeFunction(
4097 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_reg_task_with_affinity),
4098 args: {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4099 }
4100 llvm::Value *NewTaskNewTaskTTy =
4101 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4102 V: NewTask, DestTy: KmpTaskTWithPrivatesPtrTy);
4103 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(V: NewTaskNewTaskTTy,
4104 T: KmpTaskTWithPrivatesQTy);
4105 LValue TDBase =
4106 CGF.EmitLValueForField(Base, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
4107 // Fill the data in the resulting kmp_task_t record.
4108 // Copy shareds if there are any.
4109 Address KmpTaskSharedsPtr = Address::invalid();
4110 if (!SharedsTy->castAsRecordDecl()->field_empty()) {
4111 KmpTaskSharedsPtr = Address(
4112 CGF.EmitLoadOfScalar(
4113 lvalue: CGF.EmitLValueForField(
4114 Base: TDBase,
4115 Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds)),
4116 Loc),
4117 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
4118 LValue Dest = CGF.MakeAddrLValue(Addr: KmpTaskSharedsPtr, T: SharedsTy);
4119 LValue Src = CGF.MakeAddrLValue(Addr: Shareds, T: SharedsTy);
4120 CGF.EmitAggregateCopy(Dest, Src, EltTy: SharedsTy, MayOverlap: AggValueSlot::DoesNotOverlap);
4121 }
4122 // Emit initial values for private copies (if any).
4123 TaskResultTy Result;
4124 if (!Privates.empty()) {
4125 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase: Base, KmpTaskTWithPrivatesQTyRD,
4126 SharedsTy, SharedsPtrTy, Data, Privates,
4127 /*ForDup=*/false);
4128 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) &&
4129 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4130 Result.TaskDupFn = emitTaskDupFunction(
4131 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4132 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4133 /*WithLastIter=*/!Data.LastprivateVars.empty());
4134 }
4135 }
4136 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4137 enum { Priority = 0, Destructors = 1 };
4138 // Provide pointer to function with destructors for privates.
4139 auto FI = std::next(x: KmpTaskTQTyRD->field_begin(), n: Data1);
4140 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
4141 assert(KmpCmplrdataUD->isUnion());
4142 if (NeedsCleanup) {
4143 llvm::Value *DestructorFn = emitDestructorsFunction(
4144 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4145 KmpTaskTWithPrivatesQTy);
4146 LValue Data1LV = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
4147 LValue DestructorsLV = CGF.EmitLValueForField(
4148 Base: Data1LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Destructors));
4149 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4150 V: DestructorFn, DestTy: KmpRoutineEntryPtrTy),
4151 lvalue: DestructorsLV);
4152 }
4153 // Set priority.
4154 if (Data.Priority.getInt()) {
4155 LValue Data2LV = CGF.EmitLValueForField(
4156 Base: TDBase, Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: Data2));
4157 LValue PriorityLV = CGF.EmitLValueForField(
4158 Base: Data2LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Priority));
4159 CGF.EmitStoreOfScalar(value: Data.Priority.getPointer(), lvalue: PriorityLV);
4160 }
4161 Result.NewTask = NewTask;
4162 Result.TaskEntry = TaskEntry;
4163 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4164 Result.TDBase = TDBase;
4165 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4166 return Result;
4167}
4168
4169/// Translates internal dependency kind into the runtime kind.
4170static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4171 RTLDependenceKindTy DepKind;
4172 switch (K) {
4173 case OMPC_DEPEND_in:
4174 DepKind = RTLDependenceKindTy::DepIn;
4175 break;
4176 // Out and InOut dependencies must use the same code.
4177 case OMPC_DEPEND_out:
4178 case OMPC_DEPEND_inout:
4179 DepKind = RTLDependenceKindTy::DepInOut;
4180 break;
4181 case OMPC_DEPEND_mutexinoutset:
4182 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4183 break;
4184 case OMPC_DEPEND_inoutset:
4185 DepKind = RTLDependenceKindTy::DepInOutSet;
4186 break;
4187 case OMPC_DEPEND_outallmemory:
4188 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4189 break;
4190 case OMPC_DEPEND_source:
4191 case OMPC_DEPEND_sink:
4192 case OMPC_DEPEND_depobj:
4193 case OMPC_DEPEND_inoutallmemory:
4194 case OMPC_DEPEND_unknown:
4195 llvm_unreachable("Unknown task dependence type");
4196 }
4197 return DepKind;
4198}
4199
4200/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4201static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4202 QualType &FlagsTy) {
4203 FlagsTy = C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.BoolTy), /*Signed=*/false);
4204 if (KmpDependInfoTy.isNull()) {
4205 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord(Name: "kmp_depend_info");
4206 KmpDependInfoRD->startDefinition();
4207 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getIntPtrType());
4208 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getSizeType());
4209 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: FlagsTy);
4210 KmpDependInfoRD->completeDefinition();
4211 KmpDependInfoTy = C.getCanonicalTagType(TD: KmpDependInfoRD);
4212 }
4213}
4214
4215std::pair<llvm::Value *, LValue>
4216CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4217 SourceLocation Loc) {
4218 ASTContext &C = CGM.getContext();
4219 QualType FlagsTy;
4220 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4221 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4222 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4223 LValue Base = CGF.EmitLoadOfPointerLValue(
4224 Ptr: DepobjLVal.getAddress().withElementType(
4225 ElemTy: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy)),
4226 PtrTy: KmpDependInfoPtrTy->castAs<PointerType>());
4227 Address DepObjAddr = CGF.Builder.CreateGEP(
4228 CGF, Addr: Base.getAddress(),
4229 Index: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4230 LValue NumDepsBase = CGF.MakeAddrLValue(
4231 Addr: DepObjAddr, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(), TBAAInfo: Base.getTBAAInfo());
4232 // NumDeps = deps[i].base_addr;
4233 LValue BaseAddrLVal = CGF.EmitLValueForField(
4234 Base: NumDepsBase,
4235 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4236 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4237 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(lvalue: BaseAddrLVal, Loc);
4238 return std::make_pair(x&: NumDeps, y&: Base);
4239}
4240
4241static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4242 llvm::PointerUnion<unsigned *, LValue *> Pos,
4243 const OMPTaskDataTy::DependData &Data,
4244 Address DependenciesArray) {
4245 CodeGenModule &CGM = CGF.CGM;
4246 ASTContext &C = CGM.getContext();
4247 QualType FlagsTy;
4248 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4249 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4250 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4251
4252 OMPIteratorGeneratorScope IteratorScope(
4253 CGF, cast_or_null<OMPIteratorExpr>(
4254 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4255 : nullptr));
4256 for (const Expr *E : Data.DepExprs) {
4257 llvm::Value *Addr;
4258 llvm::Value *Size;
4259
4260 // The expression will be a nullptr in the 'omp_all_memory' case.
4261 if (E) {
4262 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4263 Addr = CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy);
4264 } else {
4265 Addr = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4266 Size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
4267 }
4268 LValue Base;
4269 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4270 Base = CGF.MakeAddrLValue(
4271 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: *P), T: KmpDependInfoTy);
4272 } else {
4273 assert(E && "Expected a non-null expression");
4274 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4275 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4276 Base = CGF.MakeAddrLValue(
4277 Addr: CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Idx), T: KmpDependInfoTy);
4278 }
4279 // deps[i].base_addr = &<Dependencies[i].second>;
4280 LValue BaseAddrLVal = CGF.EmitLValueForField(
4281 Base,
4282 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4283 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4284 CGF.EmitStoreOfScalar(value: Addr, lvalue: BaseAddrLVal);
4285 // deps[i].len = sizeof(<Dependencies[i].second>);
4286 LValue LenLVal = CGF.EmitLValueForField(
4287 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4288 n: static_cast<unsigned int>(RTLDependInfoFields::Len)));
4289 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4290 // deps[i].flags = <Dependencies[i].first>;
4291 RTLDependenceKindTy DepKind = translateDependencyKind(K: Data.DepKind);
4292 LValue FlagsLVal = CGF.EmitLValueForField(
4293 Base,
4294 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4295 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4296 CGF.EmitStoreOfScalar(
4297 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4298 lvalue: FlagsLVal);
4299 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4300 ++(*P);
4301 } else {
4302 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4303 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4304 Idx = CGF.Builder.CreateNUWAdd(LHS: Idx,
4305 RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4306 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4307 }
4308 }
4309}
4310
4311SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes(
4312 CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4313 const OMPTaskDataTy::DependData &Data) {
4314 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4315 "Expected depobj dependency kind.");
4316 SmallVector<llvm::Value *, 4> Sizes;
4317 SmallVector<LValue, 4> SizeLVals;
4318 ASTContext &C = CGF.getContext();
4319 {
4320 OMPIteratorGeneratorScope IteratorScope(
4321 CGF, cast_or_null<OMPIteratorExpr>(
4322 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4323 : nullptr));
4324 for (const Expr *E : Data.DepExprs) {
4325 llvm::Value *NumDeps;
4326 LValue Base;
4327 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4328 std::tie(args&: NumDeps, args&: Base) =
4329 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4330 LValue NumLVal = CGF.MakeAddrLValue(
4331 Addr: CGF.CreateMemTempWithoutCast(T: C.getUIntPtrType(), Name: "depobj.size.addr"),
4332 T: C.getUIntPtrType());
4333 CGF.Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0),
4334 Addr: NumLVal.getAddress());
4335 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(lvalue: NumLVal, Loc: E->getExprLoc());
4336 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: PrevVal, RHS: NumDeps);
4337 CGF.EmitStoreOfScalar(value: Add, lvalue: NumLVal);
4338 SizeLVals.push_back(Elt: NumLVal);
4339 }
4340 }
4341 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4342 llvm::Value *Size =
4343 CGF.EmitLoadOfScalar(lvalue: SizeLVals[I], Loc: Data.DepExprs[I]->getExprLoc());
4344 Sizes.push_back(Elt: Size);
4345 }
4346 return Sizes;
4347}
4348
4349void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
4350 QualType &KmpDependInfoTy,
4351 LValue PosLVal,
4352 const OMPTaskDataTy::DependData &Data,
4353 Address DependenciesArray) {
4354 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4355 "Expected depobj dependency kind.");
4356 llvm::Value *ElSize = CGF.getTypeSize(Ty: KmpDependInfoTy);
4357 {
4358 OMPIteratorGeneratorScope IteratorScope(
4359 CGF, cast_or_null<OMPIteratorExpr>(
4360 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4361 : nullptr));
4362 for (const Expr *E : Data.DepExprs) {
4363 llvm::Value *NumDeps;
4364 LValue Base;
4365 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4366 std::tie(args&: NumDeps, args&: Base) =
4367 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4368
4369 // memcopy dependency data.
4370 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4371 LHS: ElSize,
4372 RHS: CGF.Builder.CreateIntCast(V: NumDeps, DestTy: CGF.SizeTy, /*isSigned=*/false));
4373 llvm::Value *Pos = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4374 Address DepAddr = CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Pos);
4375 CGF.Builder.CreateMemCpy(Dest: DepAddr, Src: Base.getAddress(), Size);
4376
4377 // Increase pos.
4378 // pos += size;
4379 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: Pos, RHS: NumDeps);
4380 CGF.EmitStoreOfScalar(value: Add, lvalue: PosLVal);
4381 }
4382 }
4383}
4384
4385std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4386 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4387 SourceLocation Loc) {
4388 if (llvm::all_of(Range&: Dependencies, P: [](const OMPTaskDataTy::DependData &D) {
4389 return D.DepExprs.empty();
4390 }))
4391 return std::make_pair(x: nullptr, y: Address::invalid());
4392 // Process list of dependencies.
4393 ASTContext &C = CGM.getContext();
4394 Address DependenciesArray = Address::invalid();
4395 llvm::Value *NumOfElements = nullptr;
4396 unsigned NumDependencies = std::accumulate(
4397 first: Dependencies.begin(), last: Dependencies.end(), init: 0,
4398 binary_op: [](unsigned V, const OMPTaskDataTy::DependData &D) {
4399 return D.DepKind == OMPC_DEPEND_depobj
4400 ? V
4401 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4402 });
4403 QualType FlagsTy;
4404 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4405 bool HasDepobjDeps = false;
4406 bool HasRegularWithIterators = false;
4407 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4408 llvm::Value *NumOfRegularWithIterators =
4409 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4410 // Calculate number of depobj dependencies and regular deps with the
4411 // iterators.
4412 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4413 if (D.DepKind == OMPC_DEPEND_depobj) {
4414 SmallVector<llvm::Value *, 4> Sizes =
4415 emitDepobjElementsSizes(CGF, KmpDependInfoTy, Data: D);
4416 for (llvm::Value *Size : Sizes) {
4417 NumOfDepobjElements =
4418 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: Size);
4419 }
4420 HasDepobjDeps = true;
4421 continue;
4422 }
4423 // Include number of iterations, if any.
4424
4425 if (const auto *IE = cast_or_null<OMPIteratorExpr>(Val: D.IteratorExpr)) {
4426 llvm::Value *ClauseIteratorSpace =
4427 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 1);
4428 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4429 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4430 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4431 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(LHS: Sz, RHS: ClauseIteratorSpace);
4432 }
4433 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4434 LHS: ClauseIteratorSpace,
4435 RHS: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: D.DepExprs.size()));
4436 NumOfRegularWithIterators =
4437 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumClauseDeps);
4438 HasRegularWithIterators = true;
4439 continue;
4440 }
4441 }
4442
4443 QualType KmpDependInfoArrayTy;
4444 if (HasDepobjDeps || HasRegularWithIterators) {
4445 NumOfElements = llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: NumDependencies,
4446 /*isSigned=*/IsSigned: false);
4447 if (HasDepobjDeps) {
4448 NumOfElements =
4449 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: NumOfElements);
4450 }
4451 if (HasRegularWithIterators) {
4452 NumOfElements =
4453 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumOfElements);
4454 }
4455 auto *OVE = new (C) OpaqueValueExpr(
4456 Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4457 VK_PRValue);
4458 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4459 RValue::get(V: NumOfElements));
4460 KmpDependInfoArrayTy =
4461 C.getVariableArrayType(EltTy: KmpDependInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4462 /*IndexTypeQuals=*/0);
4463 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4464 // Properly emit variable-sized array.
4465 auto *PD = ImplicitParamDecl::Create(C, T: KmpDependInfoArrayTy,
4466 ParamKind: ImplicitParamKind::Other);
4467 CGF.EmitVarDecl(D: *PD);
4468 DependenciesArray = CGF.GetAddrOfLocalVar(VD: PD);
4469 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4470 /*isSigned=*/false);
4471 } else {
4472 KmpDependInfoArrayTy = C.getConstantArrayType(
4473 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies), SizeExpr: nullptr,
4474 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4475 DependenciesArray =
4476 CGF.CreateMemTempWithoutCast(T: KmpDependInfoArrayTy, Name: ".dep.arr.addr");
4477 DependenciesArray = CGF.Builder.CreateConstArrayGEP(Addr: DependenciesArray, Index: 0);
4478 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumDependencies,
4479 /*isSigned=*/IsSigned: false);
4480 }
4481 unsigned Pos = 0;
4482 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4483 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4484 continue;
4485 emitDependData(CGF, KmpDependInfoTy, Pos: &Pos, Data: Dep, DependenciesArray);
4486 }
4487 // Copy regular dependencies with iterators.
4488 LValue PosLVal = CGF.MakeAddrLValue(
4489 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "dep.counter.addr"),
4490 T: C.getSizeType());
4491 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4492 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4493 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4494 continue;
4495 emitDependData(CGF, KmpDependInfoTy, Pos: &PosLVal, Data: Dep, DependenciesArray);
4496 }
4497 // Copy final depobj arrays without iterators.
4498 if (HasDepobjDeps) {
4499 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4500 if (Dep.DepKind != OMPC_DEPEND_depobj)
4501 continue;
4502 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Data: Dep, DependenciesArray);
4503 }
4504 }
4505 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4506 Addr: DependenciesArray, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
4507 return std::make_pair(x&: NumOfElements, y&: DependenciesArray);
4508}
4509
4510Address CGOpenMPRuntime::emitDepobjDependClause(
4511 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4512 SourceLocation Loc) {
4513 if (Dependencies.DepExprs.empty())
4514 return Address::invalid();
4515 // Process list of dependencies.
4516 ASTContext &C = CGM.getContext();
4517 Address DependenciesArray = Address::invalid();
4518 unsigned NumDependencies = Dependencies.DepExprs.size();
4519 QualType FlagsTy;
4520 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4521 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4522
4523 llvm::Value *Size;
4524 // Define type kmp_depend_info[<Dependencies.size()>];
4525 // For depobj reserve one extra element to store the number of elements.
4526 // It is required to handle depobj(x) update(in) construct.
4527 // kmp_depend_info[<Dependencies.size()>] deps;
4528 llvm::Value *NumDepsVal;
4529 CharUnits Align = C.getTypeAlignInChars(T: KmpDependInfoTy);
4530 if (const auto *IE =
4531 cast_or_null<OMPIteratorExpr>(Val: Dependencies.IteratorExpr)) {
4532 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
4533 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4534 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4535 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
4536 NumDepsVal = CGF.Builder.CreateNUWMul(LHS: NumDepsVal, RHS: Sz);
4537 }
4538 Size = CGF.Builder.CreateNUWAdd(LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1),
4539 RHS: NumDepsVal);
4540 CharUnits SizeInBytes =
4541 C.getTypeSizeInChars(T: KmpDependInfoTy).alignTo(Align);
4542 llvm::Value *RecSize = CGM.getSize(numChars: SizeInBytes);
4543 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: RecSize);
4544 NumDepsVal =
4545 CGF.Builder.CreateIntCast(V: NumDepsVal, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4546 } else {
4547 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4548 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4549 SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4550 CharUnits Sz = C.getTypeSizeInChars(T: KmpDependInfoArrayTy);
4551 Size = CGM.getSize(numChars: Sz.alignTo(Align));
4552 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: NumDependencies);
4553 }
4554 // Need to allocate on the dynamic memory.
4555 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4556 // Use default allocator.
4557 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4558 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4559
4560 llvm::Value *Addr =
4561 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4562 M&: CGM.getModule(), FnID: OMPRTL___kmpc_alloc),
4563 args: Args, name: ".dep.arr.addr");
4564 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(T: KmpDependInfoTy);
4565 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4566 V: Addr, DestTy: CGF.Builder.getPtrTy(AddrSpace: 0));
4567 DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
4568 // Write number of elements in the first element of array for depobj.
4569 LValue Base = CGF.MakeAddrLValue(Addr: DependenciesArray, T: KmpDependInfoTy);
4570 // deps[i].base_addr = NumDependencies;
4571 LValue BaseAddrLVal = CGF.EmitLValueForField(
4572 Base,
4573 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4574 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4575 CGF.EmitStoreOfScalar(value: NumDepsVal, lvalue: BaseAddrLVal);
4576 llvm::PointerUnion<unsigned *, LValue *> Pos;
4577 unsigned Idx = 1;
4578 LValue PosLVal;
4579 if (Dependencies.IteratorExpr) {
4580 PosLVal = CGF.MakeAddrLValue(
4581 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "iterator.counter.addr"),
4582 T: C.getSizeType());
4583 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Idx), lvalue: PosLVal,
4584 /*IsInit=*/isInit: true);
4585 Pos = &PosLVal;
4586 } else {
4587 Pos = &Idx;
4588 }
4589 emitDependData(CGF, KmpDependInfoTy, Pos, Data: Dependencies, DependenciesArray);
4590 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4591 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: 1), Ty: CGF.VoidPtrTy,
4592 ElementTy: CGF.Int8Ty);
4593 return DependenciesArray;
4594}
4595
4596void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4597 SourceLocation Loc) {
4598 ASTContext &C = CGM.getContext();
4599 QualType FlagsTy;
4600 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4601 LValue Base = CGF.EmitLoadOfPointerLValue(Ptr: DepobjLVal.getAddress(),
4602 PtrTy: C.VoidPtrTy.castAs<PointerType>());
4603 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4604 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4605 Addr: Base.getAddress(), Ty: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy),
4606 ElementTy: CGF.ConvertTypeForMem(T: KmpDependInfoTy));
4607 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4608 Ty: Addr.getElementType(), Ptr: Addr.emitRawPointer(CGF),
4609 IdxList: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4610 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: DepObjAddr,
4611 DestTy: CGF.VoidPtrTy);
4612 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4613 // Use default allocator.
4614 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4615 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4616
4617 // _kmpc_free(gtid, addr, nullptr);
4618 (void)CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4619 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free),
4620 args: Args);
4621}
4622
4623void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
4624 OpenMPDependClauseKind NewDepKind,
4625 SourceLocation Loc) {
4626 ASTContext &C = CGM.getContext();
4627 QualType FlagsTy;
4628 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4629 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4630 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4631 llvm::Value *NumDeps;
4632 LValue Base;
4633 std::tie(args&: NumDeps, args&: Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4634
4635 Address Begin = Base.getAddress();
4636 // Cast from pointer to array type to pointer to single element.
4637 llvm::Value *End = CGF.Builder.CreateGEP(Ty: Begin.getElementType(),
4638 Ptr: Begin.emitRawPointer(CGF), IdxList: NumDeps);
4639 // The basic structure here is a while-do loop.
4640 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.body");
4641 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.done");
4642 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4643 CGF.EmitBlock(BB: BodyBB);
4644 llvm::PHINode *ElementPHI =
4645 CGF.Builder.CreatePHI(Ty: Begin.getType(), NumReservedValues: 2, Name: "omp.elementPast");
4646 ElementPHI->addIncoming(V: Begin.emitRawPointer(CGF), BB: EntryBB);
4647 Begin = Begin.withPointer(NewPointer: ElementPHI, IsKnownNonNull: KnownNonNull);
4648 Base = CGF.MakeAddrLValue(Addr: Begin, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(),
4649 TBAAInfo: Base.getTBAAInfo());
4650 // deps[i].flags = NewDepKind;
4651 RTLDependenceKindTy DepKind = translateDependencyKind(K: NewDepKind);
4652 LValue FlagsLVal = CGF.EmitLValueForField(
4653 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4654 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4655 CGF.EmitStoreOfScalar(
4656 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4657 lvalue: FlagsLVal);
4658
4659 // Shift the address forward by one element.
4660 llvm::Value *ElementNext =
4661 CGF.Builder.CreateConstGEP(Addr: Begin, /*Index=*/1, Name: "omp.elementNext")
4662 .emitRawPointer(CGF);
4663 ElementPHI->addIncoming(V: ElementNext, BB: CGF.Builder.GetInsertBlock());
4664 llvm::Value *IsEmpty =
4665 CGF.Builder.CreateICmpEQ(LHS: ElementNext, RHS: End, Name: "omp.isempty");
4666 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4667 // Done.
4668 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4669}
4670
4671void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4672 const OMPExecutableDirective &D,
4673 llvm::Function *TaskFunction,
4674 QualType SharedsTy, Address Shareds,
4675 const Expr *IfCond,
4676 const OMPTaskDataTy &Data) {
4677 if (!CGF.HaveInsertPoint())
4678 return;
4679
4680 TaskResultTy Result =
4681 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4682 llvm::Value *NewTask = Result.NewTask;
4683 llvm::Function *TaskEntry = Result.TaskEntry;
4684 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4685 LValue TDBase = Result.TDBase;
4686 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
4687 // Process list of dependences.
4688 Address DependenciesArray = Address::invalid();
4689 llvm::Value *NumOfElements;
4690 std::tie(args&: NumOfElements, args&: DependenciesArray) =
4691 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
4692
4693 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4694 // libcall.
4695 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
4696 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
4697 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
4698 // list is not empty
4699 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4700 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4701 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4702 llvm::Value *DepTaskArgs[7];
4703 if (!Data.Dependences.empty()) {
4704 DepTaskArgs[0] = UpLoc;
4705 DepTaskArgs[1] = ThreadID;
4706 DepTaskArgs[2] = NewTask;
4707 DepTaskArgs[3] = NumOfElements;
4708 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4709 DepTaskArgs[5] = CGF.Builder.getInt32(C: 0);
4710 DepTaskArgs[6] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4711 }
4712 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4713 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4714 if (!Data.Tied) {
4715 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
4716 LValue PartIdLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PartIdFI);
4717 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: 0), lvalue: PartIdLVal);
4718 }
4719 if (!Data.Dependences.empty()) {
4720 CGF.EmitRuntimeCall(
4721 callee: OMPBuilder.getOrCreateRuntimeFunction(
4722 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_with_deps),
4723 args: DepTaskArgs);
4724 } else {
4725 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4726 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
4727 args: TaskArgs);
4728 }
4729 // Check if parent region is untied and build return for untied task;
4730 if (auto *Region =
4731 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
4732 Region->emitUntiedSwitch(CGF);
4733 };
4734
4735 llvm::Value *DepWaitTaskArgs[7];
4736 if (!Data.Dependences.empty()) {
4737 DepWaitTaskArgs[0] = UpLoc;
4738 DepWaitTaskArgs[1] = ThreadID;
4739 DepWaitTaskArgs[2] = NumOfElements;
4740 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4741 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
4742 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4743 DepWaitTaskArgs[6] =
4744 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
4745 }
4746 auto &M = CGM.getModule();
4747 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4748 TaskEntry, &Data, &DepWaitTaskArgs,
4749 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4750 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4751 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
4752 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
4753 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
4754 // is specified.
4755 if (!Data.Dependences.empty())
4756 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4757 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
4758 args: DepWaitTaskArgs);
4759 // Call proxy_task_entry(gtid, new_task);
4760 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4761 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4762 Action.Enter(CGF);
4763 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4764 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskEntry,
4765 Args: OutlinedFnArgs);
4766 };
4767
4768 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
4769 // kmp_task_t *new_task);
4770 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
4771 // kmp_task_t *new_task);
4772 RegionCodeGenTy RCG(CodeGen);
4773 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
4774 M, FnID: OMPRTL___kmpc_omp_task_begin_if0),
4775 TaskArgs,
4776 OMPBuilder.getOrCreateRuntimeFunction(
4777 M, FnID: OMPRTL___kmpc_omp_task_complete_if0),
4778 TaskArgs);
4779 RCG.setAction(Action);
4780 RCG(CGF);
4781 };
4782
4783 if (IfCond) {
4784 emitIfClause(CGF, Cond: IfCond, ThenGen: ThenCodeGen, ElseGen: ElseCodeGen);
4785 } else {
4786 RegionCodeGenTy ThenRCG(ThenCodeGen);
4787 ThenRCG(CGF);
4788 }
4789}
4790
4791void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4792 const OMPLoopDirective &D,
4793 llvm::Function *TaskFunction,
4794 QualType SharedsTy, Address Shareds,
4795 const Expr *IfCond,
4796 const OMPTaskDataTy &Data) {
4797 if (!CGF.HaveInsertPoint())
4798 return;
4799 TaskResultTy Result =
4800 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4801 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4802 // libcall.
4803 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
4804 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
4805 // sched, kmp_uint64 grainsize, void *task_dup);
4806 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4807 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4808 llvm::Value *IfVal;
4809 if (IfCond) {
4810 IfVal = CGF.Builder.CreateIntCast(V: CGF.EvaluateExprAsBool(E: IfCond), DestTy: CGF.IntTy,
4811 /*isSigned=*/true);
4812 } else {
4813 IfVal = llvm::ConstantInt::getSigned(Ty: CGF.IntTy, /*V=*/1);
4814 }
4815
4816 LValue LBLVal = CGF.EmitLValueForField(
4817 Base: Result.TDBase,
4818 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound));
4819 const auto *LBVar =
4820 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getLowerBoundVariable())->getDecl());
4821 CGF.EmitAnyExprToMem(E: LBVar->getInit(), Location: LBLVal.getAddress(), Quals: LBLVal.getQuals(),
4822 /*IsInitializer=*/true);
4823 LValue UBLVal = CGF.EmitLValueForField(
4824 Base: Result.TDBase,
4825 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound));
4826 const auto *UBVar =
4827 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getUpperBoundVariable())->getDecl());
4828 CGF.EmitAnyExprToMem(E: UBVar->getInit(), Location: UBLVal.getAddress(), Quals: UBLVal.getQuals(),
4829 /*IsInitializer=*/true);
4830 LValue StLVal = CGF.EmitLValueForField(
4831 Base: Result.TDBase,
4832 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride));
4833 const auto *StVar =
4834 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getStrideVariable())->getDecl());
4835 CGF.EmitAnyExprToMem(E: StVar->getInit(), Location: StLVal.getAddress(), Quals: StLVal.getQuals(),
4836 /*IsInitializer=*/true);
4837 // Store reductions address.
4838 LValue RedLVal = CGF.EmitLValueForField(
4839 Base: Result.TDBase,
4840 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions));
4841 if (Data.Reductions) {
4842 CGF.EmitStoreOfScalar(value: Data.Reductions, lvalue: RedLVal);
4843 } else {
4844 CGF.EmitNullInitialization(DestPtr: RedLVal.getAddress(),
4845 Ty: CGF.getContext().VoidPtrTy);
4846 }
4847 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4848 llvm::SmallVector<llvm::Value *, 12> TaskArgs{
4849 UpLoc,
4850 ThreadID,
4851 Result.NewTask,
4852 IfVal,
4853 LBLVal.getPointer(CGF),
4854 UBLVal.getPointer(CGF),
4855 CGF.EmitLoadOfScalar(lvalue: StLVal, Loc),
4856 llvm::ConstantInt::getSigned(
4857 Ty: CGF.IntTy, V: 1), // Always 1 because taskgroup emitted by the compiler
4858 llvm::ConstantInt::getSigned(
4859 Ty: CGF.IntTy, V: Data.Schedule.getPointer()
4860 ? Data.Schedule.getInt() ? NumTasks : Grainsize
4861 : NoSchedule),
4862 Data.Schedule.getPointer()
4863 ? CGF.Builder.CreateIntCast(V: Data.Schedule.getPointer(), DestTy: CGF.Int64Ty,
4864 /*isSigned=*/false)
4865 : llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/0)};
4866 if (Data.HasModifier)
4867 TaskArgs.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 1));
4868
4869 TaskArgs.push_back(Elt: Result.TaskDupFn
4870 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4871 V: Result.TaskDupFn, DestTy: CGF.VoidPtrTy)
4872 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy));
4873 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4874 M&: CGM.getModule(), FnID: Data.HasModifier
4875 ? OMPRTL___kmpc_taskloop_5
4876 : OMPRTL___kmpc_taskloop),
4877 args: TaskArgs);
4878}
4879
4880/// Emit reduction operation for each element of array (required for
4881/// array sections) LHS op = RHS.
4882/// \param Type Type of array.
4883/// \param LHSVar Variable on the left side of the reduction operation
4884/// (references element of array in original variable).
4885/// \param RHSVar Variable on the right side of the reduction operation
4886/// (references element of array in original variable).
4887/// \param RedOpGen Generator of reduction operation with use of LHSVar and
4888/// RHSVar.
4889static void EmitOMPAggregateReduction(
4890 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
4891 const VarDecl *RHSVar,
4892 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
4893 const Expr *, const Expr *)> &RedOpGen,
4894 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
4895 const Expr *UpExpr = nullptr) {
4896 // Perform element-by-element initialization.
4897 QualType ElementTy;
4898 Address LHSAddr = CGF.GetAddrOfLocalVar(VD: LHSVar);
4899 Address RHSAddr = CGF.GetAddrOfLocalVar(VD: RHSVar);
4900
4901 // Drill down to the base element type on both arrays.
4902 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
4903 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: LHSAddr);
4904
4905 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
4906 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
4907 // Cast from pointer to array type to pointer to single element.
4908 llvm::Value *LHSEnd =
4909 CGF.Builder.CreateGEP(Ty: LHSAddr.getElementType(), Ptr: LHSBegin, IdxList: NumElements);
4910 // The basic structure here is a while-do loop.
4911 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arraycpy.body");
4912 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arraycpy.done");
4913 llvm::Value *IsEmpty =
4914 CGF.Builder.CreateICmpEQ(LHS: LHSBegin, RHS: LHSEnd, Name: "omp.arraycpy.isempty");
4915 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4916
4917 // Enter the loop body, making that address the current address.
4918 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4919 CGF.EmitBlock(BB: BodyBB);
4920
4921 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
4922
4923 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
4924 Ty: RHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
4925 RHSElementPHI->addIncoming(V: RHSBegin, BB: EntryBB);
4926 Address RHSElementCurrent(
4927 RHSElementPHI, RHSAddr.getElementType(),
4928 RHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
4929
4930 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
4931 Ty: LHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
4932 LHSElementPHI->addIncoming(V: LHSBegin, BB: EntryBB);
4933 Address LHSElementCurrent(
4934 LHSElementPHI, LHSAddr.getElementType(),
4935 LHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
4936
4937 // Emit copy.
4938 CodeGenFunction::OMPPrivateScope Scope(CGF);
4939 Scope.addPrivate(LocalVD: LHSVar, Addr: LHSElementCurrent);
4940 Scope.addPrivate(LocalVD: RHSVar, Addr: RHSElementCurrent);
4941 Scope.Privatize();
4942 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4943 Scope.ForceCleanup();
4944
4945 // Shift the address forward by one element.
4946 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
4947 Ty: LHSAddr.getElementType(), Ptr: LHSElementPHI, /*Idx0=*/1,
4948 Name: "omp.arraycpy.dest.element");
4949 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
4950 Ty: RHSAddr.getElementType(), Ptr: RHSElementPHI, /*Idx0=*/1,
4951 Name: "omp.arraycpy.src.element");
4952 // Check whether we've reached the end.
4953 llvm::Value *Done =
4954 CGF.Builder.CreateICmpEQ(LHS: LHSElementNext, RHS: LHSEnd, Name: "omp.arraycpy.done");
4955 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
4956 LHSElementPHI->addIncoming(V: LHSElementNext, BB: CGF.Builder.GetInsertBlock());
4957 RHSElementPHI->addIncoming(V: RHSElementNext, BB: CGF.Builder.GetInsertBlock());
4958
4959 // Done.
4960 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4961}
4962
4963/// Emit reduction combiner. If the combiner is a simple expression emit it as
4964/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
4965/// UDR combiner function.
4966static void emitReductionCombiner(CodeGenFunction &CGF,
4967 const Expr *ReductionOp) {
4968 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
4969 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
4970 if (const auto *DRE =
4971 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
4972 if (const auto *DRD =
4973 dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl())) {
4974 std::pair<llvm::Function *, llvm::Function *> Reduction =
4975 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
4976 RValue Func = RValue::get(V: Reduction.first);
4977 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
4978 CGF.EmitIgnoredExpr(E: ReductionOp);
4979 return;
4980 }
4981 CGF.EmitIgnoredExpr(E: ReductionOp);
4982}
4983
4984llvm::Function *CGOpenMPRuntime::emitReductionFunction(
4985 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
4986 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
4987 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
4988 ASTContext &C = CGM.getContext();
4989
4990 // void reduction_func(void *LHSArg, void *RHSArg);
4991 auto *LHSArg =
4992 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
4993 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4994 auto *RHSArg =
4995 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
4996 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4997 FunctionArgList Args{LHSArg, RHSArg};
4998 const auto &CGFI =
4999 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5000 std::string Name = getReductionFuncName(Name: ReducerName);
5001 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
5002 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
5003 M: &CGM.getModule());
5004 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
5005 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5006 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5007 Fn->setDoesNotRecurse();
5008 CodeGenFunction CGF(CGM);
5009 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
5010
5011 // Dst = (void*[n])(LHSArg);
5012 // Src = (void*[n])(RHSArg);
5013 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5014 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
5015 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5016 ArgsElemType, CGF.getPointerAlign());
5017 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5018 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
5019 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5020 ArgsElemType, CGF.getPointerAlign());
5021
5022 // ...
5023 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5024 // ...
5025 CodeGenFunction::OMPPrivateScope Scope(CGF);
5026 const auto *IPriv = Privates.begin();
5027 unsigned Idx = 0;
5028 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5029 const auto *RHSVar =
5030 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSExprs[I])->getDecl());
5031 Scope.addPrivate(LocalVD: RHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: RHS, Index: Idx, Var: RHSVar));
5032 const auto *LHSVar =
5033 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSExprs[I])->getDecl());
5034 Scope.addPrivate(LocalVD: LHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: LHS, Index: Idx, Var: LHSVar));
5035 QualType PrivTy = (*IPriv)->getType();
5036 if (PrivTy->isVariablyModifiedType()) {
5037 // Get array size and emit VLA type.
5038 ++Idx;
5039 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: LHS, Index: Idx);
5040 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Elem);
5041 const VariableArrayType *VLA =
5042 CGF.getContext().getAsVariableArrayType(T: PrivTy);
5043 const auto *OVE = cast<OpaqueValueExpr>(Val: VLA->getSizeExpr());
5044 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5045 CGF, OVE, RValue::get(V: CGF.Builder.CreatePtrToInt(V: Ptr, DestTy: CGF.SizeTy)));
5046 CGF.EmitVariablyModifiedType(Ty: PrivTy);
5047 }
5048 }
5049 Scope.Privatize();
5050 IPriv = Privates.begin();
5051 const auto *ILHS = LHSExprs.begin();
5052 const auto *IRHS = RHSExprs.begin();
5053 for (const Expr *E : ReductionOps) {
5054 if ((*IPriv)->getType()->isArrayType()) {
5055 // Emit reduction for array section.
5056 const auto *LHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5057 const auto *RHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5058 EmitOMPAggregateReduction(
5059 CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5060 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5061 emitReductionCombiner(CGF, ReductionOp: E);
5062 });
5063 } else {
5064 // Emit reduction for array subscript or single variable.
5065 emitReductionCombiner(CGF, ReductionOp: E);
5066 }
5067 ++IPriv;
5068 ++ILHS;
5069 ++IRHS;
5070 }
5071 Scope.ForceCleanup();
5072 CGF.FinishFunction();
5073 return Fn;
5074}
5075
5076void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5077 const Expr *ReductionOp,
5078 const Expr *PrivateRef,
5079 const DeclRefExpr *LHS,
5080 const DeclRefExpr *RHS) {
5081 if (PrivateRef->getType()->isArrayType()) {
5082 // Emit reduction for array section.
5083 const auto *LHSVar = cast<VarDecl>(Val: LHS->getDecl());
5084 const auto *RHSVar = cast<VarDecl>(Val: RHS->getDecl());
5085 EmitOMPAggregateReduction(
5086 CGF, Type: PrivateRef->getType(), LHSVar, RHSVar,
5087 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5088 emitReductionCombiner(CGF, ReductionOp);
5089 });
5090 } else {
5091 // Emit reduction for array subscript or single variable.
5092 emitReductionCombiner(CGF, ReductionOp);
5093 }
5094}
5095
5096static std::string generateUniqueName(CodeGenModule &CGM,
5097 llvm::StringRef Prefix, const Expr *Ref);
5098
5099void CGOpenMPRuntime::emitPrivateReduction(
5100 CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,
5101 const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {
5102
5103 // Create a shared global variable (__shared_reduction_var) to accumulate the
5104 // final result.
5105 //
5106 // Call __kmpc_barrier to synchronize threads before initialization.
5107 //
5108 // The master thread (thread_id == 0) initializes __shared_reduction_var
5109 // with the identity value or initializer.
5110 //
5111 // Call __kmpc_barrier to synchronize before combining.
5112 // For each i:
5113 // - Thread enters critical section.
5114 // - Reads its private value from LHSExprs[i].
5115 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
5116 // Privates[i]).
5117 // - Exits critical section.
5118 //
5119 // Call __kmpc_barrier after combining.
5120 //
5121 // Each thread copies __shared_reduction_var[i] back to RHSExprs[i].
5122 //
5123 // Final __kmpc_barrier to synchronize after broadcasting
5124 QualType PrivateType = Privates->getType();
5125 llvm::Type *LLVMType = CGF.ConvertTypeForMem(T: PrivateType);
5126
5127 const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOp: ReductionOps);
5128 std::string ReductionVarNameStr;
5129 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates->IgnoreParenCasts()))
5130 ReductionVarNameStr =
5131 generateUniqueName(CGM, Prefix: DRE->getDecl()->getNameAsString(), Ref: Privates);
5132 else
5133 ReductionVarNameStr = "unnamed_priv_var";
5134
5135 // Create an internal shared variable
5136 std::string SharedName =
5137 CGM.getOpenMPRuntime().getName(Parts: {"internal_pivate_", ReductionVarNameStr});
5138 llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(
5139 Ty: LLVMType, Name: ".omp.reduction." + SharedName);
5140
5141 SharedVar->setAlignment(
5142 llvm::MaybeAlign(CGF.getContext().getTypeAlign(T: PrivateType) / 8));
5143
5144 Address SharedResult =
5145 CGF.MakeNaturalAlignRawAddrLValue(V: SharedVar, T: PrivateType).getAddress();
5146
5147 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5148 llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5149 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
5150
5151 llvm::BasicBlock *InitBB = CGF.createBasicBlock(name: "init");
5152 llvm::BasicBlock *InitEndBB = CGF.createBasicBlock(name: "init.end");
5153
5154 llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(
5155 LHS: ThreadId, RHS: llvm::ConstantInt::get(Ty: ThreadId->getType(), V: 0));
5156 CGF.Builder.CreateCondBr(Cond: IsWorker, True: InitBB, False: InitEndBB);
5157
5158 CGF.EmitBlock(BB: InitBB);
5159
5160 auto EmitSharedInit = [&]() {
5161 if (UDR) { // Check if it's a User-Defined Reduction
5162 if (const Expr *UDRInitExpr = UDR->getInitializer()) {
5163 std::pair<llvm::Function *, llvm::Function *> FnPair =
5164 getUserDefinedReduction(D: UDR);
5165 llvm::Function *InitializerFn = FnPair.second;
5166 if (InitializerFn) {
5167 if (const auto *CE =
5168 dyn_cast<CallExpr>(Val: UDRInitExpr->IgnoreParenImpCasts())) {
5169 const auto *OutDRE = cast<DeclRefExpr>(
5170 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5171 ->getSubExpr());
5172 const VarDecl *OutVD = cast<VarDecl>(Val: OutDRE->getDecl());
5173
5174 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5175 LocalScope.addPrivate(LocalVD: OutVD, Addr: SharedResult);
5176
5177 (void)LocalScope.Privatize();
5178 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(
5179 Val: CE->getCallee()->IgnoreParenImpCasts())) {
5180 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5181 CGF, OVE, RValue::get(V: InitializerFn));
5182 CGF.EmitIgnoredExpr(E: CE);
5183 } else {
5184 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5185 Quals: PrivateType.getQualifiers(),
5186 /*IsInitializer=*/true);
5187 }
5188 } else {
5189 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5190 Quals: PrivateType.getQualifiers(),
5191 /*IsInitializer=*/true);
5192 }
5193 } else {
5194 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5195 Quals: PrivateType.getQualifiers(),
5196 /*IsInitializer=*/true);
5197 }
5198 } else {
5199 // EmitNullInitialization handles default construction for C++ classes
5200 // and zeroing for scalars, which is a reasonable default.
5201 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5202 }
5203 return; // UDR initialization handled
5204 }
5205 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates)) {
5206 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
5207 if (const Expr *InitExpr = VD->getInit()) {
5208 CGF.EmitAnyExprToMem(E: InitExpr, Location: SharedResult,
5209 Quals: PrivateType.getQualifiers(), IsInitializer: true);
5210 return;
5211 }
5212 }
5213 }
5214 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5215 };
5216 EmitSharedInit();
5217 CGF.Builder.CreateBr(Dest: InitEndBB);
5218 CGF.EmitBlock(BB: InitEndBB);
5219
5220 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5221 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5222 args: BarrierArgs);
5223
5224 const Expr *ReductionOp = ReductionOps;
5225 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
5226 LValue SharedLV = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5227 LValue LHSLV = CGF.EmitLValue(E: Privates);
5228
5229 auto EmitCriticalReduction = [&](auto ReductionGen) {
5230 std::string CriticalName = getName(Parts: {"reduction_critical"});
5231 emitCriticalRegion(CGF, CriticalName, CriticalOpGen: ReductionGen, Loc);
5232 };
5233
5234 if (CurrentUDR) {
5235 // Handle user-defined reduction.
5236 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5237 Action.Enter(CGF);
5238 std::pair<llvm::Function *, llvm::Function *> FnPair =
5239 getUserDefinedReduction(D: CurrentUDR);
5240 if (FnPair.first) {
5241 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp)) {
5242 const auto *OutDRE = cast<DeclRefExpr>(
5243 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5244 ->getSubExpr());
5245 const auto *InDRE = cast<DeclRefExpr>(
5246 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 1)->IgnoreParenImpCasts())
5247 ->getSubExpr());
5248 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5249 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: OutDRE->getDecl()),
5250 Addr: SharedLV.getAddress());
5251 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: InDRE->getDecl()),
5252 Addr: LHSLV.getAddress());
5253 (void)LocalScope.Privatize();
5254 emitReductionCombiner(CGF, ReductionOp);
5255 }
5256 }
5257 };
5258 EmitCriticalReduction(ReductionGen);
5259 } else {
5260 // Handle built-in reduction operations.
5261#ifndef NDEBUG
5262 const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();
5263 if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5264 ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
5265
5266 const Expr *AssignRHS = nullptr;
5267 if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5268 if (BinOp->getOpcode() == BO_Assign)
5269 AssignRHS = BinOp->getRHS();
5270 } else if (const auto *OpCall =
5271 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5272 if (OpCall->getOperator() == OO_Equal)
5273 AssignRHS = OpCall->getArg(1);
5274 }
5275
5276 assert(AssignRHS &&
5277 "Private Variable Reduction : Invalid ReductionOp expression");
5278#endif
5279
5280 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5281 Action.Enter(CGF);
5282 const auto *OmpOutDRE =
5283 dyn_cast<DeclRefExpr>(Val: LHSExprs->IgnoreParenImpCasts());
5284 const auto *OmpInDRE =
5285 dyn_cast<DeclRefExpr>(Val: RHSExprs->IgnoreParenImpCasts());
5286 assert(
5287 OmpOutDRE && OmpInDRE &&
5288 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5289 const VarDecl *OmpOutVD = cast<VarDecl>(Val: OmpOutDRE->getDecl());
5290 const VarDecl *OmpInVD = cast<VarDecl>(Val: OmpInDRE->getDecl());
5291 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5292 LocalScope.addPrivate(LocalVD: OmpOutVD, Addr: SharedLV.getAddress());
5293 LocalScope.addPrivate(LocalVD: OmpInVD, Addr: LHSLV.getAddress());
5294 (void)LocalScope.Privatize();
5295 // Emit the actual reduction operation
5296 CGF.EmitIgnoredExpr(E: ReductionOp);
5297 };
5298 EmitCriticalReduction(ReductionGen);
5299 }
5300
5301 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5302 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5303 args: BarrierArgs);
5304
5305 // Broadcast final result
5306 bool IsAggregate = PrivateType->isAggregateType();
5307 LValue SharedLV1 = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5308 llvm::Value *FinalResultVal = nullptr;
5309 Address FinalResultAddr = Address::invalid();
5310
5311 if (IsAggregate)
5312 FinalResultAddr = SharedResult;
5313 else
5314 FinalResultVal = CGF.EmitLoadOfScalar(lvalue: SharedLV1, Loc);
5315
5316 LValue TargetLHSLV = CGF.EmitLValue(E: RHSExprs);
5317 if (IsAggregate) {
5318 CGF.EmitAggregateCopy(Dest: TargetLHSLV,
5319 Src: CGF.MakeAddrLValue(Addr: FinalResultAddr, T: PrivateType),
5320 EltTy: PrivateType, MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: false);
5321 } else {
5322 CGF.EmitStoreOfScalar(value: FinalResultVal, lvalue: TargetLHSLV);
5323 }
5324 // Final synchronization barrier
5325 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5326 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5327 args: BarrierArgs);
5328
5329 // Combiner with original list item
5330 auto OriginalListCombiner = [&](CodeGenFunction &CGF,
5331 PrePostActionTy &Action) {
5332 Action.Enter(CGF);
5333 emitSingleReductionCombiner(CGF, ReductionOp: ReductionOps, PrivateRef: Privates,
5334 LHS: cast<DeclRefExpr>(Val: LHSExprs),
5335 RHS: cast<DeclRefExpr>(Val: RHSExprs));
5336 };
5337 EmitCriticalReduction(OriginalListCombiner);
5338}
5339
5340void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5341 ArrayRef<const Expr *> OrgPrivates,
5342 ArrayRef<const Expr *> OrgLHSExprs,
5343 ArrayRef<const Expr *> OrgRHSExprs,
5344 ArrayRef<const Expr *> OrgReductionOps,
5345 ReductionOptionsTy Options) {
5346 if (!CGF.HaveInsertPoint())
5347 return;
5348
5349 bool WithNowait = Options.WithNowait;
5350 bool SimpleReduction = Options.SimpleReduction;
5351
5352 // Next code should be emitted for reduction:
5353 //
5354 // static kmp_critical_name lock = { 0 };
5355 //
5356 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5357 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5358 // ...
5359 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5360 // *(Type<n>-1*)rhs[<n>-1]);
5361 // }
5362 //
5363 // ...
5364 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5365 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5366 // RedList, reduce_func, &<lock>)) {
5367 // case 1:
5368 // ...
5369 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5370 // ...
5371 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5372 // break;
5373 // case 2:
5374 // ...
5375 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5376 // ...
5377 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5378 // break;
5379 // default:;
5380 // }
5381 //
5382 // if SimpleReduction is true, only the next code is generated:
5383 // ...
5384 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5385 // ...
5386
5387 ASTContext &C = CGM.getContext();
5388
5389 if (SimpleReduction) {
5390 CodeGenFunction::RunCleanupsScope Scope(CGF);
5391 const auto *IPriv = OrgPrivates.begin();
5392 const auto *ILHS = OrgLHSExprs.begin();
5393 const auto *IRHS = OrgRHSExprs.begin();
5394 for (const Expr *E : OrgReductionOps) {
5395 emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5396 RHS: cast<DeclRefExpr>(Val: *IRHS));
5397 ++IPriv;
5398 ++ILHS;
5399 ++IRHS;
5400 }
5401 return;
5402 }
5403
5404 // Filter out shared reduction variables based on IsPrivateVarReduction flag.
5405 // Only keep entries where the corresponding variable is not private.
5406 SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
5407 FilteredRHSExprs, FilteredReductionOps;
5408 for (unsigned I : llvm::seq<unsigned>(
5409 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5410 if (!Options.IsPrivateVarReduction[I]) {
5411 FilteredPrivates.emplace_back(Args: OrgPrivates[I]);
5412 FilteredLHSExprs.emplace_back(Args: OrgLHSExprs[I]);
5413 FilteredRHSExprs.emplace_back(Args: OrgRHSExprs[I]);
5414 FilteredReductionOps.emplace_back(Args: OrgReductionOps[I]);
5415 }
5416 }
5417 // Wrap filtered vectors in ArrayRef for downstream shared reduction
5418 // processing.
5419 ArrayRef<const Expr *> Privates = FilteredPrivates;
5420 ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
5421 ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
5422 ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
5423
5424 // 1. Build a list of reduction variables.
5425 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5426 auto Size = RHSExprs.size();
5427 for (const Expr *E : Privates) {
5428 if (E->getType()->isVariablyModifiedType())
5429 // Reserve place for array size.
5430 ++Size;
5431 }
5432 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5433 QualType ReductionArrayTy = C.getConstantArrayType(
5434 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5435 /*IndexTypeQuals=*/0);
5436 RawAddress ReductionList =
5437 CGF.CreateMemTemp(T: ReductionArrayTy, Name: ".omp.reduction.red_list");
5438 const auto *IPriv = Privates.begin();
5439 unsigned Idx = 0;
5440 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5441 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5442 CGF.Builder.CreateStore(
5443 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5444 V: CGF.EmitLValue(E: RHSExprs[I]).getPointer(CGF), DestTy: CGF.VoidPtrTy),
5445 Addr: Elem);
5446 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5447 // Store array size.
5448 ++Idx;
5449 Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5450 llvm::Value *Size = CGF.Builder.CreateIntCast(
5451 V: CGF.getVLASize(
5452 vla: CGF.getContext().getAsVariableArrayType(T: (*IPriv)->getType()))
5453 .NumElts,
5454 DestTy: CGF.SizeTy, /*isSigned=*/false);
5455 CGF.Builder.CreateStore(Val: CGF.Builder.CreateIntToPtr(V: Size, DestTy: CGF.VoidPtrTy),
5456 Addr: Elem);
5457 }
5458 }
5459
5460 // 2. Emit reduce_func().
5461 llvm::Function *ReductionFn = emitReductionFunction(
5462 ReducerName: CGF.CurFn->getName(), Loc, ArgsElemType: CGF.ConvertTypeForMem(T: ReductionArrayTy),
5463 Privates, LHSExprs, RHSExprs, ReductionOps);
5464
5465 // 3. Create static kmp_critical_name lock = { 0 };
5466 std::string Name = getName(Parts: {"reduction"});
5467 llvm::Value *Lock = getCriticalRegionLock(CriticalName: Name);
5468
5469 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5470 // RedList, reduce_func, &<lock>);
5471 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5472 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5473 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(Ty: ReductionArrayTy);
5474 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5475 V: ReductionList.getPointer(), DestTy: CGF.VoidPtrTy);
5476 llvm::Value *Args[] = {
5477 IdentTLoc, // ident_t *<loc>
5478 ThreadId, // i32 <gtid>
5479 CGF.Builder.getInt32(C: RHSExprs.size()), // i32 <n>
5480 ReductionArrayTySize, // size_type sizeof(RedList)
5481 RL, // void *RedList
5482 ReductionFn, // void (*) (void *, void *) <reduce_func>
5483 Lock // kmp_critical_name *&<lock>
5484 };
5485 llvm::Value *Res = CGF.EmitRuntimeCall(
5486 callee: OMPBuilder.getOrCreateRuntimeFunction(
5487 M&: CGM.getModule(),
5488 FnID: WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5489 args: Args);
5490
5491 // 5. Build switch(res)
5492 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(name: ".omp.reduction.default");
5493 llvm::SwitchInst *SwInst =
5494 CGF.Builder.CreateSwitch(V: Res, Dest: DefaultBB, /*NumCases=*/2);
5495
5496 // 6. Build case 1:
5497 // ...
5498 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5499 // ...
5500 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5501 // break;
5502 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(name: ".omp.reduction.case1");
5503 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 1), Dest: Case1BB);
5504 CGF.EmitBlock(BB: Case1BB);
5505
5506 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5507 llvm::Value *EndArgs[] = {
5508 IdentTLoc, // ident_t *<loc>
5509 ThreadId, // i32 <gtid>
5510 Lock // kmp_critical_name *&<lock>
5511 };
5512 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5513 CodeGenFunction &CGF, PrePostActionTy &Action) {
5514 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5515 const auto *IPriv = Privates.begin();
5516 const auto *ILHS = LHSExprs.begin();
5517 const auto *IRHS = RHSExprs.begin();
5518 for (const Expr *E : ReductionOps) {
5519 RT.emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5520 RHS: cast<DeclRefExpr>(Val: *IRHS));
5521 ++IPriv;
5522 ++ILHS;
5523 ++IRHS;
5524 }
5525 };
5526 RegionCodeGenTy RCG(CodeGen);
5527 CommonActionTy Action(
5528 nullptr, {},
5529 OMPBuilder.getOrCreateRuntimeFunction(
5530 M&: CGM.getModule(), FnID: WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5531 : OMPRTL___kmpc_end_reduce),
5532 EndArgs);
5533 RCG.setAction(Action);
5534 RCG(CGF);
5535
5536 CGF.EmitBranch(Block: DefaultBB);
5537
5538 // 7. Build case 2:
5539 // ...
5540 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5541 // ...
5542 // break;
5543 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(name: ".omp.reduction.case2");
5544 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 2), Dest: Case2BB);
5545 CGF.EmitBlock(BB: Case2BB);
5546
5547 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5548 CodeGenFunction &CGF, PrePostActionTy &Action) {
5549 const auto *ILHS = LHSExprs.begin();
5550 const auto *IRHS = RHSExprs.begin();
5551 const auto *IPriv = Privates.begin();
5552 for (const Expr *E : ReductionOps) {
5553 const Expr *XExpr = nullptr;
5554 const Expr *EExpr = nullptr;
5555 const Expr *UpExpr = nullptr;
5556 BinaryOperatorKind BO = BO_Comma;
5557 if (const auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
5558 if (BO->getOpcode() == BO_Assign) {
5559 XExpr = BO->getLHS();
5560 UpExpr = BO->getRHS();
5561 }
5562 }
5563 // Try to emit update expression as a simple atomic.
5564 const Expr *RHSExpr = UpExpr;
5565 if (RHSExpr) {
5566 // Analyze RHS part of the whole expression.
5567 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5568 Val: RHSExpr->IgnoreParenImpCasts())) {
5569 // If this is a conditional operator, analyze its condition for
5570 // min/max reduction operator.
5571 RHSExpr = ACO->getCond();
5572 }
5573 if (const auto *BORHS =
5574 dyn_cast<BinaryOperator>(Val: RHSExpr->IgnoreParenImpCasts())) {
5575 EExpr = BORHS->getRHS();
5576 BO = BORHS->getOpcode();
5577 }
5578 }
5579 if (XExpr) {
5580 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5581 auto &&AtomicRedGen = [BO, VD,
5582 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5583 const Expr *EExpr, const Expr *UpExpr) {
5584 LValue X = CGF.EmitLValue(E: XExpr);
5585 RValue E;
5586 if (EExpr)
5587 E = CGF.EmitAnyExpr(E: EExpr);
5588 CGF.EmitOMPAtomicSimpleUpdateExpr(
5589 X, E, BO, /*IsXLHSInRHSPart=*/true,
5590 AO: llvm::AtomicOrdering::Monotonic, Loc,
5591 CommonGen: [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5592 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5593 Address LHSTemp = CGF.CreateMemTemp(T: VD->getType());
5594 CGF.emitOMPSimpleStore(
5595 LVal: CGF.MakeAddrLValue(Addr: LHSTemp, T: VD->getType()), RVal: XRValue,
5596 RValTy: VD->getType().getNonReferenceType(), Loc);
5597 PrivateScope.addPrivate(LocalVD: VD, Addr: LHSTemp);
5598 (void)PrivateScope.Privatize();
5599 return CGF.EmitAnyExpr(E: UpExpr);
5600 });
5601 };
5602 if ((*IPriv)->getType()->isArrayType()) {
5603 // Emit atomic reduction for array section.
5604 const auto *RHSVar =
5605 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5606 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar: VD, RHSVar,
5607 RedOpGen: AtomicRedGen, XExpr, EExpr, UpExpr);
5608 } else {
5609 // Emit atomic reduction for array subscript or single variable.
5610 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5611 }
5612 } else {
5613 // Emit as a critical region.
5614 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5615 const Expr *, const Expr *) {
5616 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5617 std::string Name = RT.getName(Parts: {"atomic_reduction"});
5618 RT.emitCriticalRegion(
5619 CGF, CriticalName: Name,
5620 CriticalOpGen: [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5621 Action.Enter(CGF);
5622 emitReductionCombiner(CGF, ReductionOp: E);
5623 },
5624 Loc);
5625 };
5626 if ((*IPriv)->getType()->isArrayType()) {
5627 const auto *LHSVar =
5628 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5629 const auto *RHSVar =
5630 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5631 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5632 RedOpGen: CritRedGen);
5633 } else {
5634 CritRedGen(CGF, nullptr, nullptr, nullptr);
5635 }
5636 }
5637 ++ILHS;
5638 ++IRHS;
5639 ++IPriv;
5640 }
5641 };
5642 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5643 if (!WithNowait) {
5644 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5645 llvm::Value *EndArgs[] = {
5646 IdentTLoc, // ident_t *<loc>
5647 ThreadId, // i32 <gtid>
5648 Lock // kmp_critical_name *&<lock>
5649 };
5650 CommonActionTy Action(nullptr, {},
5651 OMPBuilder.getOrCreateRuntimeFunction(
5652 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_reduce),
5653 EndArgs);
5654 AtomicRCG.setAction(Action);
5655 AtomicRCG(CGF);
5656 } else {
5657 AtomicRCG(CGF);
5658 }
5659
5660 CGF.EmitBranch(Block: DefaultBB);
5661 CGF.EmitBlock(BB: DefaultBB, /*IsFinished=*/true);
5662 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5663 "PrivateVarReduction: Privates size mismatch");
5664 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5665 "PrivateVarReduction: ReductionOps size mismatch");
5666 for (unsigned I : llvm::seq<unsigned>(
5667 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5668 if (Options.IsPrivateVarReduction[I])
5669 emitPrivateReduction(CGF, Loc, Privates: OrgPrivates[I], LHSExprs: OrgLHSExprs[I],
5670 RHSExprs: OrgRHSExprs[I], ReductionOps: OrgReductionOps[I]);
5671 }
5672}
5673
5674/// Generates unique name for artificial threadprivate variables.
5675/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5676static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5677 const Expr *Ref) {
5678 SmallString<256> Buffer;
5679 llvm::raw_svector_ostream Out(Buffer);
5680 const clang::DeclRefExpr *DE;
5681 const VarDecl *D = ::getBaseDecl(Ref, DE);
5682 if (!D)
5683 D = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Ref)->getDecl());
5684 D = D->getCanonicalDecl();
5685 std::string Name = CGM.getOpenMPRuntime().getName(
5686 Parts: {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(GD: D)});
5687 Out << Prefix << Name << "_"
5688 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5689 return std::string(Out.str());
5690}
5691
5692/// Emits reduction initializer function:
5693/// \code
5694/// void @.red_init(void* %arg, void* %orig) {
5695/// %0 = bitcast void* %arg to <type>*
5696/// store <type> <init>, <type>* %0
5697/// ret void
5698/// }
5699/// \endcode
5700static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5701 SourceLocation Loc,
5702 ReductionCodeGen &RCG, unsigned N) {
5703 ASTContext &C = CGM.getContext();
5704 QualType VoidPtrTy = C.VoidPtrTy;
5705 VoidPtrTy.addRestrict();
5706 FunctionArgList Args;
5707 auto *Param =
5708 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5709 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5710 auto *ParamOrig =
5711 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5712 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5713 Args.emplace_back(Args&: Param);
5714 Args.emplace_back(Args&: ParamOrig);
5715 const auto &FnInfo =
5716 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5717 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5718 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_init", ""});
5719 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5720 N: Name, M: &CGM.getModule());
5721 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5722 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5723 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5724 Fn->setDoesNotRecurse();
5725 CodeGenFunction CGF(CGM);
5726 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5727 QualType PrivateType = RCG.getPrivateType(N);
5728 Address PrivateAddr = CGF.EmitLoadOfPointer(
5729 Ptr: CGF.GetAddrOfLocalVar(VD: Param).withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5730 PtrTy: C.getPointerType(T: PrivateType)->castAs<PointerType>());
5731 llvm::Value *Size = nullptr;
5732 // If the size of the reduction item is non-constant, load it from global
5733 // threadprivate variable.
5734 if (RCG.getSizes(N).second) {
5735 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5736 CGF, VarType: CGM.getContext().getSizeType(),
5737 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5738 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5739 Ty: CGM.getContext().getSizeType(), Loc);
5740 }
5741 RCG.emitAggregateType(CGF, N, Size);
5742 Address OrigAddr = Address::invalid();
5743 // If initializer uses initializer from declare reduction construct, emit a
5744 // pointer to the address of the original reduction item (reuired by reduction
5745 // initializer)
5746 if (RCG.usesReductionInitializer(N)) {
5747 Address SharedAddr = CGF.GetAddrOfLocalVar(VD: ParamOrig);
5748 OrigAddr = CGF.EmitLoadOfPointer(
5749 Ptr: SharedAddr,
5750 PtrTy: CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5751 }
5752 // Emit the initializer:
5753 // %0 = bitcast void* %arg to <type>*
5754 // store <type> <init>, <type>* %0
5755 RCG.emitInitialization(CGF, N, PrivateAddr, SharedAddr: OrigAddr,
5756 DefaultInit: [](CodeGenFunction &) { return false; });
5757 CGF.FinishFunction();
5758 return Fn;
5759}
5760
5761/// Emits reduction combiner function:
5762/// \code
5763/// void @.red_comb(void* %arg0, void* %arg1) {
5764/// %lhs = bitcast void* %arg0 to <type>*
5765/// %rhs = bitcast void* %arg1 to <type>*
5766/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5767/// store <type> %2, <type>* %lhs
5768/// ret void
5769/// }
5770/// \endcode
5771static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5772 SourceLocation Loc,
5773 ReductionCodeGen &RCG, unsigned N,
5774 const Expr *ReductionOp,
5775 const Expr *LHS, const Expr *RHS,
5776 const Expr *PrivateRef) {
5777 ASTContext &C = CGM.getContext();
5778 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHS)->getDecl());
5779 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHS)->getDecl());
5780 FunctionArgList Args;
5781 auto *ParamInOut =
5782 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5783 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5784 auto *ParamIn =
5785 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5786 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5787 Args.emplace_back(Args&: ParamInOut);
5788 Args.emplace_back(Args&: ParamIn);
5789 const auto &FnInfo =
5790 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5791 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5792 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_comb", ""});
5793 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5794 N: Name, M: &CGM.getModule());
5795 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5796 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5797 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5798 Fn->setDoesNotRecurse();
5799 CodeGenFunction CGF(CGM);
5800 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5801 llvm::Value *Size = nullptr;
5802 // If the size of the reduction item is non-constant, load it from global
5803 // threadprivate variable.
5804 if (RCG.getSizes(N).second) {
5805 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5806 CGF, VarType: CGM.getContext().getSizeType(),
5807 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5808 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5809 Ty: CGM.getContext().getSizeType(), Loc);
5810 }
5811 RCG.emitAggregateType(CGF, N, Size);
5812 // Remap lhs and rhs variables to the addresses of the function arguments.
5813 // %lhs = bitcast void* %arg0 to <type>*
5814 // %rhs = bitcast void* %arg1 to <type>*
5815 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5816 PrivateScope.addPrivate(
5817 LocalVD: LHSVD,
5818 // Pull out the pointer to the variable.
5819 Addr: CGF.EmitLoadOfPointer(
5820 Ptr: CGF.GetAddrOfLocalVar(VD: ParamInOut)
5821 .withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5822 PtrTy: C.getPointerType(T: LHSVD->getType())->castAs<PointerType>()));
5823 PrivateScope.addPrivate(
5824 LocalVD: RHSVD,
5825 // Pull out the pointer to the variable.
5826 Addr: CGF.EmitLoadOfPointer(
5827 Ptr: CGF.GetAddrOfLocalVar(VD: ParamIn).withElementType(
5828 ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5829 PtrTy: C.getPointerType(T: RHSVD->getType())->castAs<PointerType>()));
5830 PrivateScope.Privatize();
5831 // Emit the combiner body:
5832 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
5833 // store <type> %2, <type>* %lhs
5834 CGM.getOpenMPRuntime().emitSingleReductionCombiner(
5835 CGF, ReductionOp, PrivateRef, LHS: cast<DeclRefExpr>(Val: LHS),
5836 RHS: cast<DeclRefExpr>(Val: RHS));
5837 CGF.FinishFunction();
5838 return Fn;
5839}
5840
5841/// Emits reduction finalizer function:
5842/// \code
5843/// void @.red_fini(void* %arg) {
5844/// %0 = bitcast void* %arg to <type>*
5845/// <destroy>(<type>* %0)
5846/// ret void
5847/// }
5848/// \endcode
5849static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
5850 SourceLocation Loc,
5851 ReductionCodeGen &RCG, unsigned N) {
5852 if (!RCG.needCleanups(N))
5853 return nullptr;
5854 ASTContext &C = CGM.getContext();
5855 FunctionArgList Args;
5856 auto *Param =
5857 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5858 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5859 Args.emplace_back(Args&: Param);
5860 const auto &FnInfo =
5861 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5862 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5863 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_fini", ""});
5864 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5865 N: Name, M: &CGM.getModule());
5866 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5867 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5868 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5869 Fn->setDoesNotRecurse();
5870 CodeGenFunction CGF(CGM);
5871 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5872 Address PrivateAddr = CGF.EmitLoadOfPointer(
5873 Ptr: CGF.GetAddrOfLocalVar(VD: Param), PtrTy: C.VoidPtrTy.castAs<PointerType>());
5874 llvm::Value *Size = nullptr;
5875 // If the size of the reduction item is non-constant, load it from global
5876 // threadprivate variable.
5877 if (RCG.getSizes(N).second) {
5878 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5879 CGF, VarType: CGM.getContext().getSizeType(),
5880 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5881 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5882 Ty: CGM.getContext().getSizeType(), Loc);
5883 }
5884 RCG.emitAggregateType(CGF, N, Size);
5885 // Emit the finalizer body:
5886 // <destroy>(<type>* %0)
5887 RCG.emitCleanups(CGF, N, PrivateAddr);
5888 CGF.FinishFunction(EndLoc: Loc);
5889 return Fn;
5890}
5891
5892llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
5893 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
5894 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
5895 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
5896 return nullptr;
5897
5898 // Build typedef struct:
5899 // kmp_taskred_input {
5900 // void *reduce_shar; // shared reduction item
5901 // void *reduce_orig; // original reduction item used for initialization
5902 // size_t reduce_size; // size of data item
5903 // void *reduce_init; // data initialization routine
5904 // void *reduce_fini; // data finalization routine
5905 // void *reduce_comb; // data combiner routine
5906 // kmp_task_red_flags_t flags; // flags for additional info from compiler
5907 // } kmp_taskred_input_t;
5908 ASTContext &C = CGM.getContext();
5909 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_taskred_input_t");
5910 RD->startDefinition();
5911 const FieldDecl *SharedFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5912 const FieldDecl *OrigFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5913 const FieldDecl *SizeFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.getSizeType());
5914 const FieldDecl *InitFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5915 const FieldDecl *FiniFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5916 const FieldDecl *CombFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5917 const FieldDecl *FlagsFD = addFieldToRecordDecl(
5918 C, DC: RD, FieldTy: C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
5919 RD->completeDefinition();
5920 CanQualType RDType = C.getCanonicalTagType(TD: RD);
5921 unsigned Size = Data.ReductionVars.size();
5922 llvm::APInt ArraySize(/*numBits=*/64, Size);
5923 QualType ArrayRDType =
5924 C.getConstantArrayType(EltTy: RDType, ArySize: ArraySize, SizeExpr: nullptr,
5925 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
5926 // kmp_task_red_input_t .rd_input.[Size];
5927 RawAddress TaskRedInput = CGF.CreateMemTemp(T: ArrayRDType, Name: ".rd_input.");
5928 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
5929 Data.ReductionCopies, Data.ReductionOps);
5930 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5931 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
5932 llvm::Value *Idxs[] = {llvm::ConstantInt::get(Ty: CGM.SizeTy, /*V=*/0),
5933 llvm::ConstantInt::get(Ty: CGM.SizeTy, V: Cnt)};
5934 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
5935 ElemTy: TaskRedInput.getElementType(), Ptr: TaskRedInput.getPointer(), IdxList: Idxs,
5936 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
5937 Name: ".rd_input.gep.");
5938 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(V: GEP, T: RDType);
5939 // ElemLVal.reduce_shar = &Shareds[Cnt];
5940 LValue SharedLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SharedFD);
5941 RCG.emitSharedOrigLValue(CGF, N: Cnt);
5942 llvm::Value *Shared = RCG.getSharedLValue(N: Cnt).getPointer(CGF);
5943 CGF.EmitStoreOfScalar(value: Shared, lvalue: SharedLVal);
5944 // ElemLVal.reduce_orig = &Origs[Cnt];
5945 LValue OrigLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: OrigFD);
5946 llvm::Value *Orig = RCG.getOrigLValue(N: Cnt).getPointer(CGF);
5947 CGF.EmitStoreOfScalar(value: Orig, lvalue: OrigLVal);
5948 RCG.emitAggregateType(CGF, N: Cnt);
5949 llvm::Value *SizeValInChars;
5950 llvm::Value *SizeVal;
5951 std::tie(args&: SizeValInChars, args&: SizeVal) = RCG.getSizes(N: Cnt);
5952 // We use delayed creation/initialization for VLAs and array sections. It is
5953 // required because runtime does not provide the way to pass the sizes of
5954 // VLAs/array sections to initializer/combiner/finalizer functions. Instead
5955 // threadprivate global variables are used to store these values and use
5956 // them in the functions.
5957 bool DelayedCreation = !!SizeVal;
5958 SizeValInChars = CGF.Builder.CreateIntCast(V: SizeValInChars, DestTy: CGM.SizeTy,
5959 /*isSigned=*/false);
5960 LValue SizeLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SizeFD);
5961 CGF.EmitStoreOfScalar(value: SizeValInChars, lvalue: SizeLVal);
5962 // ElemLVal.reduce_init = init;
5963 LValue InitLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: InitFD);
5964 llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, N: Cnt);
5965 CGF.EmitStoreOfScalar(value: InitAddr, lvalue: InitLVal);
5966 // ElemLVal.reduce_fini = fini;
5967 LValue FiniLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FiniFD);
5968 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, N: Cnt);
5969 llvm::Value *FiniAddr =
5970 Fini ? Fini : llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
5971 CGF.EmitStoreOfScalar(value: FiniAddr, lvalue: FiniLVal);
5972 // ElemLVal.reduce_comb = comb;
5973 LValue CombLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: CombFD);
5974 llvm::Value *CombAddr = emitReduceCombFunction(
5975 CGM, Loc, RCG, N: Cnt, ReductionOp: Data.ReductionOps[Cnt], LHS: LHSExprs[Cnt],
5976 RHS: RHSExprs[Cnt], PrivateRef: Data.ReductionCopies[Cnt]);
5977 CGF.EmitStoreOfScalar(value: CombAddr, lvalue: CombLVal);
5978 // ElemLVal.flags = 0;
5979 LValue FlagsLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FlagsFD);
5980 if (DelayedCreation) {
5981 CGF.EmitStoreOfScalar(
5982 value: llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/1, /*isSigned=*/IsSigned: true),
5983 lvalue: FlagsLVal);
5984 } else
5985 CGF.EmitNullInitialization(DestPtr: FlagsLVal.getAddress(), Ty: FlagsLVal.getType());
5986 }
5987 if (Data.IsReductionWithTaskMod) {
5988 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
5989 // is_ws, int num, void *data);
5990 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
5991 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
5992 DestTy: CGM.IntTy, /*isSigned=*/true);
5993 llvm::Value *Args[] = {
5994 IdentTLoc, GTid,
5995 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Data.IsWorksharingReduction ? 1 : 0,
5996 /*isSigned=*/IsSigned: true),
5997 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
5998 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5999 V: TaskRedInput.getPointer(), DestTy: CGM.VoidPtrTy)};
6000 return CGF.EmitRuntimeCall(
6001 callee: OMPBuilder.getOrCreateRuntimeFunction(
6002 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_modifier_init),
6003 args: Args);
6004 }
6005 // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data);
6006 llvm::Value *Args[] = {
6007 CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc), DestTy: CGM.IntTy,
6008 /*isSigned=*/true),
6009 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6010 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: TaskRedInput.getPointer(),
6011 DestTy: CGM.VoidPtrTy)};
6012 return CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6013 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_init),
6014 args: Args);
6015}
6016
6017void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
6018 SourceLocation Loc,
6019 bool IsWorksharingReduction) {
6020 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6021 // is_ws, int num, void *data);
6022 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6023 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6024 DestTy: CGM.IntTy, /*isSigned=*/true);
6025 llvm::Value *Args[] = {IdentTLoc, GTid,
6026 llvm::ConstantInt::get(Ty: CGM.IntTy,
6027 V: IsWorksharingReduction ? 1 : 0,
6028 /*isSigned=*/IsSigned: true)};
6029 (void)CGF.EmitRuntimeCall(
6030 callee: OMPBuilder.getOrCreateRuntimeFunction(
6031 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_modifier_fini),
6032 args: Args);
6033}
6034
6035void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6036 SourceLocation Loc,
6037 ReductionCodeGen &RCG,
6038 unsigned N) {
6039 auto Sizes = RCG.getSizes(N);
6040 // Emit threadprivate global variable if the type is non-constant
6041 // (Sizes.second = nullptr).
6042 if (Sizes.second) {
6043 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(V: Sizes.second, DestTy: CGM.SizeTy,
6044 /*isSigned=*/false);
6045 Address SizeAddr = getAddrOfArtificialThreadPrivate(
6046 CGF, VarType: CGM.getContext().getSizeType(),
6047 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6048 CGF.Builder.CreateStore(Val: SizeVal, Addr: SizeAddr, /*IsVolatile=*/false);
6049 }
6050}
6051
6052Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6053 SourceLocation Loc,
6054 llvm::Value *ReductionsPtr,
6055 LValue SharedLVal) {
6056 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6057 // *d);
6058 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6059 DestTy: CGM.IntTy,
6060 /*isSigned=*/true),
6061 ReductionsPtr,
6062 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6063 V: SharedLVal.getPointer(CGF), DestTy: CGM.VoidPtrTy)};
6064 return Address(
6065 CGF.EmitRuntimeCall(
6066 callee: OMPBuilder.getOrCreateRuntimeFunction(
6067 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_get_th_data),
6068 args: Args),
6069 CGF.Int8Ty, SharedLVal.getAlignment());
6070}
6071
6072void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
6073 const OMPTaskDataTy &Data) {
6074 if (!CGF.HaveInsertPoint())
6075 return;
6076
6077 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
6078 // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder.
6079 OMPBuilder.createTaskwait(Loc: CGF.Builder);
6080 } else {
6081 llvm::Value *ThreadID = getThreadID(CGF, Loc);
6082 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
6083 auto &M = CGM.getModule();
6084 Address DependenciesArray = Address::invalid();
6085 llvm::Value *NumOfElements;
6086 std::tie(args&: NumOfElements, args&: DependenciesArray) =
6087 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
6088 if (!Data.Dependences.empty()) {
6089 llvm::Value *DepWaitTaskArgs[7];
6090 DepWaitTaskArgs[0] = UpLoc;
6091 DepWaitTaskArgs[1] = ThreadID;
6092 DepWaitTaskArgs[2] = NumOfElements;
6093 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
6094 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
6095 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6096 DepWaitTaskArgs[6] =
6097 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
6098
6099 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
6100
6101 // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
6102 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
6103 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
6104 // kmp_int32 has_no_wait); if dependence info is specified.
6105 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6106 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
6107 args: DepWaitTaskArgs);
6108
6109 } else {
6110
6111 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6112 // global_tid);
6113 llvm::Value *Args[] = {UpLoc, ThreadID};
6114 // Ignore return result until untied tasks are supported.
6115 CGF.EmitRuntimeCall(
6116 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_omp_taskwait),
6117 args: Args);
6118 }
6119 }
6120
6121 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
6122 Region->emitUntiedSwitch(CGF);
6123}
6124
6125void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6126 OpenMPDirectiveKind InnerKind,
6127 const RegionCodeGenTy &CodeGen,
6128 bool HasCancel) {
6129 if (!CGF.HaveInsertPoint())
6130 return;
6131 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
6132 InnerKind != OMPD_critical &&
6133 InnerKind != OMPD_master &&
6134 InnerKind != OMPD_masked);
6135 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6136}
6137
6138namespace {
6139enum RTCancelKind {
6140 CancelNoreq = 0,
6141 CancelParallel = 1,
6142 CancelLoop = 2,
6143 CancelSections = 3,
6144 CancelTaskgroup = 4
6145};
6146} // anonymous namespace
6147
6148static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6149 RTCancelKind CancelKind = CancelNoreq;
6150 if (CancelRegion == OMPD_parallel)
6151 CancelKind = CancelParallel;
6152 else if (CancelRegion == OMPD_for)
6153 CancelKind = CancelLoop;
6154 else if (CancelRegion == OMPD_sections)
6155 CancelKind = CancelSections;
6156 else {
6157 assert(CancelRegion == OMPD_taskgroup);
6158 CancelKind = CancelTaskgroup;
6159 }
6160 return CancelKind;
6161}
6162
6163void CGOpenMPRuntime::emitCancellationPointCall(
6164 CodeGenFunction &CGF, SourceLocation Loc,
6165 OpenMPDirectiveKind CancelRegion) {
6166 if (!CGF.HaveInsertPoint())
6167 return;
6168 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6169 // global_tid, kmp_int32 cncl_kind);
6170 if (auto *OMPRegionInfo =
6171 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6172 // For 'cancellation point taskgroup', the task region info may not have a
6173 // cancel. This may instead happen in another adjacent task.
6174 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6175 llvm::Value *Args[] = {
6176 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6177 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6178 // Ignore return result until untied tasks are supported.
6179 llvm::Value *Result = CGF.EmitRuntimeCall(
6180 callee: OMPBuilder.getOrCreateRuntimeFunction(
6181 M&: CGM.getModule(), FnID: OMPRTL___kmpc_cancellationpoint),
6182 args: Args);
6183 // if (__kmpc_cancellationpoint()) {
6184 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6185 // exit from construct;
6186 // }
6187 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6188 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6189 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6190 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6191 CGF.EmitBlock(BB: ExitBB);
6192 if (CancelRegion == OMPD_parallel)
6193 emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6194 // exit from construct;
6195 CodeGenFunction::JumpDest CancelDest =
6196 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6197 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6198 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6199 }
6200 }
6201}
6202
6203void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6204 const Expr *IfCond,
6205 OpenMPDirectiveKind CancelRegion) {
6206 if (!CGF.HaveInsertPoint())
6207 return;
6208 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6209 // kmp_int32 cncl_kind);
6210 auto &M = CGM.getModule();
6211 if (auto *OMPRegionInfo =
6212 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6213 auto &&ThenGen = [this, &M, Loc, CancelRegion,
6214 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
6215 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6216 llvm::Value *Args[] = {
6217 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6218 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6219 // Ignore return result until untied tasks are supported.
6220 llvm::Value *Result = CGF.EmitRuntimeCall(
6221 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_cancel), args: Args);
6222 // if (__kmpc_cancel()) {
6223 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6224 // exit from construct;
6225 // }
6226 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6227 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6228 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6229 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6230 CGF.EmitBlock(BB: ExitBB);
6231 if (CancelRegion == OMPD_parallel)
6232 RT.emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6233 // exit from construct;
6234 CodeGenFunction::JumpDest CancelDest =
6235 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6236 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6237 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6238 };
6239 if (IfCond) {
6240 emitIfClause(CGF, Cond: IfCond, ThenGen,
6241 ElseGen: [](CodeGenFunction &, PrePostActionTy &) {});
6242 } else {
6243 RegionCodeGenTy ThenRCG(ThenGen);
6244 ThenRCG(CGF);
6245 }
6246 }
6247}
6248
6249namespace {
6250/// Cleanup action for uses_allocators support.
6251class OMPUsesAllocatorsActionTy final : public PrePostActionTy {
6252 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators;
6253
6254public:
6255 OMPUsesAllocatorsActionTy(
6256 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
6257 : Allocators(Allocators) {}
6258 void Enter(CodeGenFunction &CGF) override {
6259 if (!CGF.HaveInsertPoint())
6260 return;
6261 for (const auto &AllocatorData : Allocators) {
6262 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(
6263 CGF, Allocator: AllocatorData.first, AllocatorTraits: AllocatorData.second);
6264 }
6265 }
6266 void Exit(CodeGenFunction &CGF) override {
6267 if (!CGF.HaveInsertPoint())
6268 return;
6269 for (const auto &AllocatorData : Allocators) {
6270 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,
6271 Allocator: AllocatorData.first);
6272 }
6273 }
6274};
6275} // namespace
6276
6277void CGOpenMPRuntime::emitTargetOutlinedFunction(
6278 const OMPExecutableDirective &D, StringRef ParentName,
6279 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6280 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6281 assert(!ParentName.empty() && "Invalid target entry parent name!");
6282 HasEmittedTargetRegion = true;
6283 SmallVector<std::pair<const Expr *, const Expr *>, 4> Allocators;
6284 for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) {
6285 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
6286 const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
6287 if (!D.AllocatorTraits)
6288 continue;
6289 Allocators.emplace_back(Args: D.Allocator, Args: D.AllocatorTraits);
6290 }
6291 }
6292 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
6293 CodeGen.setAction(UsesAllocatorAction);
6294 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6295 IsOffloadEntry, CodeGen);
6296}
6297
6298void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
6299 const Expr *Allocator,
6300 const Expr *AllocatorTraits) {
6301 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6302 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6303 // Use default memspace handle.
6304 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6305 llvm::Value *NumTraits = llvm::ConstantInt::get(
6306 Ty: CGF.IntTy, V: cast<ConstantArrayType>(
6307 Val: AllocatorTraits->getType()->getAsArrayTypeUnsafe())
6308 ->getSize()
6309 .getLimitedValue());
6310 LValue AllocatorTraitsLVal = CGF.EmitLValue(E: AllocatorTraits);
6311 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6312 Addr: AllocatorTraitsLVal.getAddress(), Ty: CGF.VoidPtrPtrTy, ElementTy: CGF.VoidPtrTy);
6313 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, T: CGF.getContext().VoidPtrTy,
6314 BaseInfo: AllocatorTraitsLVal.getBaseInfo(),
6315 TBAAInfo: AllocatorTraitsLVal.getTBAAInfo());
6316 llvm::Value *Traits = Addr.emitRawPointer(CGF);
6317
6318 llvm::Value *AllocatorVal =
6319 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6320 M&: CGM.getModule(), FnID: OMPRTL___kmpc_init_allocator),
6321 args: {ThreadId, MemSpaceHandle, NumTraits, Traits});
6322 // Store to allocator.
6323 CGF.EmitAutoVarAlloca(var: *cast<VarDecl>(
6324 Val: cast<DeclRefExpr>(Val: Allocator->IgnoreParenImpCasts())->getDecl()));
6325 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6326 AllocatorVal =
6327 CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: CGF.getContext().VoidPtrTy,
6328 DstTy: Allocator->getType(), Loc: Allocator->getExprLoc());
6329 CGF.EmitStoreOfScalar(value: AllocatorVal, lvalue: AllocatorLVal);
6330}
6331
6332void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
6333 const Expr *Allocator) {
6334 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6335 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6336 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6337 llvm::Value *AllocatorVal =
6338 CGF.EmitLoadOfScalar(lvalue: AllocatorLVal, Loc: Allocator->getExprLoc());
6339 AllocatorVal = CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: Allocator->getType(),
6340 DstTy: CGF.getContext().VoidPtrTy,
6341 Loc: Allocator->getExprLoc());
6342 (void)CGF.EmitRuntimeCall(
6343 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
6344 FnID: OMPRTL___kmpc_destroy_allocator),
6345 args: {ThreadId, AllocatorVal});
6346}
6347
6348void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams(
6349 const OMPExecutableDirective &D, CodeGenFunction &CGF,
6350 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6351 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6352 "invalid default attrs structure");
6353 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6354 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6355
6356 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: Attrs.MinTeams, MaxTeamsVal);
6357 getNumThreadsExprForTargetDirective(CGF, D, UpperBound&: MaxThreadsVal,
6358 /*UpperBoundOnly=*/true);
6359
6360 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6361 for (auto *A : C->getAttrs()) {
6362 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6363 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6364 if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(Val: A))
6365 CGM.handleCUDALaunchBoundsAttr(F: nullptr, A: Attr, MaxThreadsVal: &AttrMaxThreadsVal,
6366 MinBlocksVal: &AttrMinBlocksVal, MaxClusterRankVal: &AttrMaxBlocksVal);
6367 else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(Val: A))
6368 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6369 F: nullptr, A: Attr, /*ReqdWGS=*/nullptr, MinThreadsVal: &AttrMinThreadsVal,
6370 MaxThreadsVal: &AttrMaxThreadsVal);
6371 else
6372 continue;
6373
6374 Attrs.MinThreads = std::max(a: Attrs.MinThreads, b: AttrMinThreadsVal);
6375 if (AttrMaxThreadsVal > 0)
6376 MaxThreadsVal = MaxThreadsVal > 0
6377 ? std::min(a: MaxThreadsVal, b: AttrMaxThreadsVal)
6378 : AttrMaxThreadsVal;
6379 Attrs.MinTeams = std::max(a: Attrs.MinTeams, b: AttrMinBlocksVal);
6380 if (AttrMaxBlocksVal > 0)
6381 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(a: MaxTeamsVal, b: AttrMaxBlocksVal)
6382 : AttrMaxBlocksVal;
6383 }
6384 }
6385}
6386
6387void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6388 const OMPExecutableDirective &D, StringRef ParentName,
6389 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6390 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6391
6392 llvm::TargetRegionEntryInfo EntryInfo =
6393 getEntryInfoFromPresumedLoc(CGM, OMPBuilder, BeginLoc: D.getBeginLoc(), ParentName);
6394
6395 CodeGenFunction CGF(CGM, true);
6396 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6397 [&CGF, &D, &CodeGen, this](StringRef EntryFnName) {
6398 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
6399
6400 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6401 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6402 if (CGM.getLangOpts().OpenMPIsTargetDevice && !isGPU())
6403 return CGF.GenerateOpenMPCapturedStmtFunctionAggregate(S: CS, D);
6404 return CGF.GenerateOpenMPCapturedStmtFunction(S: CS, D);
6405 };
6406
6407 cantFail(Err: OMPBuilder.emitTargetRegionFunction(
6408 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6409 OutlinedFnID));
6410
6411 if (!OutlinedFn)
6412 return;
6413
6414 CGM.getTargetCodeGenInfo().setTargetAttributes(D: nullptr, GV: OutlinedFn, M&: CGM);
6415
6416 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6417 for (auto *A : C->getAttrs()) {
6418 if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(Val: A))
6419 CGM.handleAMDGPUWavesPerEUAttr(F: OutlinedFn, A: Attr);
6420 }
6421 }
6422 registerVTable(D);
6423}
6424
6425/// Checks if the expression is constant or does not have non-trivial function
6426/// calls.
6427static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6428 // We can skip constant expressions.
6429 // We can skip expressions with trivial calls or simple expressions.
6430 return (E->isEvaluatable(Ctx, AllowSideEffects: Expr::SE_AllowUndefinedBehavior) ||
6431 !E->hasNonTrivialCall(Ctx)) &&
6432 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6433}
6434
6435const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,
6436 const Stmt *Body) {
6437 const Stmt *Child = Body->IgnoreContainers();
6438 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Val: Child)) {
6439 Child = nullptr;
6440 for (const Stmt *S : C->body()) {
6441 if (const auto *E = dyn_cast<Expr>(Val: S)) {
6442 if (isTrivial(Ctx, E))
6443 continue;
6444 }
6445 // Some of the statements can be ignored.
6446 if (isa<AsmStmt>(Val: S) || isa<NullStmt>(Val: S) || isa<OMPFlushDirective>(Val: S) ||
6447 isa<OMPBarrierDirective>(Val: S) || isa<OMPTaskyieldDirective>(Val: S))
6448 continue;
6449 // Analyze declarations.
6450 if (const auto *DS = dyn_cast<DeclStmt>(Val: S)) {
6451 if (llvm::all_of(Range: DS->decls(), P: [](const Decl *D) {
6452 if (isa<EmptyDecl>(Val: D) || isa<DeclContext>(Val: D) ||
6453 isa<TypeDecl>(Val: D) || isa<PragmaCommentDecl>(Val: D) ||
6454 isa<PragmaDetectMismatchDecl>(Val: D) || isa<UsingDecl>(Val: D) ||
6455 isa<UsingDirectiveDecl>(Val: D) ||
6456 isa<OMPDeclareReductionDecl>(Val: D) ||
6457 isa<OMPThreadPrivateDecl>(Val: D) || isa<OMPAllocateDecl>(Val: D))
6458 return true;
6459 const auto *VD = dyn_cast<VarDecl>(Val: D);
6460 if (!VD)
6461 return false;
6462 return VD->hasGlobalStorage() || !VD->isUsed();
6463 }))
6464 continue;
6465 }
6466 // Found multiple children - cannot get the one child only.
6467 if (Child)
6468 return nullptr;
6469 Child = S;
6470 }
6471 if (Child)
6472 Child = Child->IgnoreContainers();
6473 }
6474 return Child;
6475}
6476
6477const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(
6478 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6479 int32_t &MaxTeamsVal) {
6480
6481 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6482 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6483 "Expected target-based executable directive.");
6484 switch (DirectiveKind) {
6485 case OMPD_target: {
6486 const auto *CS = D.getInnermostCapturedStmt();
6487 const auto *Body =
6488 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6489 const Stmt *ChildStmt =
6490 CGOpenMPRuntime::getSingleCompoundChild(Ctx&: CGF.getContext(), Body);
6491 if (const auto *NestedDir =
6492 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
6493 if (isOpenMPTeamsDirective(DKind: NestedDir->getDirectiveKind())) {
6494 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6495 const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>()
6496 ->getNumTeams()
6497 .front();
6498 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6499 if (auto Constant =
6500 NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6501 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6502 return NumTeams;
6503 }
6504 MinTeamsVal = MaxTeamsVal = 0;
6505 return nullptr;
6506 }
6507 MinTeamsVal = MaxTeamsVal = 1;
6508 return nullptr;
6509 }
6510 // A value of -1 is used to check if we need to emit no teams region
6511 MinTeamsVal = MaxTeamsVal = -1;
6512 return nullptr;
6513 }
6514 case OMPD_target_teams_loop:
6515 case OMPD_target_teams:
6516 case OMPD_target_teams_distribute:
6517 case OMPD_target_teams_distribute_simd:
6518 case OMPD_target_teams_distribute_parallel_for:
6519 case OMPD_target_teams_distribute_parallel_for_simd: {
6520 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6521 const Expr *NumTeams =
6522 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front();
6523 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6524 if (auto Constant = NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6525 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6526 return NumTeams;
6527 }
6528 MinTeamsVal = MaxTeamsVal = 0;
6529 return nullptr;
6530 }
6531 case OMPD_target_parallel:
6532 case OMPD_target_parallel_for:
6533 case OMPD_target_parallel_for_simd:
6534 case OMPD_target_parallel_loop:
6535 case OMPD_target_simd:
6536 MinTeamsVal = MaxTeamsVal = 1;
6537 return nullptr;
6538 case OMPD_parallel:
6539 case OMPD_for:
6540 case OMPD_parallel_for:
6541 case OMPD_parallel_loop:
6542 case OMPD_parallel_master:
6543 case OMPD_parallel_sections:
6544 case OMPD_for_simd:
6545 case OMPD_parallel_for_simd:
6546 case OMPD_cancel:
6547 case OMPD_cancellation_point:
6548 case OMPD_ordered:
6549 case OMPD_threadprivate:
6550 case OMPD_allocate:
6551 case OMPD_task:
6552 case OMPD_simd:
6553 case OMPD_tile:
6554 case OMPD_unroll:
6555 case OMPD_sections:
6556 case OMPD_section:
6557 case OMPD_single:
6558 case OMPD_master:
6559 case OMPD_critical:
6560 case OMPD_taskyield:
6561 case OMPD_barrier:
6562 case OMPD_taskwait:
6563 case OMPD_taskgroup:
6564 case OMPD_atomic:
6565 case OMPD_flush:
6566 case OMPD_depobj:
6567 case OMPD_scan:
6568 case OMPD_teams:
6569 case OMPD_target_data:
6570 case OMPD_target_exit_data:
6571 case OMPD_target_enter_data:
6572 case OMPD_distribute:
6573 case OMPD_distribute_simd:
6574 case OMPD_distribute_parallel_for:
6575 case OMPD_distribute_parallel_for_simd:
6576 case OMPD_teams_distribute:
6577 case OMPD_teams_distribute_simd:
6578 case OMPD_teams_distribute_parallel_for:
6579 case OMPD_teams_distribute_parallel_for_simd:
6580 case OMPD_target_update:
6581 case OMPD_declare_simd:
6582 case OMPD_declare_variant:
6583 case OMPD_begin_declare_variant:
6584 case OMPD_end_declare_variant:
6585 case OMPD_declare_target:
6586 case OMPD_end_declare_target:
6587 case OMPD_declare_reduction:
6588 case OMPD_declare_mapper:
6589 case OMPD_taskloop:
6590 case OMPD_taskloop_simd:
6591 case OMPD_master_taskloop:
6592 case OMPD_master_taskloop_simd:
6593 case OMPD_parallel_master_taskloop:
6594 case OMPD_parallel_master_taskloop_simd:
6595 case OMPD_requires:
6596 case OMPD_metadirective:
6597 case OMPD_unknown:
6598 break;
6599 default:
6600 break;
6601 }
6602 llvm_unreachable("Unexpected directive kind.");
6603}
6604
6605llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective(
6606 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6607 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6608 "Clauses associated with the teams directive expected to be emitted "
6609 "only for the host!");
6610 CGBuilderTy &Bld = CGF.Builder;
6611 int32_t MinNT = -1, MaxNT = -1;
6612 const Expr *NumTeams =
6613 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: MinNT, MaxTeamsVal&: MaxNT);
6614 if (NumTeams != nullptr) {
6615 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6616
6617 switch (DirectiveKind) {
6618 case OMPD_target: {
6619 const auto *CS = D.getInnermostCapturedStmt();
6620 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6621 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6622 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6623 /*IgnoreResultAssign*/ true);
6624 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6625 /*isSigned=*/true);
6626 }
6627 case OMPD_target_teams:
6628 case OMPD_target_teams_distribute:
6629 case OMPD_target_teams_distribute_simd:
6630 case OMPD_target_teams_distribute_parallel_for:
6631 case OMPD_target_teams_distribute_parallel_for_simd: {
6632 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6633 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6634 /*IgnoreResultAssign*/ true);
6635 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6636 /*isSigned=*/true);
6637 }
6638 default:
6639 break;
6640 }
6641 }
6642
6643 assert(MinNT == MaxNT && "Num threads ranges require handling here.");
6644 return llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: MinNT);
6645}
6646
6647/// Check for a num threads constant value (stored in \p DefaultVal), or
6648/// expression (stored in \p E). If the value is conditional (via an if-clause),
6649/// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are
6650/// nullptr, no expression evaluation is perfomed.
6651static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6652 const Expr **E, int32_t &UpperBound,
6653 bool UpperBoundOnly, llvm::Value **CondVal) {
6654 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6655 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6656 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6657 if (!Dir)
6658 return;
6659
6660 if (isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6661 // Handle if clause. If if clause present, the number of threads is
6662 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6663 if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) {
6664 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6665 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6666 const OMPIfClause *IfClause = nullptr;
6667 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6668 if (C->getNameModifier() == OMPD_unknown ||
6669 C->getNameModifier() == OMPD_parallel) {
6670 IfClause = C;
6671 break;
6672 }
6673 }
6674 if (IfClause) {
6675 const Expr *CondExpr = IfClause->getCondition();
6676 bool Result;
6677 if (CondExpr->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6678 if (!Result) {
6679 UpperBound = 1;
6680 return;
6681 }
6682 } else {
6683 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());
6684 if (const auto *PreInit =
6685 cast_or_null<DeclStmt>(Val: IfClause->getPreInitStmt())) {
6686 for (const auto *I : PreInit->decls()) {
6687 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6688 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6689 } else {
6690 CodeGenFunction::AutoVarEmission Emission =
6691 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6692 CGF.EmitAutoVarCleanups(emission: Emission);
6693 }
6694 }
6695 *CondVal = CGF.EvaluateExprAsBool(E: CondExpr);
6696 }
6697 }
6698 }
6699 }
6700 // Check the value of num_threads clause iff if clause was not specified
6701 // or is not evaluated to false.
6702 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6703 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6704 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6705 const auto *NumThreadsClause =
6706 Dir->getSingleClause<OMPNumThreadsClause>();
6707 const Expr *NTExpr = NumThreadsClause->getNumThreads();
6708 if (NTExpr->isIntegerConstantExpr(Ctx: CGF.getContext()))
6709 if (auto Constant = NTExpr->getIntegerConstantExpr(Ctx: CGF.getContext()))
6710 UpperBound =
6711 UpperBound
6712 ? Constant->getZExtValue()
6713 : std::min(a: UpperBound,
6714 b: static_cast<int32_t>(Constant->getZExtValue()));
6715 // If we haven't found a upper bound, remember we saw a thread limiting
6716 // clause.
6717 if (UpperBound == -1)
6718 UpperBound = 0;
6719 if (!E)
6720 return;
6721 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6722 if (const auto *PreInit =
6723 cast_or_null<DeclStmt>(Val: NumThreadsClause->getPreInitStmt())) {
6724 for (const auto *I : PreInit->decls()) {
6725 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6726 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6727 } else {
6728 CodeGenFunction::AutoVarEmission Emission =
6729 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6730 CGF.EmitAutoVarCleanups(emission: Emission);
6731 }
6732 }
6733 }
6734 *E = NTExpr;
6735 }
6736 return;
6737 }
6738 if (isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6739 UpperBound = 1;
6740}
6741
6742const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective(
6743 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6744 bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) {
6745 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6746 "Clauses associated with the teams directive expected to be emitted "
6747 "only for the host!");
6748 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6749 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6750 "Expected target-based executable directive.");
6751
6752 const Expr *NT = nullptr;
6753 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6754
6755 auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) {
6756 if (E->isIntegerConstantExpr(Ctx: CGF.getContext())) {
6757 if (auto Constant = E->getIntegerConstantExpr(Ctx: CGF.getContext()))
6758 UpperBound = UpperBound ? Constant->getZExtValue()
6759 : std::min(a: UpperBound,
6760 b: int32_t(Constant->getZExtValue()));
6761 }
6762 // If we haven't found a upper bound, remember we saw a thread limiting
6763 // clause.
6764 if (UpperBound == -1)
6765 UpperBound = 0;
6766 if (EPtr)
6767 *EPtr = E;
6768 };
6769
6770 auto ReturnSequential = [&]() {
6771 UpperBound = 1;
6772 return NT;
6773 };
6774
6775 switch (DirectiveKind) {
6776 case OMPD_target: {
6777 const CapturedStmt *CS = D.getInnermostCapturedStmt();
6778 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6779 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6780 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6781 // TODO: The standard is not clear how to resolve two thread limit clauses,
6782 // let's pick the teams one if it's present, otherwise the target one.
6783 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6784 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6785 if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) {
6786 ThreadLimitClause = TLC;
6787 if (ThreadLimitExpr) {
6788 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6789 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6790 CodeGenFunction::LexicalScope Scope(
6791 CGF,
6792 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
6793 if (const auto *PreInit =
6794 cast_or_null<DeclStmt>(Val: ThreadLimitClause->getPreInitStmt())) {
6795 for (const auto *I : PreInit->decls()) {
6796 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6797 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6798 } else {
6799 CodeGenFunction::AutoVarEmission Emission =
6800 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6801 CGF.EmitAutoVarCleanups(emission: Emission);
6802 }
6803 }
6804 }
6805 }
6806 }
6807 }
6808 if (ThreadLimitClause)
6809 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6810 ThreadLimitExpr);
6811 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6812 if (isOpenMPTeamsDirective(DKind: Dir->getDirectiveKind()) &&
6813 !isOpenMPDistributeDirective(DKind: Dir->getDirectiveKind())) {
6814 CS = Dir->getInnermostCapturedStmt();
6815 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6816 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6817 Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6818 }
6819 if (Dir && isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6820 CS = Dir->getInnermostCapturedStmt();
6821 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6822 } else if (Dir && isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6823 return ReturnSequential();
6824 }
6825 return NT;
6826 }
6827 case OMPD_target_teams: {
6828 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6829 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6830 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6831 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6832 ThreadLimitExpr);
6833 }
6834 const CapturedStmt *CS = D.getInnermostCapturedStmt();
6835 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6836 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6837 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6838 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6839 if (Dir->getDirectiveKind() == OMPD_distribute) {
6840 CS = Dir->getInnermostCapturedStmt();
6841 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6842 }
6843 }
6844 return NT;
6845 }
6846 case OMPD_target_teams_distribute:
6847 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6848 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6849 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6850 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6851 ThreadLimitExpr);
6852 }
6853 getNumThreads(CGF, CS: D.getInnermostCapturedStmt(), E: NTPtr, UpperBound,
6854 UpperBoundOnly, CondVal);
6855 return NT;
6856 case OMPD_target_teams_loop:
6857 case OMPD_target_parallel_loop:
6858 case OMPD_target_parallel:
6859 case OMPD_target_parallel_for:
6860 case OMPD_target_parallel_for_simd:
6861 case OMPD_target_teams_distribute_parallel_for:
6862 case OMPD_target_teams_distribute_parallel_for_simd: {
6863 if (CondVal && D.hasClausesOfKind<OMPIfClause>()) {
6864 const OMPIfClause *IfClause = nullptr;
6865 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
6866 if (C->getNameModifier() == OMPD_unknown ||
6867 C->getNameModifier() == OMPD_parallel) {
6868 IfClause = C;
6869 break;
6870 }
6871 }
6872 if (IfClause) {
6873 const Expr *Cond = IfClause->getCondition();
6874 bool Result;
6875 if (Cond->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6876 if (!Result)
6877 return ReturnSequential();
6878 } else {
6879 CodeGenFunction::RunCleanupsScope Scope(CGF);
6880 *CondVal = CGF.EvaluateExprAsBool(E: Cond);
6881 }
6882 }
6883 }
6884 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6885 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6886 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6887 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6888 ThreadLimitExpr);
6889 }
6890 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
6891 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6892 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
6893 CheckForConstExpr(NumThreadsClause->getNumThreads(), nullptr);
6894 return NumThreadsClause->getNumThreads();
6895 }
6896 return NT;
6897 }
6898 case OMPD_target_teams_distribute_simd:
6899 case OMPD_target_simd:
6900 return ReturnSequential();
6901 default:
6902 break;
6903 }
6904 llvm_unreachable("Unsupported directive kind.");
6905}
6906
6907llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective(
6908 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6909 llvm::Value *NumThreadsVal = nullptr;
6910 llvm::Value *CondVal = nullptr;
6911 llvm::Value *ThreadLimitVal = nullptr;
6912 const Expr *ThreadLimitExpr = nullptr;
6913 int32_t UpperBound = -1;
6914
6915 const Expr *NT = getNumThreadsExprForTargetDirective(
6916 CGF, D, UpperBound, /* UpperBoundOnly */ false, CondVal: &CondVal,
6917 ThreadLimitExpr: &ThreadLimitExpr);
6918
6919 // Thread limit expressions are used below, emit them.
6920 if (ThreadLimitExpr) {
6921 ThreadLimitVal =
6922 CGF.EmitScalarExpr(E: ThreadLimitExpr, /*IgnoreResultAssign=*/true);
6923 ThreadLimitVal = CGF.Builder.CreateIntCast(V: ThreadLimitVal, DestTy: CGF.Int32Ty,
6924 /*isSigned=*/false);
6925 }
6926
6927 // Generate the num teams expression.
6928 if (UpperBound == 1) {
6929 NumThreadsVal = CGF.Builder.getInt32(C: UpperBound);
6930 } else if (NT) {
6931 NumThreadsVal = CGF.EmitScalarExpr(E: NT, /*IgnoreResultAssign=*/true);
6932 NumThreadsVal = CGF.Builder.CreateIntCast(V: NumThreadsVal, DestTy: CGF.Int32Ty,
6933 /*isSigned=*/false);
6934 } else if (ThreadLimitVal) {
6935 // If we do not have a num threads value but a thread limit, replace the
6936 // former with the latter. We know handled the thread limit expression.
6937 NumThreadsVal = ThreadLimitVal;
6938 ThreadLimitVal = nullptr;
6939 } else {
6940 // Default to "0" which means runtime choice.
6941 assert(!ThreadLimitVal && "Default not applicable with thread limit value");
6942 NumThreadsVal = CGF.Builder.getInt32(C: 0);
6943 }
6944
6945 // Handle if clause. If if clause present, the number of threads is
6946 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6947 if (CondVal) {
6948 CodeGenFunction::RunCleanupsScope Scope(CGF);
6949 NumThreadsVal = CGF.Builder.CreateSelect(C: CondVal, True: NumThreadsVal,
6950 False: CGF.Builder.getInt32(C: 1));
6951 }
6952
6953 // If the thread limit and num teams expression were present, take the
6954 // minimum.
6955 if (ThreadLimitVal) {
6956 NumThreadsVal = CGF.Builder.CreateSelect(
6957 C: CGF.Builder.CreateICmpULT(LHS: ThreadLimitVal, RHS: NumThreadsVal),
6958 True: ThreadLimitVal, False: NumThreadsVal);
6959 }
6960
6961 return NumThreadsVal;
6962}
6963
6964namespace {
6965LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
6966
6967// Utility to handle information from clauses associated with a given
6968// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
6969// It provides a convenient interface to obtain the information and generate
6970// code for that information.
6971class MappableExprsHandler {
6972public:
6973 /// Custom comparator for attach-pointer expressions that compares them by
6974 /// complexity (i.e. their component-depth) first, then by the order in which
6975 /// they were computed by collectAttachPtrExprInfo(), if they are semantically
6976 /// different.
6977 struct AttachPtrExprComparator {
6978 const MappableExprsHandler &Handler;
6979 // Cache of previous equality comparison results.
6980 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool>
6981 CachedEqualityComparisons;
6982
6983 AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {}
6984 AttachPtrExprComparator() = delete;
6985
6986 // Return true iff LHS is "less than" RHS.
6987 bool operator()(const Expr *LHS, const Expr *RHS) const {
6988 if (LHS == RHS)
6989 return false;
6990
6991 // First, compare by complexity (depth)
6992 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(Val: LHS);
6993 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(Val: RHS);
6994
6995 std::optional<size_t> DepthLHS =
6996 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
6997 : std::nullopt;
6998 std::optional<size_t> DepthRHS =
6999 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
7000 : std::nullopt;
7001
7002 // std::nullopt (no attach pointer) has lowest complexity
7003 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
7004 // Both have same complexity, now check semantic equality
7005 if (areEqual(LHS, RHS))
7006 return false;
7007 // Different semantically, compare by computation order
7008 return wasComputedBefore(LHS, RHS);
7009 }
7010 if (!DepthLHS.has_value())
7011 return true; // LHS has lower complexity
7012 if (!DepthRHS.has_value())
7013 return false; // RHS has lower complexity
7014
7015 // Both have values, compare by depth (lower depth = lower complexity)
7016 if (DepthLHS.value() != DepthRHS.value())
7017 return DepthLHS.value() < DepthRHS.value();
7018
7019 // Same complexity, now check semantic equality
7020 if (areEqual(LHS, RHS))
7021 return false;
7022 // Different semantically, compare by computation order
7023 return wasComputedBefore(LHS, RHS);
7024 }
7025
7026 public:
7027 /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached
7028 /// results, if available, otherwise does a recursive semantic comparison.
7029 bool areEqual(const Expr *LHS, const Expr *RHS) const {
7030 // Check cache first for faster lookup
7031 const auto CachedResultIt = CachedEqualityComparisons.find(Val: {LHS, RHS});
7032 if (CachedResultIt != CachedEqualityComparisons.end())
7033 return CachedResultIt->second;
7034
7035 bool ComparisonResult = areSemanticallyEqual(LHS, RHS);
7036
7037 // Cache the result for future lookups (both orders since semantic
7038 // equality is commutative)
7039 CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult;
7040 CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult;
7041 return ComparisonResult;
7042 }
7043
7044 /// Compare the two attach-ptr expressions by their computation order.
7045 /// Returns true iff LHS was computed before RHS by
7046 /// collectAttachPtrExprInfo().
7047 bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const {
7048 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(Val: LHS);
7049 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(Val: RHS);
7050
7051 return OrderLHS < OrderRHS;
7052 }
7053
7054 private:
7055 /// Helper function to compare attach-pointer expressions semantically.
7056 /// This function handles various expression types that can be part of an
7057 /// attach-pointer.
7058 /// TODO: Not urgent, but we should ideally return true when comparing
7059 /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc.
7060 bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const {
7061 if (LHS == RHS)
7062 return true;
7063
7064 // If only one is null, they aren't equal
7065 if (!LHS || !RHS)
7066 return false;
7067
7068 ASTContext &Ctx = Handler.CGF.getContext();
7069 // Strip away parentheses and no-op casts to get to the core expression
7070 LHS = LHS->IgnoreParenNoopCasts(Ctx);
7071 RHS = RHS->IgnoreParenNoopCasts(Ctx);
7072
7073 // Direct pointer comparison of the underlying expressions
7074 if (LHS == RHS)
7075 return true;
7076
7077 // Check if the expression classes match
7078 if (LHS->getStmtClass() != RHS->getStmtClass())
7079 return false;
7080
7081 // Handle DeclRefExpr (variable references)
7082 if (const auto *LD = dyn_cast<DeclRefExpr>(Val: LHS)) {
7083 const auto *RD = dyn_cast<DeclRefExpr>(Val: RHS);
7084 if (!RD)
7085 return false;
7086 return LD->getDecl()->getCanonicalDecl() ==
7087 RD->getDecl()->getCanonicalDecl();
7088 }
7089
7090 // Handle ArraySubscriptExpr (array indexing like a[i])
7091 if (const auto *LA = dyn_cast<ArraySubscriptExpr>(Val: LHS)) {
7092 const auto *RA = dyn_cast<ArraySubscriptExpr>(Val: RHS);
7093 if (!RA)
7094 return false;
7095 return areSemanticallyEqual(LHS: LA->getBase(), RHS: RA->getBase()) &&
7096 areSemanticallyEqual(LHS: LA->getIdx(), RHS: RA->getIdx());
7097 }
7098
7099 // Handle MemberExpr (member access like s.m or p->m)
7100 if (const auto *LM = dyn_cast<MemberExpr>(Val: LHS)) {
7101 const auto *RM = dyn_cast<MemberExpr>(Val: RHS);
7102 if (!RM)
7103 return false;
7104 if (LM->getMemberDecl()->getCanonicalDecl() !=
7105 RM->getMemberDecl()->getCanonicalDecl())
7106 return false;
7107 return areSemanticallyEqual(LHS: LM->getBase(), RHS: RM->getBase());
7108 }
7109
7110 // Handle UnaryOperator (unary operations like *p, &x, etc.)
7111 if (const auto *LU = dyn_cast<UnaryOperator>(Val: LHS)) {
7112 const auto *RU = dyn_cast<UnaryOperator>(Val: RHS);
7113 if (!RU)
7114 return false;
7115 if (LU->getOpcode() != RU->getOpcode())
7116 return false;
7117 return areSemanticallyEqual(LHS: LU->getSubExpr(), RHS: RU->getSubExpr());
7118 }
7119
7120 // Handle BinaryOperator (binary operations like p + offset)
7121 if (const auto *LB = dyn_cast<BinaryOperator>(Val: LHS)) {
7122 const auto *RB = dyn_cast<BinaryOperator>(Val: RHS);
7123 if (!RB)
7124 return false;
7125 if (LB->getOpcode() != RB->getOpcode())
7126 return false;
7127 return areSemanticallyEqual(LHS: LB->getLHS(), RHS: RB->getLHS()) &&
7128 areSemanticallyEqual(LHS: LB->getRHS(), RHS: RB->getRHS());
7129 }
7130
7131 // Handle ArraySectionExpr (array sections like a[0:1])
7132 // Attach pointers should not contain array-sections, but currently we
7133 // don't emit an error.
7134 if (const auto *LAS = dyn_cast<ArraySectionExpr>(Val: LHS)) {
7135 const auto *RAS = dyn_cast<ArraySectionExpr>(Val: RHS);
7136 if (!RAS)
7137 return false;
7138 return areSemanticallyEqual(LHS: LAS->getBase(), RHS: RAS->getBase()) &&
7139 areSemanticallyEqual(LHS: LAS->getLowerBound(),
7140 RHS: RAS->getLowerBound()) &&
7141 areSemanticallyEqual(LHS: LAS->getLength(), RHS: RAS->getLength());
7142 }
7143
7144 // Handle CastExpr (explicit casts)
7145 if (const auto *LC = dyn_cast<CastExpr>(Val: LHS)) {
7146 const auto *RC = dyn_cast<CastExpr>(Val: RHS);
7147 if (!RC)
7148 return false;
7149 if (LC->getCastKind() != RC->getCastKind())
7150 return false;
7151 return areSemanticallyEqual(LHS: LC->getSubExpr(), RHS: RC->getSubExpr());
7152 }
7153
7154 // Handle CXXThisExpr (this pointer)
7155 if (isa<CXXThisExpr>(Val: LHS) && isa<CXXThisExpr>(Val: RHS))
7156 return true;
7157
7158 // Handle IntegerLiteral (integer constants)
7159 if (const auto *LI = dyn_cast<IntegerLiteral>(Val: LHS)) {
7160 const auto *RI = dyn_cast<IntegerLiteral>(Val: RHS);
7161 if (!RI)
7162 return false;
7163 return LI->getValue() == RI->getValue();
7164 }
7165
7166 // Handle CharacterLiteral (character constants)
7167 if (const auto *LC = dyn_cast<CharacterLiteral>(Val: LHS)) {
7168 const auto *RC = dyn_cast<CharacterLiteral>(Val: RHS);
7169 if (!RC)
7170 return false;
7171 return LC->getValue() == RC->getValue();
7172 }
7173
7174 // Handle FloatingLiteral (floating point constants)
7175 if (const auto *LF = dyn_cast<FloatingLiteral>(Val: LHS)) {
7176 const auto *RF = dyn_cast<FloatingLiteral>(Val: RHS);
7177 if (!RF)
7178 return false;
7179 // Use bitwise comparison for floating point literals
7180 return LF->getValue().bitwiseIsEqual(RHS: RF->getValue());
7181 }
7182
7183 // Handle StringLiteral (string constants)
7184 if (const auto *LS = dyn_cast<StringLiteral>(Val: LHS)) {
7185 const auto *RS = dyn_cast<StringLiteral>(Val: RHS);
7186 if (!RS)
7187 return false;
7188 return LS->getString() == RS->getString();
7189 }
7190
7191 // Handle CXXNullPtrLiteralExpr (nullptr)
7192 if (isa<CXXNullPtrLiteralExpr>(Val: LHS) && isa<CXXNullPtrLiteralExpr>(Val: RHS))
7193 return true;
7194
7195 // Handle CXXBoolLiteralExpr (true/false)
7196 if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(Val: LHS)) {
7197 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(Val: RHS);
7198 if (!RB)
7199 return false;
7200 return LB->getValue() == RB->getValue();
7201 }
7202
7203 // Fallback for other forms - use the existing comparison method
7204 return Expr::isSameComparisonOperand(E1: LHS, E2: RHS);
7205 }
7206 };
7207
7208 /// Get the offset of the OMP_MAP_MEMBER_OF field.
7209 static unsigned getFlagMemberOffset() {
7210 unsigned Offset = 0;
7211 for (uint64_t Remain =
7212 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
7213 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7214 !(Remain & 1); Remain = Remain >> 1)
7215 Offset++;
7216 return Offset;
7217 }
7218
7219 /// Class that holds debugging information for a data mapping to be passed to
7220 /// the runtime library.
7221 class MappingExprInfo {
7222 /// The variable declaration used for the data mapping.
7223 const ValueDecl *MapDecl = nullptr;
7224 /// The original expression used in the map clause, or null if there is
7225 /// none.
7226 const Expr *MapExpr = nullptr;
7227
7228 public:
7229 MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr)
7230 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7231
7232 const ValueDecl *getMapDecl() const { return MapDecl; }
7233 const Expr *getMapExpr() const { return MapExpr; }
7234 };
7235
7236 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7237 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7238 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7239 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7240 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7241 using MapNonContiguousArrayTy =
7242 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7243 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7244 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7245 using MapData =
7246 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7247 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,
7248 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;
7249 using MapDataArrayTy = SmallVector<MapData, 4>;
7250
7251 /// This structure contains combined information generated for mappable
7252 /// clauses, including base pointers, pointers, sizes, map types, user-defined
7253 /// mappers, and non-contiguous information.
7254 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7255 MapExprsArrayTy Exprs;
7256 MapValueDeclsArrayTy Mappers;
7257 MapValueDeclsArrayTy DevicePtrDecls;
7258
7259 /// Append arrays in \a CurInfo.
7260 void append(MapCombinedInfoTy &CurInfo) {
7261 Exprs.append(in_start: CurInfo.Exprs.begin(), in_end: CurInfo.Exprs.end());
7262 DevicePtrDecls.append(in_start: CurInfo.DevicePtrDecls.begin(),
7263 in_end: CurInfo.DevicePtrDecls.end());
7264 Mappers.append(in_start: CurInfo.Mappers.begin(), in_end: CurInfo.Mappers.end());
7265 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7266 }
7267 };
7268
7269 /// Map between a struct and the its lowest & highest elements which have been
7270 /// mapped.
7271 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7272 /// HE(FieldIndex, Pointer)}
7273 struct StructRangeInfoTy {
7274 MapCombinedInfoTy PreliminaryMapData;
7275 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7276 0, Address::invalid()};
7277 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7278 0, Address::invalid()};
7279 Address Base = Address::invalid();
7280 Address LB = Address::invalid();
7281 bool IsArraySection = false;
7282 bool HasCompleteRecord = false;
7283 };
7284
7285 /// A struct to store the attach pointer and pointee information, to be used
7286 /// when emitting an attach entry.
7287 struct AttachInfoTy {
7288 Address AttachPtrAddr = Address::invalid();
7289 Address AttachPteeAddr = Address::invalid();
7290 const ValueDecl *AttachPtrDecl = nullptr;
7291 const Expr *AttachMapExpr = nullptr;
7292
7293 bool isValid() const {
7294 return AttachPtrAddr.isValid() && AttachPteeAddr.isValid();
7295 }
7296 };
7297
7298 /// Check if there's any component list where the attach pointer expression
7299 /// matches the given captured variable.
7300 bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const {
7301 for (const auto &AttachEntry : AttachPtrExprMap) {
7302 if (AttachEntry.second) {
7303 // Check if the attach pointer expression is a DeclRefExpr that
7304 // references the captured variable
7305 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: AttachEntry.second))
7306 if (DRE->getDecl() == VD)
7307 return true;
7308 }
7309 }
7310 return false;
7311 }
7312
7313 /// Get the previously-cached attach pointer for a component list, if-any.
7314 const Expr *getAttachPtrExpr(
7315 OMPClauseMappableExprCommon::MappableExprComponentListRef Components)
7316 const {
7317 const auto It = AttachPtrExprMap.find(Val: Components);
7318 if (It != AttachPtrExprMap.end())
7319 return It->second;
7320
7321 return nullptr;
7322 }
7323
7324private:
7325 /// Kind that defines how a device pointer has to be returned.
7326 struct MapInfo {
7327 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7328 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
7329 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7330 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7331 bool ReturnDevicePointer = false;
7332 bool IsImplicit = false;
7333 const ValueDecl *Mapper = nullptr;
7334 const Expr *VarRef = nullptr;
7335 bool ForDeviceAddr = false;
7336 bool HasUdpFbNullify = false;
7337
7338 MapInfo() = default;
7339 MapInfo(
7340 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7341 OpenMPMapClauseKind MapType,
7342 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7343 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7344 bool ReturnDevicePointer, bool IsImplicit,
7345 const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr,
7346 bool ForDeviceAddr = false, bool HasUdpFbNullify = false)
7347 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7348 MotionModifiers(MotionModifiers),
7349 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7350 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr),
7351 HasUdpFbNullify(HasUdpFbNullify) {}
7352 };
7353
7354 /// The target directive from where the mappable clauses were extracted. It
7355 /// is either a executable directive or a user-defined mapper directive.
7356 llvm::PointerUnion<const OMPExecutableDirective *,
7357 const OMPDeclareMapperDecl *>
7358 CurDir;
7359
7360 /// Function the directive is being generated for.
7361 CodeGenFunction &CGF;
7362
7363 /// Set of all first private variables in the current directive.
7364 /// bool data is set to true if the variable is implicitly marked as
7365 /// firstprivate, false otherwise.
7366 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7367
7368 /// Set of defaultmap clause kinds that use firstprivate behavior.
7369 llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
7370
7371 /// Map between device pointer declarations and their expression components.
7372 /// The key value for declarations in 'this' is null.
7373 llvm::DenseMap<
7374 const ValueDecl *,
7375 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7376 DevPointersMap;
7377
7378 /// Map between device addr declarations and their expression components.
7379 /// The key value for declarations in 'this' is null.
7380 llvm::DenseMap<
7381 const ValueDecl *,
7382 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7383 HasDevAddrsMap;
7384
7385 /// Map between lambda declarations and their map type.
7386 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7387
7388 /// Map from component lists to their attach pointer expressions.
7389 llvm::DenseMap<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7390 const Expr *>
7391 AttachPtrExprMap;
7392
7393 /// Map from attach pointer expressions to their component depth.
7394 /// nullptr key has std::nullopt depth. This can be used to order attach-ptr
7395 /// expressions with increasing/decreasing depth.
7396 /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`.
7397 /// TODO: Not urgent, but we should ideally use the number of pointer
7398 /// dereferences in an expr as an indicator of its complexity, instead of the
7399 /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`,
7400 /// `*(p + 5 + 5)` together.
7401 llvm::DenseMap<const Expr *, std::optional<size_t>>
7402 AttachPtrComponentDepthMap = {{nullptr, std::nullopt}};
7403
7404 /// Map from attach pointer expressions to the order they were computed in, in
7405 /// collectAttachPtrExprInfo().
7406 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7407 {nullptr, 0}};
7408
7409 /// An instance of attach-ptr-expr comparator that can be used throughout the
7410 /// lifetime of this handler.
7411 AttachPtrExprComparator AttachPtrComparator;
7412
7413 llvm::Value *getExprTypeSize(const Expr *E) const {
7414 QualType ExprTy = E->getType().getCanonicalType();
7415
7416 // Calculate the size for array shaping expression.
7417 if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(Val: E)) {
7418 llvm::Value *Size =
7419 CGF.getTypeSize(Ty: OAE->getBase()->getType()->getPointeeType());
7420 for (const Expr *SE : OAE->getDimensions()) {
7421 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
7422 Sz = CGF.EmitScalarConversion(Src: Sz, SrcTy: SE->getType(),
7423 DstTy: CGF.getContext().getSizeType(),
7424 Loc: SE->getExprLoc());
7425 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: Sz);
7426 }
7427 return Size;
7428 }
7429
7430 // Reference types are ignored for mapping purposes.
7431 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7432 ExprTy = RefTy->getPointeeType().getCanonicalType();
7433
7434 // Given that an array section is considered a built-in type, we need to
7435 // do the calculation based on the length of the section instead of relying
7436 // on CGF.getTypeSize(E->getType()).
7437 if (const auto *OAE = dyn_cast<ArraySectionExpr>(Val: E)) {
7438 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(
7439 Base: OAE->getBase()->IgnoreParenImpCasts())
7440 .getCanonicalType();
7441
7442 // If there is no length associated with the expression and lower bound is
7443 // not specified too, that means we are using the whole length of the
7444 // base.
7445 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7446 !OAE->getLowerBound())
7447 return CGF.getTypeSize(Ty: BaseTy);
7448
7449 llvm::Value *ElemSize;
7450 if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7451 ElemSize = CGF.getTypeSize(Ty: PTy->getPointeeType().getCanonicalType());
7452 } else {
7453 const auto *ATy = cast<ArrayType>(Val: BaseTy.getTypePtr());
7454 assert(ATy && "Expecting array type if not a pointer type.");
7455 ElemSize = CGF.getTypeSize(Ty: ATy->getElementType().getCanonicalType());
7456 }
7457
7458 // If we don't have a length at this point, that is because we have an
7459 // array section with a single element.
7460 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7461 return ElemSize;
7462
7463 if (const Expr *LenExpr = OAE->getLength()) {
7464 llvm::Value *LengthVal = CGF.EmitScalarExpr(E: LenExpr);
7465 LengthVal = CGF.EmitScalarConversion(Src: LengthVal, SrcTy: LenExpr->getType(),
7466 DstTy: CGF.getContext().getSizeType(),
7467 Loc: LenExpr->getExprLoc());
7468 return CGF.Builder.CreateNUWMul(LHS: LengthVal, RHS: ElemSize);
7469 }
7470 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7471 OAE->getLowerBound() && "expected array_section[lb:].");
7472 // Size = sizetype - lb * elemtype;
7473 llvm::Value *LengthVal = CGF.getTypeSize(Ty: BaseTy);
7474 llvm::Value *LBVal = CGF.EmitScalarExpr(E: OAE->getLowerBound());
7475 LBVal = CGF.EmitScalarConversion(Src: LBVal, SrcTy: OAE->getLowerBound()->getType(),
7476 DstTy: CGF.getContext().getSizeType(),
7477 Loc: OAE->getLowerBound()->getExprLoc());
7478 LBVal = CGF.Builder.CreateNUWMul(LHS: LBVal, RHS: ElemSize);
7479 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LHS: LengthVal, RHS: LBVal);
7480 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LHS: LengthVal, RHS: LBVal);
7481 LengthVal = CGF.Builder.CreateSelect(
7482 C: Cmp, True: TrueVal, False: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0));
7483 return LengthVal;
7484 }
7485 return CGF.getTypeSize(Ty: ExprTy);
7486 }
7487
7488 /// Return the corresponding bits for a given map clause modifier. Add
7489 /// a flag marking the map as a pointer if requested. Add a flag marking the
7490 /// map as the first one of a series of maps that relate to the same map
7491 /// expression.
7492 OpenMPOffloadMappingFlags getMapTypeBits(
7493 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7494 ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,
7495 bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const {
7496 OpenMPOffloadMappingFlags Bits =
7497 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7498 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7499 switch (MapType) {
7500 case OMPC_MAP_alloc:
7501 case OMPC_MAP_release:
7502 // alloc and release is the default behavior in the runtime library, i.e.
7503 // if we don't pass any bits alloc/release that is what the runtime is
7504 // going to do. Therefore, we don't need to signal anything for these two
7505 // type modifiers.
7506 break;
7507 case OMPC_MAP_to:
7508 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7509 break;
7510 case OMPC_MAP_from:
7511 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7512 break;
7513 case OMPC_MAP_tofrom:
7514 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7515 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7516 break;
7517 case OMPC_MAP_delete:
7518 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7519 break;
7520 case OMPC_MAP_unknown:
7521 llvm_unreachable("Unexpected map type!");
7522 }
7523 if (AddPtrFlag)
7524 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7525 if (AddIsTargetParamFlag)
7526 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7527 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_always))
7528 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7529 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_close))
7530 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7531 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_present) ||
7532 llvm::is_contained(Range&: MotionModifiers, Element: OMPC_MOTION_MODIFIER_present))
7533 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7534 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_ompx_hold))
7535 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7536 if (IsNonContiguous)
7537 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7538 return Bits;
7539 }
7540
7541 /// Return true if the provided expression is a final array section. A
7542 /// final array section, is one whose length can't be proved to be one.
7543 bool isFinalArraySectionExpression(const Expr *E) const {
7544 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E);
7545
7546 // It is not an array section and therefore not a unity-size one.
7547 if (!OASE)
7548 return false;
7549
7550 // An array section with no colon always refer to a single element.
7551 if (OASE->getColonLocFirst().isInvalid())
7552 return false;
7553
7554 const Expr *Length = OASE->getLength();
7555
7556 // If we don't have a length we have to check if the array has size 1
7557 // for this dimension. Also, we should always expect a length if the
7558 // base type is pointer.
7559 if (!Length) {
7560 QualType BaseQTy = ArraySectionExpr::getBaseOriginalType(
7561 Base: OASE->getBase()->IgnoreParenImpCasts())
7562 .getCanonicalType();
7563 if (const auto *ATy = dyn_cast<ConstantArrayType>(Val: BaseQTy.getTypePtr()))
7564 return ATy->getSExtSize() != 1;
7565 // If we don't have a constant dimension length, we have to consider
7566 // the current section as having any size, so it is not necessarily
7567 // unitary. If it happen to be unity size, that's user fault.
7568 return true;
7569 }
7570
7571 // Check if the length evaluates to 1.
7572 Expr::EvalResult Result;
7573 if (!Length->EvaluateAsInt(Result, Ctx: CGF.getContext()))
7574 return true; // Can have more that size 1.
7575
7576 llvm::APSInt ConstLength = Result.Val.getInt();
7577 return ConstLength.getSExtValue() != 1;
7578 }
7579
7580 /// Emit an attach entry into \p CombinedInfo, using the information from \p
7581 /// AttachInfo. For example, for a map of form `int *p; ... map(p[1:10])`,
7582 /// an attach entry has the following form:
7583 /// &p, &p[1], sizeof(void*), ATTACH
7584 void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo,
7585 const AttachInfoTy &AttachInfo) const {
7586 assert(AttachInfo.isValid() &&
7587 "Expected valid attach pointer/pointee information!");
7588
7589 // Size is the size of the pointer itself - use pointer size, not BaseDecl
7590 // size
7591 llvm::Value *PointerSize = CGF.Builder.CreateIntCast(
7592 V: llvm::ConstantInt::get(
7593 Ty: CGF.CGM.SizeTy, V: CGF.getContext()
7594 .getTypeSizeInChars(T: CGF.getContext().VoidPtrTy)
7595 .getQuantity()),
7596 DestTy: CGF.Int64Ty, /*isSigned=*/true);
7597
7598 CombinedInfo.Exprs.emplace_back(Args: AttachInfo.AttachPtrDecl,
7599 Args: AttachInfo.AttachMapExpr);
7600 CombinedInfo.BasePointers.push_back(
7601 Elt: AttachInfo.AttachPtrAddr.emitRawPointer(CGF));
7602 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7603 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7604 CombinedInfo.Pointers.push_back(
7605 Elt: AttachInfo.AttachPteeAddr.emitRawPointer(CGF));
7606 CombinedInfo.Sizes.push_back(Elt: PointerSize);
7607 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
7608 CombinedInfo.Mappers.push_back(Elt: nullptr);
7609 CombinedInfo.NonContigInfo.Dims.push_back(Elt: 1);
7610 }
7611
7612 /// A helper class to copy structures with overlapped elements, i.e. those
7613 /// which have mappings of both "s" and "s.mem". Consecutive elements that
7614 /// are not explicitly copied have mapping nodes synthesized for them,
7615 /// taking care to avoid generating zero-sized copies.
7616 class CopyOverlappedEntryGaps {
7617 CodeGenFunction &CGF;
7618 MapCombinedInfoTy &CombinedInfo;
7619 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7620 const ValueDecl *MapDecl = nullptr;
7621 const Expr *MapExpr = nullptr;
7622 Address BP = Address::invalid();
7623 bool IsNonContiguous = false;
7624 uint64_t DimSize = 0;
7625 // These elements track the position as the struct is iterated over
7626 // (in order of increasing element address).
7627 const RecordDecl *LastParent = nullptr;
7628 uint64_t Cursor = 0;
7629 unsigned LastIndex = -1u;
7630 Address LB = Address::invalid();
7631
7632 public:
7633 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7634 MapCombinedInfoTy &CombinedInfo,
7635 OpenMPOffloadMappingFlags Flags,
7636 const ValueDecl *MapDecl, const Expr *MapExpr,
7637 Address BP, Address LB, bool IsNonContiguous,
7638 uint64_t DimSize)
7639 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7640 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7641 DimSize(DimSize), LB(LB) {}
7642
7643 void processField(
7644 const OMPClauseMappableExprCommon::MappableComponent &MC,
7645 const FieldDecl *FD,
7646 llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)>
7647 EmitMemberExprBase) {
7648 const RecordDecl *RD = FD->getParent();
7649 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(D: RD);
7650 uint64_t FieldOffset = RL.getFieldOffset(FieldNo: FD->getFieldIndex());
7651 uint64_t FieldSize =
7652 CGF.getContext().getTypeSize(T: FD->getType().getCanonicalType());
7653 Address ComponentLB = Address::invalid();
7654
7655 if (FD->getType()->isLValueReferenceType()) {
7656 const auto *ME = cast<MemberExpr>(Val: MC.getAssociatedExpression());
7657 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7658 ComponentLB =
7659 CGF.EmitLValueForFieldInitialization(Base: BaseLVal, Field: FD).getAddress();
7660 } else {
7661 ComponentLB =
7662 CGF.EmitOMPSharedLValue(E: MC.getAssociatedExpression()).getAddress();
7663 }
7664
7665 if (!LastParent)
7666 LastParent = RD;
7667 if (FD->getParent() == LastParent) {
7668 if (FD->getFieldIndex() != LastIndex + 1)
7669 copyUntilField(FD, ComponentLB);
7670 } else {
7671 LastParent = FD->getParent();
7672 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7673 copyUntilField(FD, ComponentLB);
7674 }
7675 Cursor = FieldOffset + FieldSize;
7676 LastIndex = FD->getFieldIndex();
7677 LB = CGF.Builder.CreateConstGEP(Addr: ComponentLB, Index: 1);
7678 }
7679
7680 void copyUntilField(const FieldDecl *FD, Address ComponentLB) {
7681 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7682 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7683 llvm::Value *Size = CGF.Builder.CreatePtrDiff(LHS: ComponentLBPtr, RHS: LBPtr);
7684 copySizedChunk(Base: LBPtr, Size);
7685 }
7686
7687 void copyUntilEnd(Address HB) {
7688 if (LastParent) {
7689 const ASTRecordLayout &RL =
7690 CGF.getContext().getASTRecordLayout(D: LastParent);
7691 if ((uint64_t)CGF.getContext().toBits(CharSize: RL.getSize()) <= Cursor)
7692 return;
7693 }
7694 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7695 llvm::Value *Size = CGF.Builder.CreatePtrDiff(
7696 LHS: CGF.Builder.CreateConstGEP(Addr: HB, Index: 1).emitRawPointer(CGF), RHS: LBPtr);
7697 copySizedChunk(Base: LBPtr, Size);
7698 }
7699
7700 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7701 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
7702 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
7703 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7704 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7705 CombinedInfo.Pointers.push_back(Elt: Base);
7706 CombinedInfo.Sizes.push_back(
7707 Elt: CGF.Builder.CreateIntCast(V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/false));
7708 CombinedInfo.Types.push_back(Elt: Flags);
7709 CombinedInfo.Mappers.push_back(Elt: nullptr);
7710 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize : 1);
7711 }
7712 };
7713
7714 /// Generate the base pointers, section pointers, sizes, map type bits, and
7715 /// user-defined mappers (all included in \a CombinedInfo) for the provided
7716 /// map type, map or motion modifiers, and expression components.
7717 /// \a IsFirstComponent should be set to true if the provided set of
7718 /// components is the first associated with a capture.
7719 void generateInfoForComponentList(
7720 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7721 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7722 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7723 MapCombinedInfoTy &CombinedInfo,
7724 MapCombinedInfoTy &StructBaseCombinedInfo,
7725 StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo,
7726 bool IsFirstComponentList, bool IsImplicit,
7727 bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr,
7728 bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr,
7729 const Expr *MapExpr = nullptr,
7730 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7731 OverlappedElements = {}) const {
7732
7733 // The following summarizes what has to be generated for each map and the
7734 // types below. The generated information is expressed in this order:
7735 // base pointer, section pointer, size, flags
7736 // (to add to the ones that come from the map type and modifier).
7737 // Entries annotated with (+) are only generated for "target" constructs,
7738 // and only if the variable at the beginning of the expression is used in
7739 // the region.
7740 //
7741 // double d;
7742 // int i[100];
7743 // float *p;
7744 // int **a = &i;
7745 //
7746 // struct S1 {
7747 // int i;
7748 // float f[50];
7749 // }
7750 // struct S2 {
7751 // int i;
7752 // float f[50];
7753 // S1 s;
7754 // double *p;
7755 // double *&pref;
7756 // struct S2 *ps;
7757 // int &ref;
7758 // }
7759 // S2 s;
7760 // S2 *ps;
7761 //
7762 // map(d)
7763 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
7764 //
7765 // map(i)
7766 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
7767 //
7768 // map(i[1:23])
7769 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
7770 //
7771 // map(p)
7772 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7773 //
7774 // map(p[1:24])
7775 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM // map pointee
7776 // &p, &p[1], sizeof(void*), ATTACH // attach pointer/pointee, if both
7777 // // are present, and either is new
7778 //
7779 // map(([22])p)
7780 // p, p, 22*sizeof(float), TARGET_PARAM | TO | FROM
7781 // &p, p, sizeof(void*), ATTACH
7782 //
7783 // map((*a)[0:3])
7784 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
7785 // (*a)[0], &(*a)[0], 3 * sizeof(int), TO | FROM
7786 // &(*a), &(*a)[0], sizeof(void*), ATTACH
7787 // (+) Only on target, if a is used in the region
7788 // Note: Since the attach base-pointer is `*a`, which is not a scalar
7789 // variable, it doesn't determine the clause on `a`. `a` is mapped using
7790 // a zero-length-array-section map by generateDefaultMapInfo, if it is
7791 // referenced in the target region, because it is a pointer.
7792 //
7793 // map(**a)
7794 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
7795 // &(*a)[0], &(*a)[0], sizeof(int), TO | FROM
7796 // &(*a), &(*a)[0], sizeof(void*), ATTACH
7797 // (+) Only on target, if a is used in the region
7798 //
7799 // map(s)
7800 // FIXME: This needs to also imply map(ref_ptr_ptee: s.ref), since the
7801 // effect is supposed to be same as if the user had a map for every element
7802 // of the struct. We currently do a shallow-map of s.
7803 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
7804 //
7805 // map(s.i)
7806 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
7807 //
7808 // map(s.s.f)
7809 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7810 //
7811 // map(s.p)
7812 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
7813 //
7814 // map(to: s.p[:22])
7815 // &s, &(s.p), sizeof(double*), TARGET_PARAM | IMPLICIT // (+)
7816 // &(s.p[0]), &(s.p[0]), 22 * sizeof(double*), TO | FROM
7817 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7818 //
7819 // map(to: s.ref)
7820 // &s, &(ptr(s.ref)), sizeof(int*), TARGET_PARAM (*)
7821 // &s, &(ptee(s.ref)), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
7822 // (*) alloc space for struct members, only this is a target parameter.
7823 // (**) map the pointer (nothing to be mapped in this example) (the compiler
7824 // optimizes this entry out, same in the examples below)
7825 // (***) map the pointee (map: to)
7826 // Note: ptr(s.ref) represents the referring pointer of s.ref
7827 // ptee(s.ref) represents the referenced pointee of s.ref
7828 //
7829 // map(to: s.pref)
7830 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM
7831 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO
7832 //
7833 // map(to: s.pref[:22])
7834 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | IMPLICIT // (+)
7835 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO |
7836 // FROM | IMPLICIT // (+)
7837 // &(ptee(s.pref)[0]), &(ptee(s.pref)[0]), 22 * sizeof(double), TO
7838 // &(ptee(s.pref)), &(ptee(s.pref)[0]), sizeof(void*), ATTACH
7839 //
7840 // map(s.ps)
7841 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7842 //
7843 // map(from: s.ps->s.i)
7844 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7845 // &(s.ps[0]), &(s.ps->s.i), sizeof(int), FROM
7846 // &(s.ps), &(s.ps->s.i), sizeof(void*), ATTACH
7847 //
7848 // map(to: s.ps->ps)
7849 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7850 // &(s.ps[0]), &(s.ps->ps), sizeof(S2*), TO
7851 // &(s.ps), &(s.ps->ps), sizeof(void*), ATTACH
7852 //
7853 // map(s.ps->ps->ps)
7854 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7855 // &(s.ps->ps[0]), &(s.ps->ps->ps), sizeof(S2*), TO
7856 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(void*), ATTACH
7857 //
7858 // map(to: s.ps->ps->s.f[:22])
7859 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7860 // &(s.ps->ps[0]), &(s.ps->ps->s.f[0]), 22*sizeof(float), TO
7861 // &(s.ps->ps), &(s.ps->ps->s.f[0]), sizeof(void*), ATTACH
7862 //
7863 // map(ps)
7864 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
7865 //
7866 // map(ps->i)
7867 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
7868 // &ps, &(ps->i), sizeof(void*), ATTACH
7869 //
7870 // map(ps->s.f)
7871 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7872 // &ps, &(ps->s.f[0]), sizeof(ps), ATTACH
7873 //
7874 // map(from: ps->p)
7875 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
7876 // &ps, &(ps->p), sizeof(ps), ATTACH
7877 //
7878 // map(to: ps->p[:22])
7879 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
7880 // &(ps->p[0]), &(ps->p[0]), 22*sizeof(double), TO
7881 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7882 //
7883 // map(ps->ps)
7884 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7885 // &ps, &(ps->ps), sizeof(ps), ATTACH
7886 //
7887 // map(from: ps->ps->s.i)
7888 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
7889 // &(ps->ps[0]), &(ps->ps->s.i), sizeof(int), FROM
7890 // &(ps->ps), &(ps->ps->s.i), sizeof(void*), ATTACH
7891 //
7892 // map(from: ps->ps->ps)
7893 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7894 // &(ps->ps[0]), &(ps->ps->ps), sizeof(S2*), FROM
7895 // &(ps->ps), &(ps->ps->ps), sizeof(void*), ATTACH
7896 //
7897 // map(ps->ps->ps->ps)
7898 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7899 // &(ps->ps->ps[0]), &(ps->ps->ps->ps), sizeof(S2*), FROM
7900 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(void*), ATTACH
7901 //
7902 // map(to: ps->ps->ps->s.f[:22])
7903 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7904 // &(ps->ps->ps[0]), &(ps->ps->ps->s.f[0]), 22*sizeof(float), TO
7905 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), sizeof(void*), ATTACH
7906 //
7907 // map(to: s.f[:22]) map(from: s.p[:33])
7908 // On target, and if s is used in the region:
7909 //
7910 // &s, &(s.f[0]), 50*sizeof(float) +
7911 // sizeof(struct S1) +
7912 // sizeof(double*) (**), TARGET_PARAM
7913 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
7914 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) | TO |
7915 // FROM | IMPLICIT
7916 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
7917 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7918 // (**) allocate contiguous space needed to fit all mapped members even if
7919 // we allocate space for members not mapped (in this example,
7920 // s.f[22..49] and s.s are not mapped, yet we must allocate space for
7921 // them as well because they fall between &s.f[0] and &s.p)
7922 //
7923 // On other constructs, and, if s is not used in the region, on target:
7924 // &s, &(s.f[0]), 22*sizeof(float), TO
7925 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
7926 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7927 //
7928 // map(from: s.f[:22]) map(to: ps->p[:33])
7929 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
7930 // &ps[0], &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7931 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
7932 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7933 //
7934 // map(from: s.f[:22], s.s) map(to: ps->p[:33])
7935 // &s, &(s.f[0]), 50*sizeof(float) +
7936 // sizeof(struct S1), TARGET_PARAM
7937 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
7938 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
7939 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7940 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
7941 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7942 //
7943 // map(p[:100], p)
7944 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7945 // p, &p[0], 100*sizeof(float), TO | FROM
7946 // &p, &p[0], sizeof(float*), ATTACH
7947
7948 // Track if the map information being generated is the first for a capture.
7949 bool IsCaptureFirstInfo = IsFirstComponentList;
7950 // When the variable is on a declare target link or in a to clause with
7951 // unified memory, a reference is needed to hold the host/device address
7952 // of the variable.
7953 bool RequiresReference = false;
7954
7955 // Scan the components from the base to the complete expression.
7956 auto CI = Components.rbegin();
7957 auto CE = Components.rend();
7958 auto I = CI;
7959
7960 // Track if the map information being generated is the first for a list of
7961 // components.
7962 bool IsExpressionFirstInfo = true;
7963 bool FirstPointerInComplexData = false;
7964 Address BP = Address::invalid();
7965 Address FinalLowestElem = Address::invalid();
7966 const Expr *AssocExpr = I->getAssociatedExpression();
7967 const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr);
7968 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
7969 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(Val: AssocExpr);
7970
7971 // Get the pointer-attachment base-pointer for the given list, if any.
7972 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
7973 auto [AttachPtrAddr, AttachPteeBaseAddr] =
7974 getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF);
7975
7976 bool HasAttachPtr = AttachPtrExpr != nullptr;
7977 bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr;
7978 bool SeenAttachPtr = FirstComponentIsForAttachPtr;
7979
7980 if (FirstComponentIsForAttachPtr) {
7981 // No need to process AttachPtr here. It will be processed at the end
7982 // after we have computed the pointee's address.
7983 ++I;
7984 } else if (isa<MemberExpr>(Val: AssocExpr)) {
7985 // The base is the 'this' pointer. The content of the pointer is going
7986 // to be the base of the field being mapped.
7987 BP = CGF.LoadCXXThisAddress();
7988 } else if ((AE && isa<CXXThisExpr>(Val: AE->getBase()->IgnoreParenImpCasts())) ||
7989 (OASE &&
7990 isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))) {
7991 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
7992 } else if (OAShE &&
7993 isa<CXXThisExpr>(Val: OAShE->getBase()->IgnoreParenCasts())) {
7994 BP = Address(
7995 CGF.EmitScalarExpr(E: OAShE->getBase()),
7996 CGF.ConvertTypeForMem(T: OAShE->getBase()->getType()->getPointeeType()),
7997 CGF.getContext().getTypeAlignInChars(T: OAShE->getBase()->getType()));
7998 } else {
7999 // The base is the reference to the variable.
8000 // BP = &Var.
8001 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8002 if (const auto *VD =
8003 dyn_cast_or_null<VarDecl>(Val: I->getAssociatedDeclaration())) {
8004 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8005 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
8006 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
8007 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
8008 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
8009 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
8010 RequiresReference = true;
8011 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
8012 }
8013 }
8014 }
8015
8016 // If the variable is a pointer and is being dereferenced (i.e. is not
8017 // the last component), the base has to be the pointer itself, not its
8018 // reference. References are ignored for mapping purposes.
8019 QualType Ty =
8020 I->getAssociatedDeclaration()->getType().getNonReferenceType();
8021 if (Ty->isAnyPointerType() && std::next(x: I) != CE) {
8022 // No need to generate individual map information for the pointer, it
8023 // can be associated with the combined storage if shared memory mode is
8024 // active or the base declaration is not global variable.
8025 const auto *VD = dyn_cast<VarDecl>(Val: I->getAssociatedDeclaration());
8026 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8027 !VD || VD->hasLocalStorage() || HasAttachPtr)
8028 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8029 else
8030 FirstPointerInComplexData = true;
8031 ++I;
8032 }
8033 }
8034
8035 // Track whether a component of the list should be marked as MEMBER_OF some
8036 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
8037 // in a component list should be marked as MEMBER_OF, all subsequent entries
8038 // do not belong to the base struct. E.g.
8039 // struct S2 s;
8040 // s.ps->ps->ps->f[:]
8041 // (1) (2) (3) (4)
8042 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
8043 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
8044 // is the pointee of ps(2) which is not member of struct s, so it should not
8045 // be marked as such (it is still PTR_AND_OBJ).
8046 // The variable is initialized to false so that PTR_AND_OBJ entries which
8047 // are not struct members are not considered (e.g. array of pointers to
8048 // data).
8049 bool ShouldBeMemberOf = false;
8050
8051 // Variable keeping track of whether or not we have encountered a component
8052 // in the component list which is a member expression. Useful when we have a
8053 // pointer or a final array section, in which case it is the previous
8054 // component in the list which tells us whether we have a member expression.
8055 // E.g. X.f[:]
8056 // While processing the final array section "[:]" it is "f" which tells us
8057 // whether we are dealing with a member of a declared struct.
8058 const MemberExpr *EncounteredME = nullptr;
8059
8060 // Track for the total number of dimension. Start from one for the dummy
8061 // dimension.
8062 uint64_t DimSize = 1;
8063
8064 // Detects non-contiguous updates due to strided accesses.
8065 // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set
8066 // correctly when generating information to be passed to the runtime. The
8067 // flag is set to true if any array section has a stride not equal to 1, or
8068 // if the stride is not a constant expression (conservatively assumed
8069 // non-contiguous).
8070 bool IsNonContiguous =
8071 CombinedInfo.NonContigInfo.IsNonContiguous ||
8072 any_of(Range&: Components, P: [&](const auto &Component) {
8073 const auto *OASE =
8074 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
8075 if (!OASE)
8076 return false;
8077
8078 const Expr *StrideExpr = OASE->getStride();
8079 if (!StrideExpr)
8080 return false;
8081
8082 assert(StrideExpr->getType()->isIntegerType() &&
8083 "Stride expression must be of integer type");
8084
8085 // If stride is not evaluatable as a constant, treat as
8086 // non-contiguous.
8087 const auto Constant =
8088 StrideExpr->getIntegerConstantExpr(Ctx: CGF.getContext());
8089 if (!Constant)
8090 return true;
8091
8092 // Treat non-unitary strides as non-contiguous.
8093 return !Constant->isOne();
8094 });
8095
8096 bool IsPrevMemberReference = false;
8097
8098 bool IsPartialMapped =
8099 !PartialStruct.PreliminaryMapData.BasePointers.empty();
8100
8101 // We need to check if we will be encountering any MEs. If we do not
8102 // encounter any ME expression it means we will be mapping the whole struct.
8103 // In that case we need to skip adding an entry for the struct to the
8104 // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo
8105 // list only when generating all info for clauses.
8106 bool IsMappingWholeStruct = true;
8107 if (!GenerateAllInfoForClauses) {
8108 IsMappingWholeStruct = false;
8109 } else {
8110 for (auto TempI = I; TempI != CE; ++TempI) {
8111 const MemberExpr *PossibleME =
8112 dyn_cast<MemberExpr>(Val: TempI->getAssociatedExpression());
8113 if (PossibleME) {
8114 IsMappingWholeStruct = false;
8115 break;
8116 }
8117 }
8118 }
8119
8120 bool SeenFirstNonBinOpExprAfterAttachPtr = false;
8121 for (; I != CE; ++I) {
8122 // If we have a valid attach-ptr, we skip processing all components until
8123 // after the attach-ptr.
8124 if (HasAttachPtr && !SeenAttachPtr) {
8125 SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr;
8126 continue;
8127 }
8128
8129 // After finding the attach pointer, skip binary-ops, to skip past
8130 // expressions like (p + 10), for a map like map(*(p + 10)), where p is
8131 // the attach-ptr.
8132 if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) {
8133 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8134 if (BO)
8135 continue;
8136
8137 // Found the first non-binary-operator component after attach
8138 SeenFirstNonBinOpExprAfterAttachPtr = true;
8139 BP = AttachPteeBaseAddr;
8140 }
8141
8142 // If the current component is member of a struct (parent struct) mark it.
8143 if (!EncounteredME) {
8144 EncounteredME = dyn_cast<MemberExpr>(Val: I->getAssociatedExpression());
8145 // If we encounter a PTR_AND_OBJ entry from now on it should be marked
8146 // as MEMBER_OF the parent struct.
8147 if (EncounteredME) {
8148 ShouldBeMemberOf = true;
8149 // Do not emit as complex pointer if this is actually not array-like
8150 // expression.
8151 if (FirstPointerInComplexData) {
8152 QualType Ty = std::prev(x: I)
8153 ->getAssociatedDeclaration()
8154 ->getType()
8155 .getNonReferenceType();
8156 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8157 FirstPointerInComplexData = false;
8158 }
8159 }
8160 }
8161
8162 auto Next = std::next(x: I);
8163
8164 // We need to generate the addresses and sizes if this is the last
8165 // component, if the component is a pointer or if it is an array section
8166 // whose length can't be proved to be one. If this is a pointer, it
8167 // becomes the base address for the following components.
8168
8169 // A final array section, is one whose length can't be proved to be one.
8170 // If the map item is non-contiguous then we don't treat any array section
8171 // as final array section.
8172 bool IsFinalArraySection =
8173 !IsNonContiguous &&
8174 isFinalArraySectionExpression(E: I->getAssociatedExpression());
8175
8176 // If we have a declaration for the mapping use that, otherwise use
8177 // the base declaration of the map clause.
8178 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
8179 ? I->getAssociatedDeclaration()
8180 : BaseDecl;
8181 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
8182 : MapExpr;
8183
8184 // Get information on whether the element is a pointer. Have to do a
8185 // special treatment for array sections given that they are built-in
8186 // types.
8187 const auto *OASE =
8188 dyn_cast<ArraySectionExpr>(Val: I->getAssociatedExpression());
8189 const auto *OAShE =
8190 dyn_cast<OMPArrayShapingExpr>(Val: I->getAssociatedExpression());
8191 const auto *UO = dyn_cast<UnaryOperator>(Val: I->getAssociatedExpression());
8192 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8193 bool IsPointer =
8194 OAShE ||
8195 (OASE && ArraySectionExpr::getBaseOriginalType(Base: OASE)
8196 .getCanonicalType()
8197 ->isAnyPointerType()) ||
8198 I->getAssociatedExpression()->getType()->isAnyPointerType();
8199 bool IsMemberReference = isa<MemberExpr>(Val: I->getAssociatedExpression()) &&
8200 MapDecl &&
8201 MapDecl->getType()->isLValueReferenceType();
8202 bool IsNonDerefPointer = IsPointer &&
8203 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
8204 !IsNonContiguous;
8205
8206 if (OASE)
8207 ++DimSize;
8208
8209 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
8210 IsFinalArraySection) {
8211 // If this is not the last component, we expect the pointer to be
8212 // associated with an array expression or member expression.
8213 assert((Next == CE ||
8214 isa<MemberExpr>(Next->getAssociatedExpression()) ||
8215 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
8216 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
8217 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
8218 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
8219 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
8220 "Unexpected expression");
8221
8222 Address LB = Address::invalid();
8223 Address LowestElem = Address::invalid();
8224 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
8225 const MemberExpr *E) {
8226 const Expr *BaseExpr = E->getBase();
8227 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a
8228 // scalar.
8229 LValue BaseLV;
8230 if (E->isArrow()) {
8231 LValueBaseInfo BaseInfo;
8232 TBAAAccessInfo TBAAInfo;
8233 Address Addr =
8234 CGF.EmitPointerWithAlignment(Addr: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
8235 QualType PtrTy = BaseExpr->getType()->getPointeeType();
8236 BaseLV = CGF.MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
8237 } else {
8238 BaseLV = CGF.EmitOMPSharedLValue(E: BaseExpr);
8239 }
8240 return BaseLV;
8241 };
8242 if (OAShE) {
8243 LowestElem = LB =
8244 Address(CGF.EmitScalarExpr(E: OAShE->getBase()),
8245 CGF.ConvertTypeForMem(
8246 T: OAShE->getBase()->getType()->getPointeeType()),
8247 CGF.getContext().getTypeAlignInChars(
8248 T: OAShE->getBase()->getType()));
8249 } else if (IsMemberReference) {
8250 const auto *ME = cast<MemberExpr>(Val: I->getAssociatedExpression());
8251 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
8252 LowestElem = CGF.EmitLValueForFieldInitialization(
8253 Base: BaseLVal, Field: cast<FieldDecl>(Val: MapDecl))
8254 .getAddress();
8255 LB = CGF.EmitLoadOfReferenceLValue(RefAddr: LowestElem, RefTy: MapDecl->getType())
8256 .getAddress();
8257 } else {
8258 LowestElem = LB =
8259 CGF.EmitOMPSharedLValue(E: I->getAssociatedExpression())
8260 .getAddress();
8261 }
8262
8263 // Save the final LowestElem, to use it as the pointee in attach maps,
8264 // if emitted.
8265 if (Next == CE)
8266 FinalLowestElem = LowestElem;
8267
8268 // If this component is a pointer inside the base struct then we don't
8269 // need to create any entry for it - it will be combined with the object
8270 // it is pointing to into a single PTR_AND_OBJ entry.
8271 bool IsMemberPointerOrAddr =
8272 EncounteredME &&
8273 (((IsPointer || ForDeviceAddr) &&
8274 I->getAssociatedExpression() == EncounteredME) ||
8275 (IsPrevMemberReference && !IsPointer) ||
8276 (IsMemberReference && Next != CE &&
8277 !Next->getAssociatedExpression()->getType()->isPointerType()));
8278 if (!OverlappedElements.empty() && Next == CE) {
8279 // Handle base element with the info for overlapped elements.
8280 assert(!PartialStruct.Base.isValid() && "The base element is set.");
8281 assert(!IsPointer &&
8282 "Unexpected base element with the pointer type.");
8283 // Mark the whole struct as the struct that requires allocation on the
8284 // device.
8285 PartialStruct.LowestElem = {0, LowestElem};
8286 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
8287 T: I->getAssociatedExpression()->getType());
8288 Address HB = CGF.Builder.CreateConstGEP(
8289 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8290 Addr: LowestElem, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty),
8291 Index: TypeSize.getQuantity() - 1);
8292 PartialStruct.HighestElem = {
8293 std::numeric_limits<decltype(
8294 PartialStruct.HighestElem.first)>::max(),
8295 HB};
8296 PartialStruct.Base = BP;
8297 PartialStruct.LB = LB;
8298 assert(
8299 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8300 "Overlapped elements must be used only once for the variable.");
8301 std::swap(a&: PartialStruct.PreliminaryMapData, b&: CombinedInfo);
8302 // Emit data for non-overlapped data.
8303 OpenMPOffloadMappingFlags Flags =
8304 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8305 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8306 /*AddPtrFlag=*/false,
8307 /*AddIsTargetParamFlag=*/false, IsNonContiguous);
8308 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8309 MapExpr, BP, LB, IsNonContiguous,
8310 DimSize);
8311 // Do bitcopy of all non-overlapped structure elements.
8312 for (OMPClauseMappableExprCommon::MappableExprComponentListRef
8313 Component : OverlappedElements) {
8314 for (const OMPClauseMappableExprCommon::MappableComponent &MC :
8315 Component) {
8316 if (const ValueDecl *VD = MC.getAssociatedDeclaration()) {
8317 if (const auto *FD = dyn_cast<FieldDecl>(Val: VD)) {
8318 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8319 }
8320 }
8321 }
8322 }
8323 CopyGaps.copyUntilEnd(HB);
8324 break;
8325 }
8326 llvm::Value *Size = getExprTypeSize(E: I->getAssociatedExpression());
8327 // Skip adding an entry in the CurInfo of this combined entry if the
8328 // whole struct is currently being mapped. The struct needs to be added
8329 // in the first position before any data internal to the struct is being
8330 // mapped.
8331 // Skip adding an entry in the CurInfo of this combined entry if the
8332 // PartialStruct.PreliminaryMapData.BasePointers has been mapped.
8333 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8334 (Next == CE && MapType != OMPC_MAP_unknown)) {
8335 if (!IsMappingWholeStruct) {
8336 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8337 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
8338 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8339 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8340 CombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8341 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8342 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8343 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize
8344 : 1);
8345 } else {
8346 StructBaseCombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8347 StructBaseCombinedInfo.BasePointers.push_back(
8348 Elt: BP.emitRawPointer(CGF));
8349 StructBaseCombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8350 StructBaseCombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8351 StructBaseCombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8352 StructBaseCombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8353 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8354 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8355 Elt: IsNonContiguous ? DimSize : 1);
8356 }
8357
8358 // If Mapper is valid, the last component inherits the mapper.
8359 bool HasMapper = Mapper && Next == CE;
8360 if (!IsMappingWholeStruct)
8361 CombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper : nullptr);
8362 else
8363 StructBaseCombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper
8364 : nullptr);
8365
8366 // We need to add a pointer flag for each map that comes from the
8367 // same expression except for the first one. We also need to signal
8368 // this map is the first one that relates with the current capture
8369 // (there is a set of entries for each capture).
8370 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
8371 MapType, MapModifiers, MotionModifiers, IsImplicit,
8372 AddPtrFlag: !IsExpressionFirstInfo || RequiresReference ||
8373 FirstPointerInComplexData || IsMemberReference,
8374 AddIsTargetParamFlag: IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
8375
8376 if (!IsExpressionFirstInfo || IsMemberReference) {
8377 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
8378 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
8379 if (IsPointer || (IsMemberReference && Next != CE))
8380 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8381 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8382 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8383 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8384 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8385
8386 if (ShouldBeMemberOf) {
8387 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
8388 // should be later updated with the correct value of MEMBER_OF.
8389 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8390 // From now on, all subsequent PTR_AND_OBJ entries should not be
8391 // marked as MEMBER_OF.
8392 ShouldBeMemberOf = false;
8393 }
8394 }
8395
8396 if (!IsMappingWholeStruct)
8397 CombinedInfo.Types.push_back(Elt: Flags);
8398 else
8399 StructBaseCombinedInfo.Types.push_back(Elt: Flags);
8400 }
8401
8402 // If we have encountered a member expression so far, keep track of the
8403 // mapped member. If the parent is "*this", then the value declaration
8404 // is nullptr.
8405 if (EncounteredME) {
8406 const auto *FD = cast<FieldDecl>(Val: EncounteredME->getMemberDecl());
8407 unsigned FieldIndex = FD->getFieldIndex();
8408
8409 // Update info about the lowest and highest elements for this struct
8410 if (!PartialStruct.Base.isValid()) {
8411 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8412 if (IsFinalArraySection && OASE) {
8413 Address HB =
8414 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8415 .getAddress();
8416 PartialStruct.HighestElem = {FieldIndex, HB};
8417 } else {
8418 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8419 }
8420 PartialStruct.Base = BP;
8421 PartialStruct.LB = BP;
8422 } else if (FieldIndex < PartialStruct.LowestElem.first) {
8423 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8424 } else if (FieldIndex > PartialStruct.HighestElem.first) {
8425 if (IsFinalArraySection && OASE) {
8426 Address HB =
8427 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8428 .getAddress();
8429 PartialStruct.HighestElem = {FieldIndex, HB};
8430 } else {
8431 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8432 }
8433 }
8434 }
8435
8436 // Need to emit combined struct for array sections.
8437 if (IsFinalArraySection || IsNonContiguous)
8438 PartialStruct.IsArraySection = true;
8439
8440 // If we have a final array section, we are done with this expression.
8441 if (IsFinalArraySection)
8442 break;
8443
8444 // The pointer becomes the base for the next element.
8445 if (Next != CE)
8446 BP = IsMemberReference ? LowestElem : LB;
8447 if (!IsPartialMapped)
8448 IsExpressionFirstInfo = false;
8449 IsCaptureFirstInfo = false;
8450 FirstPointerInComplexData = false;
8451 IsPrevMemberReference = IsMemberReference;
8452 } else if (FirstPointerInComplexData) {
8453 QualType Ty = Components.rbegin()
8454 ->getAssociatedDeclaration()
8455 ->getType()
8456 .getNonReferenceType();
8457 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8458 FirstPointerInComplexData = false;
8459 }
8460 }
8461 // If ran into the whole component - allocate the space for the whole
8462 // record.
8463 if (!EncounteredME)
8464 PartialStruct.HasCompleteRecord = true;
8465
8466 // Populate ATTACH information for later processing by emitAttachEntry.
8467 if (shouldEmitAttachEntry(PointerExpr: AttachPtrExpr, MapBaseDecl: BaseDecl, CGF, CurDir)) {
8468 AttachInfo.AttachPtrAddr = AttachPtrAddr;
8469 AttachInfo.AttachPteeAddr = FinalLowestElem;
8470 AttachInfo.AttachPtrDecl = BaseDecl;
8471 AttachInfo.AttachMapExpr = MapExpr;
8472 }
8473
8474 if (!IsNonContiguous)
8475 return;
8476
8477 const ASTContext &Context = CGF.getContext();
8478
8479 // For supporting stride in array section, we need to initialize the first
8480 // dimension size as 1, first offset as 0, and first count as 1
8481 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 0)};
8482 MapValuesArrayTy CurCounts;
8483 MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8484 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8485 uint64_t ElementTypeSize;
8486
8487 // Collect Size information for each dimension and get the element size as
8488 // the first Stride. For example, for `int arr[10][10]`, the DimSizes
8489 // should be [10, 10] and the first stride is 4 btyes.
8490 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8491 Components) {
8492 const Expr *AssocExpr = Component.getAssociatedExpression();
8493 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8494
8495 if (!OASE)
8496 continue;
8497
8498 QualType Ty = ArraySectionExpr::getBaseOriginalType(Base: OASE->getBase());
8499 auto *CAT = Context.getAsConstantArrayType(T: Ty);
8500 auto *VAT = Context.getAsVariableArrayType(T: Ty);
8501
8502 // We need all the dimension size except for the last dimension.
8503 assert((VAT || CAT || &Component == &*Components.begin()) &&
8504 "Should be either ConstantArray or VariableArray if not the "
8505 "first Component");
8506
8507 // Get element size if CurCounts is empty.
8508 if (CurCounts.empty()) {
8509 const Type *ElementType = nullptr;
8510 if (CAT)
8511 ElementType = CAT->getElementType().getTypePtr();
8512 else if (VAT)
8513 ElementType = VAT->getElementType().getTypePtr();
8514 else if (&Component == &*Components.begin()) {
8515 // If the base is a raw pointer (e.g. T *data with data[a:b:c]),
8516 // there was no earlier CAT/VAT/array handling to establish
8517 // ElementType. Capture the pointee type now so that subsequent
8518 // components (offset/length/stride) have a concrete element type to
8519 // work with. This makes pointer-backed sections behave consistently
8520 // with CAT/VAT/array bases.
8521 if (const auto *PtrType = Ty->getAs<PointerType>())
8522 ElementType = PtrType->getPointeeType().getTypePtr();
8523 } else {
8524 // Any component after the first should never have a raw pointer type;
8525 // by this point. ElementType must already be known (set above or in
8526 // prior array / CAT / VAT handling).
8527 assert(!Ty->isPointerType() &&
8528 "Non-first components should not be raw pointers");
8529 }
8530
8531 // At this stage, if ElementType was a base pointer and we are in the
8532 // first iteration, it has been computed.
8533 if (ElementType) {
8534 // For the case that having pointer as base, we need to remove one
8535 // level of indirection.
8536 if (&Component != &*Components.begin())
8537 ElementType = ElementType->getPointeeOrArrayElementType();
8538 ElementTypeSize =
8539 Context.getTypeSizeInChars(T: ElementType).getQuantity();
8540 CurCounts.push_back(
8541 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: ElementTypeSize));
8542 }
8543 }
8544 // Get dimension value except for the last dimension since we don't need
8545 // it.
8546 if (DimSizes.size() < Components.size() - 1) {
8547 if (CAT)
8548 DimSizes.push_back(
8549 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: CAT->getZExtSize()));
8550 else if (VAT)
8551 DimSizes.push_back(Elt: CGF.Builder.CreateIntCast(
8552 V: CGF.EmitScalarExpr(E: VAT->getSizeExpr()), DestTy: CGF.Int64Ty,
8553 /*IsSigned=*/isSigned: false));
8554 }
8555 }
8556
8557 // Skip the dummy dimension since we have already have its information.
8558 auto *DI = DimSizes.begin() + 1;
8559 // Product of dimension.
8560 llvm::Value *DimProd =
8561 llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: ElementTypeSize);
8562
8563 // Collect info for non-contiguous. Notice that offset, count, and stride
8564 // are only meaningful for array-section, so we insert a null for anything
8565 // other than array-section.
8566 // Also, the size of offset, count, and stride are not the same as
8567 // pointers, base_pointers, sizes, or dims. Instead, the size of offset,
8568 // count, and stride are the same as the number of non-contiguous
8569 // declaration in target update to/from clause.
8570 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8571 Components) {
8572 const Expr *AssocExpr = Component.getAssociatedExpression();
8573
8574 if (const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr)) {
8575 llvm::Value *Offset = CGF.Builder.CreateIntCast(
8576 V: CGF.EmitScalarExpr(E: AE->getIdx()), DestTy: CGF.Int64Ty,
8577 /*isSigned=*/false);
8578 CurOffsets.push_back(Elt: Offset);
8579 CurCounts.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/1));
8580 CurStrides.push_back(Elt: CurStrides.back());
8581 continue;
8582 }
8583
8584 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8585
8586 if (!OASE)
8587 continue;
8588
8589 // Offset
8590 const Expr *OffsetExpr = OASE->getLowerBound();
8591 llvm::Value *Offset = nullptr;
8592 if (!OffsetExpr) {
8593 // If offset is absent, then we just set it to zero.
8594 Offset = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
8595 } else {
8596 Offset = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: OffsetExpr),
8597 DestTy: CGF.Int64Ty,
8598 /*isSigned=*/false);
8599 }
8600
8601 // Count
8602 const Expr *CountExpr = OASE->getLength();
8603 llvm::Value *Count = nullptr;
8604 if (!CountExpr) {
8605 // In Clang, once a high dimension is an array section, we construct all
8606 // the lower dimension as array section, however, for case like
8607 // arr[0:2][2], Clang construct the inner dimension as an array section
8608 // but it actually is not in an array section form according to spec.
8609 if (!OASE->getColonLocFirst().isValid() &&
8610 !OASE->getColonLocSecond().isValid()) {
8611 Count = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 1);
8612 } else {
8613 // OpenMP 5.0, 2.1.5 Array Sections, Description.
8614 // When the length is absent it defaults to ⌈(size −
8615 // lower-bound)/stride⌉, where size is the size of the array
8616 // dimension.
8617 const Expr *StrideExpr = OASE->getStride();
8618 llvm::Value *Stride =
8619 StrideExpr
8620 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8621 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8622 : nullptr;
8623 if (Stride)
8624 Count = CGF.Builder.CreateUDiv(
8625 LHS: CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset), RHS: Stride);
8626 else
8627 Count = CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset);
8628 }
8629 } else {
8630 Count = CGF.EmitScalarExpr(E: CountExpr);
8631 }
8632 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: CGF.Int64Ty, /*isSigned=*/false);
8633 CurCounts.push_back(Elt: Count);
8634
8635 // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size
8636 // Offset_n' = Offset_n * (D_0 * D_1 ... * D_n-1) * Unit size
8637 // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example:
8638 // Offset Count Stride
8639 // D0 0 4 1 (int) <- dummy dimension
8640 // D1 0 2 8 (2 * (1) * 4)
8641 // D2 100 2 20 (1 * (1 * 5) * 4)
8642 // D3 0 2 200 (2 * (1 * 5 * 4) * 4)
8643 const Expr *StrideExpr = OASE->getStride();
8644 llvm::Value *Stride =
8645 StrideExpr
8646 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8647 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8648 : nullptr;
8649 DimProd = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: *(DI - 1));
8650 if (Stride)
8651 CurStrides.push_back(Elt: CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Stride));
8652 else
8653 CurStrides.push_back(Elt: DimProd);
8654
8655 Offset = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Offset);
8656 CurOffsets.push_back(Elt: Offset);
8657
8658 if (DI != DimSizes.end())
8659 ++DI;
8660 }
8661
8662 CombinedInfo.NonContigInfo.Offsets.push_back(Elt: CurOffsets);
8663 CombinedInfo.NonContigInfo.Counts.push_back(Elt: CurCounts);
8664 CombinedInfo.NonContigInfo.Strides.push_back(Elt: CurStrides);
8665 }
8666
8667 /// Return the adjusted map modifiers if the declaration a capture refers to
8668 /// appears in a first-private clause. This is expected to be used only with
8669 /// directives that start with 'target'.
8670 OpenMPOffloadMappingFlags
8671 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
8672 assert(Cap.capturesVariable() && "Expected capture by reference only!");
8673
8674 // A first private variable captured by reference will use only the
8675 // 'private ptr' and 'map to' flag. Return the right flags if the captured
8676 // declaration is known as first-private in this handler.
8677 if (FirstPrivateDecls.count(Val: Cap.getCapturedVar())) {
8678 if (Cap.getCapturedVar()->getType()->isAnyPointerType())
8679 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8680 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8681 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8682 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8683 }
8684 auto I = LambdasMap.find(Val: Cap.getCapturedVar()->getCanonicalDecl());
8685 if (I != LambdasMap.end())
8686 // for map(to: lambda): using user specified map type.
8687 return getMapTypeBits(
8688 MapType: I->getSecond()->getMapType(), MapModifiers: I->getSecond()->getMapTypeModifiers(),
8689 /*MotionModifiers=*/{}, IsImplicit: I->getSecond()->isImplicit(),
8690 /*AddPtrFlag=*/false,
8691 /*AddIsTargetParamFlag=*/false,
8692 /*isNonContiguous=*/IsNonContiguous: false);
8693 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8694 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8695 }
8696
8697 void getPlainLayout(const CXXRecordDecl *RD,
8698 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8699 bool AsBase) const {
8700 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
8701
8702 llvm::StructType *St =
8703 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
8704
8705 unsigned NumElements = St->getNumElements();
8706 llvm::SmallVector<
8707 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8708 RecordLayout(NumElements);
8709
8710 // Fill bases.
8711 for (const auto &I : RD->bases()) {
8712 if (I.isVirtual())
8713 continue;
8714
8715 QualType BaseTy = I.getType();
8716 const auto *Base = BaseTy->getAsCXXRecordDecl();
8717 // Ignore empty bases.
8718 if (isEmptyRecordForLayout(Context: CGF.getContext(), T: BaseTy) ||
8719 CGF.getContext()
8720 .getASTRecordLayout(D: Base)
8721 .getNonVirtualSize()
8722 .isZero())
8723 continue;
8724
8725 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(RD: Base);
8726 RecordLayout[FieldIndex] = Base;
8727 }
8728 // Fill in virtual bases.
8729 for (const auto &I : RD->vbases()) {
8730 QualType BaseTy = I.getType();
8731 // Ignore empty bases.
8732 if (isEmptyRecordForLayout(Context: CGF.getContext(), T: BaseTy))
8733 continue;
8734
8735 const auto *Base = BaseTy->getAsCXXRecordDecl();
8736 unsigned FieldIndex = RL.getVirtualBaseIndex(base: Base);
8737 if (RecordLayout[FieldIndex])
8738 continue;
8739 RecordLayout[FieldIndex] = Base;
8740 }
8741 // Fill in all the fields.
8742 assert(!RD->isUnion() && "Unexpected union.");
8743 for (const auto *Field : RD->fields()) {
8744 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
8745 // will fill in later.)
8746 if (!Field->isBitField() &&
8747 !isEmptyFieldForLayout(Context: CGF.getContext(), FD: Field)) {
8748 unsigned FieldIndex = RL.getLLVMFieldNo(FD: Field);
8749 RecordLayout[FieldIndex] = Field;
8750 }
8751 }
8752 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
8753 &Data : RecordLayout) {
8754 if (Data.isNull())
8755 continue;
8756 if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Val: Data))
8757 getPlainLayout(RD: Base, Layout, /*AsBase=*/true);
8758 else
8759 Layout.push_back(Elt: cast<const FieldDecl *>(Val: Data));
8760 }
8761 }
8762
8763 /// Returns the address corresponding to \p PointerExpr.
8764 static Address getAttachPtrAddr(const Expr *PointerExpr,
8765 CodeGenFunction &CGF) {
8766 assert(PointerExpr && "Cannot get addr from null attach-ptr expr");
8767 Address AttachPtrAddr = Address::invalid();
8768
8769 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: PointerExpr)) {
8770 // If the pointer is a variable, we can use its address directly.
8771 AttachPtrAddr = CGF.EmitLValue(E: DRE).getAddress();
8772 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(Val: PointerExpr)) {
8773 AttachPtrAddr =
8774 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/true).getAddress();
8775 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: PointerExpr)) {
8776 AttachPtrAddr = CGF.EmitLValue(E: ASE).getAddress();
8777 } else if (auto *ME = dyn_cast<MemberExpr>(Val: PointerExpr)) {
8778 AttachPtrAddr = CGF.EmitMemberExpr(E: ME).getAddress();
8779 } else if (auto *UO = dyn_cast<UnaryOperator>(Val: PointerExpr)) {
8780 assert(UO->getOpcode() == UO_Deref &&
8781 "Unexpected unary-operator on attach-ptr-expr");
8782 AttachPtrAddr = CGF.EmitLValue(E: UO).getAddress();
8783 }
8784 assert(AttachPtrAddr.isValid() &&
8785 "Failed to get address for attach pointer expression");
8786 return AttachPtrAddr;
8787 }
8788
8789 /// Get the address of the attach pointer, and a load from it, to get the
8790 /// pointee base address.
8791 /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair
8792 /// contains invalid addresses if \p AttachPtrExpr is null.
8793 static std::pair<Address, Address>
8794 getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr,
8795 CodeGenFunction &CGF) {
8796
8797 if (!AttachPtrExpr)
8798 return {Address::invalid(), Address::invalid()};
8799
8800 Address AttachPtrAddr = getAttachPtrAddr(PointerExpr: AttachPtrExpr, CGF);
8801 assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr");
8802
8803 QualType AttachPtrType =
8804 OMPClauseMappableExprCommon::getComponentExprElementType(Exp: AttachPtrExpr)
8805 .getCanonicalType();
8806
8807 Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer(
8808 Ptr: AttachPtrAddr, PtrTy: AttachPtrType->castAs<PointerType>());
8809 assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr");
8810
8811 return {AttachPtrAddr, AttachPteeBaseAddr};
8812 }
8813
8814 /// Returns whether an attach entry should be emitted for a map on
8815 /// \p MapBaseDecl on the directive \p CurDir.
8816 static bool
8817 shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl,
8818 CodeGenFunction &CGF,
8819 llvm::PointerUnion<const OMPExecutableDirective *,
8820 const OMPDeclareMapperDecl *>
8821 CurDir) {
8822 if (!PointerExpr)
8823 return false;
8824
8825 // Pointer attachment is needed at map-entering time or for declare
8826 // mappers.
8827 return isa<const OMPDeclareMapperDecl *>(Val: CurDir) ||
8828 isOpenMPTargetMapEnteringDirective(
8829 DKind: cast<const OMPExecutableDirective *>(Val&: CurDir)
8830 ->getDirectiveKind());
8831 }
8832
8833 /// Computes the attach-ptr expr for \p Components, and updates various maps
8834 /// with the information.
8835 /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr()
8836 /// with the OpenMPDirectiveKind extracted from \p CurDir.
8837 /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and
8838 /// AttachPtrExprMap.
8839 void collectAttachPtrExprInfo(
8840 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
8841 llvm::PointerUnion<const OMPExecutableDirective *,
8842 const OMPDeclareMapperDecl *>
8843 CurDir) {
8844
8845 OpenMPDirectiveKind CurDirectiveID =
8846 isa<const OMPDeclareMapperDecl *>(Val: CurDir)
8847 ? OMPD_declare_mapper
8848 : cast<const OMPExecutableDirective *>(Val&: CurDir)->getDirectiveKind();
8849
8850 const auto &[AttachPtrExpr, Depth] =
8851 OMPClauseMappableExprCommon::findAttachPtrExpr(Components,
8852 CurDirKind: CurDirectiveID);
8853
8854 AttachPtrComputationOrderMap.try_emplace(
8855 Key: AttachPtrExpr, Args: AttachPtrComputationOrderMap.size());
8856 AttachPtrComponentDepthMap.try_emplace(Key: AttachPtrExpr, Args: Depth);
8857 AttachPtrExprMap.try_emplace(Key: Components, Args: AttachPtrExpr);
8858 }
8859
8860 /// Generate all the base pointers, section pointers, sizes, map types, and
8861 /// mappers for the extracted mappable expressions (all included in \a
8862 /// CombinedInfo). Also, for each item that relates with a device pointer, a
8863 /// pair of the relevant declaration and index where it occurs is appended to
8864 /// the device pointers info array.
8865 void generateAllInfoForClauses(
8866 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
8867 llvm::OpenMPIRBuilder &OMPBuilder,
8868 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
8869 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
8870 // We have to process the component lists that relate with the same
8871 // declaration in a single chunk so that we can generate the map flags
8872 // correctly. Therefore, we organize all lists in a map.
8873 enum MapKind { Present, Allocs, Other, Total };
8874 llvm::MapVector<CanonicalDeclPtr<const Decl>,
8875 SmallVector<SmallVector<MapInfo, 8>, 4>>
8876 Info;
8877
8878 // Helper function to fill the information map for the different supported
8879 // clauses.
8880 auto &&InfoGen =
8881 [&Info, &SkipVarSet](
8882 const ValueDecl *D, MapKind Kind,
8883 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
8884 OpenMPMapClauseKind MapType,
8885 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8886 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
8887 bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,
8888 const Expr *VarRef = nullptr, bool ForDeviceAddr = false) {
8889 if (SkipVarSet.contains(V: D))
8890 return;
8891 auto It = Info.try_emplace(Key: D, Args: Total).first;
8892 It->second[Kind].emplace_back(
8893 Args&: L, Args&: MapType, Args&: MapModifiers, Args&: MotionModifiers, Args&: ReturnDevicePointer,
8894 Args&: IsImplicit, Args&: Mapper, Args&: VarRef, Args&: ForDeviceAddr);
8895 };
8896
8897 for (const auto *Cl : Clauses) {
8898 const auto *C = dyn_cast<OMPMapClause>(Val: Cl);
8899 if (!C)
8900 continue;
8901 MapKind Kind = Other;
8902 if (llvm::is_contained(Range: C->getMapTypeModifiers(),
8903 Element: OMPC_MAP_MODIFIER_present))
8904 Kind = Present;
8905 else if (C->getMapType() == OMPC_MAP_alloc)
8906 Kind = Allocs;
8907 const auto *EI = C->getVarRefs().begin();
8908 for (const auto L : C->component_lists()) {
8909 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
8910 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), C->getMapType(),
8911 C->getMapTypeModifiers(), {},
8912 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
8913 E);
8914 ++EI;
8915 }
8916 }
8917 for (const auto *Cl : Clauses) {
8918 const auto *C = dyn_cast<OMPToClause>(Val: Cl);
8919 if (!C)
8920 continue;
8921 MapKind Kind = Other;
8922 if (llvm::is_contained(Range: C->getMotionModifiers(),
8923 Element: OMPC_MOTION_MODIFIER_present))
8924 Kind = Present;
8925 if (llvm::is_contained(Range: C->getMotionModifiers(),
8926 Element: OMPC_MOTION_MODIFIER_iterator)) {
8927 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
8928 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
8929 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
8930 CGF.EmitVarDecl(D: *VD);
8931 }
8932 }
8933
8934 const auto *EI = C->getVarRefs().begin();
8935 for (const auto L : C->component_lists()) {
8936 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_to, {},
8937 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
8938 C->isImplicit(), std::get<2>(t: L), *EI);
8939 ++EI;
8940 }
8941 }
8942 for (const auto *Cl : Clauses) {
8943 const auto *C = dyn_cast<OMPFromClause>(Val: Cl);
8944 if (!C)
8945 continue;
8946 MapKind Kind = Other;
8947 if (llvm::is_contained(Range: C->getMotionModifiers(),
8948 Element: OMPC_MOTION_MODIFIER_present))
8949 Kind = Present;
8950 if (llvm::is_contained(Range: C->getMotionModifiers(),
8951 Element: OMPC_MOTION_MODIFIER_iterator)) {
8952 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
8953 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
8954 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
8955 CGF.EmitVarDecl(D: *VD);
8956 }
8957 }
8958
8959 const auto *EI = C->getVarRefs().begin();
8960 for (const auto L : C->component_lists()) {
8961 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_from, {},
8962 C->getMotionModifiers(),
8963 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
8964 *EI);
8965 ++EI;
8966 }
8967 }
8968
8969 // Look at the use_device_ptr and use_device_addr clauses information and
8970 // mark the existing map entries as such. If there is no map information for
8971 // an entry in the use_device_ptr and use_device_addr list, we create one
8972 // with map type 'return_param' and zero size section. It is the user's
8973 // fault if that was not mapped before. If there is no map information, then
8974 // we defer the emission of that entry until all the maps for the same VD
8975 // have been handled.
8976 MapCombinedInfoTy UseDeviceDataCombinedInfo;
8977
8978 auto &&UseDeviceDataCombinedInfoGen =
8979 [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,
8980 CodeGenFunction &CGF, bool IsDevAddr,
8981 bool HasUdpFbNullify = false) {
8982 UseDeviceDataCombinedInfo.Exprs.push_back(Elt: VD);
8983 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Args&: Ptr);
8984 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(Args&: VD);
8985 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
8986 Args: IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
8987 // FIXME: For use_device_addr on array-sections, this should
8988 // be the starting address of the section.
8989 // e.g. int *p;
8990 // ... use_device_addr(p[3])
8991 // &p[0], &p[3], /*size=*/0, RETURN_PARAM
8992 UseDeviceDataCombinedInfo.Pointers.push_back(Elt: Ptr);
8993 UseDeviceDataCombinedInfo.Sizes.push_back(
8994 Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
8995 OpenMPOffloadMappingFlags Flags =
8996 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8997 if (HasUdpFbNullify)
8998 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
8999 UseDeviceDataCombinedInfo.Types.push_back(Elt: Flags);
9000 UseDeviceDataCombinedInfo.Mappers.push_back(Elt: nullptr);
9001 };
9002
9003 auto &&MapInfoGen =
9004 [&UseDeviceDataCombinedInfoGen](
9005 CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
9006 OMPClauseMappableExprCommon::MappableExprComponentListRef
9007 Components,
9008 bool IsDevAddr, bool IEIsAttachPtrForDevAddr = false,
9009 bool HasUdpFbNullify = false) {
9010 // We didn't find any match in our map information - generate a zero
9011 // size array section.
9012 llvm::Value *Ptr;
9013 if (IsDevAddr && !IEIsAttachPtrForDevAddr) {
9014 if (IE->isGLValue())
9015 Ptr = CGF.EmitLValue(E: IE).getPointer(CGF);
9016 else
9017 Ptr = CGF.EmitScalarExpr(E: IE);
9018 } else {
9019 Ptr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: IE), Loc: IE->getExprLoc());
9020 }
9021 bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr;
9022 // For the purpose of address-translation, treat something like the
9023 // following:
9024 // int *p;
9025 // ... use_device_addr(p[1])
9026 // equivalent to
9027 // ... use_device_ptr(p)
9028 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, /*IsDevAddr=*/IsDevAddr &&
9029 !TreatDevAddrAsDevPtr,
9030 HasUdpFbNullify);
9031 };
9032
9033 auto &&IsMapInfoExist =
9034 [&Info, this](CodeGenFunction &CGF, const ValueDecl *VD, const Expr *IE,
9035 const Expr *DesiredAttachPtrExpr, bool IsDevAddr,
9036 bool HasUdpFbNullify = false) -> bool {
9037 // We potentially have map information for this declaration already.
9038 // Look for the first set of components that refer to it. If found,
9039 // return true.
9040 // If the first component is a member expression, we have to look into
9041 // 'this', which maps to null in the map of map information. Otherwise
9042 // look directly for the information.
9043 auto It = Info.find(Key: isa<MemberExpr>(Val: IE) ? nullptr : VD);
9044 if (It != Info.end()) {
9045 bool Found = false;
9046 for (auto &Data : It->second) {
9047 MapInfo *CI = nullptr;
9048 // We potentially have multiple maps for the same decl. We need to
9049 // only consider those for which the attach-ptr matches the desired
9050 // attach-ptr.
9051 auto *It = llvm::find_if(Range&: Data, P: [&](const MapInfo &MI) {
9052 if (MI.Components.back().getAssociatedDeclaration() != VD)
9053 return false;
9054
9055 const Expr *MapAttachPtr = getAttachPtrExpr(Components: MI.Components);
9056 bool Match = AttachPtrComparator.areEqual(LHS: MapAttachPtr,
9057 RHS: DesiredAttachPtrExpr);
9058 return Match;
9059 });
9060
9061 if (It != Data.end())
9062 CI = &*It;
9063
9064 if (CI) {
9065 if (IsDevAddr) {
9066 CI->ForDeviceAddr = true;
9067 CI->ReturnDevicePointer = true;
9068 CI->HasUdpFbNullify = HasUdpFbNullify;
9069 Found = true;
9070 break;
9071 } else {
9072 auto PrevCI = std::next(x: CI->Components.rbegin());
9073 const auto *VarD = dyn_cast<VarDecl>(Val: VD);
9074 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: CI->Components);
9075 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
9076 isa<MemberExpr>(Val: IE) ||
9077 !VD->getType().getNonReferenceType()->isPointerType() ||
9078 PrevCI == CI->Components.rend() ||
9079 isa<MemberExpr>(Val: PrevCI->getAssociatedExpression()) || !VarD ||
9080 VarD->hasLocalStorage() ||
9081 (isa_and_nonnull<DeclRefExpr>(Val: AttachPtrExpr) &&
9082 VD == cast<DeclRefExpr>(Val: AttachPtrExpr)->getDecl())) {
9083 CI->ForDeviceAddr = IsDevAddr;
9084 CI->ReturnDevicePointer = true;
9085 CI->HasUdpFbNullify = HasUdpFbNullify;
9086 Found = true;
9087 break;
9088 }
9089 }
9090 }
9091 }
9092 return Found;
9093 }
9094 return false;
9095 };
9096
9097 // Look at the use_device_ptr clause information and mark the existing map
9098 // entries as such. If there is no map information for an entry in the
9099 // use_device_ptr list, we create one with map type 'alloc' and zero size
9100 // section. It is the user fault if that was not mapped before. If there is
9101 // no map information and the pointer is a struct member, then we defer the
9102 // emission of that entry until the whole struct has been processed.
9103 for (const auto *Cl : Clauses) {
9104 const auto *C = dyn_cast<OMPUseDevicePtrClause>(Val: Cl);
9105 if (!C)
9106 continue;
9107 bool HasUdpFbNullify =
9108 C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify;
9109 for (const auto L : C->component_lists()) {
9110 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9111 std::get<1>(t: L);
9112 assert(!Components.empty() &&
9113 "Not expecting empty list of components!");
9114 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
9115 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9116 const Expr *IE = Components.back().getAssociatedExpression();
9117 // For use_device_ptr, we match an existing map clause if its attach-ptr
9118 // is same as the use_device_ptr operand. e.g.
9119 // map expr | use_device_ptr expr | current behavior
9120 // ---------|---------------------|-----------------
9121 // p[1] | p | match
9122 // ps->a | ps | match
9123 // p | p | no match
9124 const Expr *UDPOperandExpr =
9125 Components.front().getAssociatedExpression();
9126 if (IsMapInfoExist(CGF, VD, IE,
9127 /*DesiredAttachPtrExpr=*/UDPOperandExpr,
9128 /*IsDevAddr=*/false, HasUdpFbNullify))
9129 continue;
9130 MapInfoGen(CGF, IE, VD, Components, /*IsDevAddr=*/false,
9131 /*IEIsAttachPtrForDevAddr=*/false, HasUdpFbNullify);
9132 }
9133 }
9134
9135 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
9136 for (const auto *Cl : Clauses) {
9137 const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Val: Cl);
9138 if (!C)
9139 continue;
9140 for (const auto L : C->component_lists()) {
9141 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9142 std::get<1>(t: L);
9143 assert(!std::get<1>(L).empty() &&
9144 "Not expecting empty list of components!");
9145 const ValueDecl *VD = std::get<1>(t: L).back().getAssociatedDeclaration();
9146 if (!Processed.insert(V: VD).second)
9147 continue;
9148 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9149 // For use_device_addr, we match an existing map clause if the
9150 // use_device_addr operand's attach-ptr matches the map operand's
9151 // attach-ptr.
9152 // We chould also restrict to only match cases when there is a full
9153 // match between the map/use_device_addr clause exprs, but that may be
9154 // unnecessary.
9155 //
9156 // map expr | use_device_addr expr | current | possible restrictive/
9157 // | | behavior | safer behavior
9158 // ---------|----------------------|-----------|-----------------------
9159 // p | p | match | match
9160 // p[0] | p[0] | match | match
9161 // p[0:1] | p[0] | match | no match
9162 // p[0:1] | p[2:1] | match | no match
9163 // p[1] | p[0] | match | no match
9164 // ps->a | ps->b | match | no match
9165 // p | p[0] | no match | no match
9166 // pp | pp[0][0] | no match | no match
9167 const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components);
9168 const Expr *IE = std::get<1>(t: L).back().getAssociatedExpression();
9169 assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) &&
9170 "use_device_addr operand has an attach-ptr, but does not match "
9171 "last component's expr.");
9172 if (IsMapInfoExist(CGF, VD, IE,
9173 /*DesiredAttachPtrExpr=*/UDAAttachPtrExpr,
9174 /*IsDevAddr=*/true))
9175 continue;
9176 MapInfoGen(CGF, IE, VD, Components,
9177 /*IsDevAddr=*/true,
9178 /*IEIsAttachPtrForDevAddr=*/UDAAttachPtrExpr != nullptr);
9179 }
9180 }
9181
9182 for (const auto &Data : Info) {
9183 MapCombinedInfoTy CurInfo;
9184 const Decl *D = Data.first;
9185 const ValueDecl *VD = cast_or_null<ValueDecl>(Val: D);
9186 // Group component lists by their AttachPtrExpr and process them in order
9187 // of increasing complexity (nullptr first, then simple expressions like
9188 // p, then more complex ones like p[0], etc.)
9189 //
9190 // This is similar to how generateInfoForCaptureFromClauseInfo handles
9191 // grouping for target constructs.
9192 SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs;
9193
9194 // First, collect all MapData entries with their attach-ptr exprs.
9195 for (const auto &M : Data.second) {
9196 for (const MapInfo &L : M) {
9197 assert(!L.Components.empty() &&
9198 "Not expecting declaration with no component lists.");
9199
9200 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: L.Components);
9201 AttachPtrMapInfoPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
9202 }
9203 }
9204
9205 // Next, sort by increasing order of their complexity.
9206 llvm::stable_sort(Range&: AttachPtrMapInfoPairs,
9207 C: [this](const auto &LHS, const auto &RHS) {
9208 return AttachPtrComparator(LHS.first, RHS.first);
9209 });
9210
9211 // And finally, process them all in order, grouping those with
9212 // equivalent attach-ptr exprs together.
9213 auto *It = AttachPtrMapInfoPairs.begin();
9214 while (It != AttachPtrMapInfoPairs.end()) {
9215 const Expr *AttachPtrExpr = It->first;
9216
9217 SmallVector<MapInfo, 8> GroupLists;
9218 while (It != AttachPtrMapInfoPairs.end() &&
9219 (It->first == AttachPtrExpr ||
9220 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
9221 GroupLists.push_back(Elt: It->second);
9222 ++It;
9223 }
9224 assert(!GroupLists.empty() && "GroupLists should not be empty");
9225
9226 StructRangeInfoTy PartialStruct;
9227 AttachInfoTy AttachInfo;
9228 MapCombinedInfoTy GroupCurInfo;
9229 // Current group's struct base information:
9230 MapCombinedInfoTy GroupStructBaseCurInfo;
9231 for (const MapInfo &L : GroupLists) {
9232 // Remember the current base pointer index.
9233 unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size();
9234 unsigned StructBasePointersIdx =
9235 GroupStructBaseCurInfo.BasePointers.size();
9236
9237 GroupCurInfo.NonContigInfo.IsNonContiguous =
9238 L.Components.back().isNonContiguous();
9239 generateInfoForComponentList(
9240 MapType: L.MapType, MapModifiers: L.MapModifiers, MotionModifiers: L.MotionModifiers, Components: L.Components,
9241 CombinedInfo&: GroupCurInfo, StructBaseCombinedInfo&: GroupStructBaseCurInfo, PartialStruct, AttachInfo,
9242 /*IsFirstComponentList=*/false, IsImplicit: L.IsImplicit,
9243 /*GenerateAllInfoForClauses*/ true, Mapper: L.Mapper, ForDeviceAddr: L.ForDeviceAddr, BaseDecl: VD,
9244 MapExpr: L.VarRef, /*OverlappedElements*/ {});
9245
9246 // If this entry relates to a device pointer, set the relevant
9247 // declaration and add the 'return pointer' flag.
9248 if (L.ReturnDevicePointer) {
9249 // Check whether a value was added to either GroupCurInfo or
9250 // GroupStructBaseCurInfo and error if no value was added to either
9251 // of them:
9252 assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() ||
9253 StructBasePointersIdx <
9254 GroupStructBaseCurInfo.BasePointers.size()) &&
9255 "Unexpected number of mapped base pointers.");
9256
9257 // Choose a base pointer index which is always valid:
9258 const ValueDecl *RelevantVD =
9259 L.Components.back().getAssociatedDeclaration();
9260 assert(RelevantVD &&
9261 "No relevant declaration related with device pointer??");
9262
9263 // If GroupStructBaseCurInfo has been updated this iteration then
9264 // work on the first new entry added to it i.e. make sure that when
9265 // multiple values are added to any of the lists, the first value
9266 // added is being modified by the assignments below (not the last
9267 // value added).
9268 auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info,
9269 unsigned Idx) {
9270 Info.DevicePtrDecls[Idx] = RelevantVD;
9271 Info.DevicePointers[Idx] = L.ForDeviceAddr
9272 ? DeviceInfoTy::Address
9273 : DeviceInfoTy::Pointer;
9274 Info.Types[Idx] |=
9275 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9276 if (L.HasUdpFbNullify)
9277 Info.Types[Idx] |=
9278 OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9279 };
9280
9281 if (StructBasePointersIdx <
9282 GroupStructBaseCurInfo.BasePointers.size())
9283 SetDevicePointerInfo(GroupStructBaseCurInfo,
9284 StructBasePointersIdx);
9285 else
9286 SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx);
9287 }
9288 }
9289
9290 // Unify entries in one list making sure the struct mapping precedes the
9291 // individual fields:
9292 MapCombinedInfoTy GroupUnionCurInfo;
9293 GroupUnionCurInfo.append(CurInfo&: GroupStructBaseCurInfo);
9294 GroupUnionCurInfo.append(CurInfo&: GroupCurInfo);
9295
9296 // If there is an entry in PartialStruct it means we have a struct with
9297 // individual members mapped. Emit an extra combined entry.
9298 if (PartialStruct.Base.isValid()) {
9299 // Prepend a synthetic dimension of length 1 to represent the
9300 // aggregated struct object. Using 1 (not 0, as 0 produced an
9301 // incorrect non-contiguous descriptor (DimSize==1), causing the
9302 // non-contiguous motion clause path to be skipped.) is important:
9303 // * It preserves the correct rank so targetDataUpdate() computes
9304 // DimSize == 2 for cases like strided array sections originating
9305 // from user-defined mappers (e.g. test with s.data[0:8:2]).
9306 GroupUnionCurInfo.NonContigInfo.Dims.insert(
9307 I: GroupUnionCurInfo.NonContigInfo.Dims.begin(), Elt: 1);
9308 emitCombinedEntry(
9309 CombinedInfo&: CurInfo, CurTypes&: GroupUnionCurInfo.Types, PartialStruct, AttachInfo,
9310 /*IsMapThis=*/!VD, OMPBuilder, VD,
9311 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size(),
9312 /*NotTargetParams=*/true);
9313 }
9314
9315 // Append this group's results to the overall CurInfo in the correct
9316 // order: combined-entry -> original-field-entries -> attach-entry
9317 CurInfo.append(CurInfo&: GroupUnionCurInfo);
9318 if (AttachInfo.isValid())
9319 emitAttachEntry(CGF, CombinedInfo&: CurInfo, AttachInfo);
9320 }
9321
9322 // We need to append the results of this capture to what we already have.
9323 CombinedInfo.append(CurInfo);
9324 }
9325 // Append data for use_device_ptr/addr clauses.
9326 CombinedInfo.append(CurInfo&: UseDeviceDataCombinedInfo);
9327 }
9328
9329public:
9330 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
9331 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9332 // Extract firstprivate clause information.
9333 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
9334 for (const auto *D : C->varlist())
9335 FirstPrivateDecls.try_emplace(
9336 Key: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D)->getDecl()), Args: C->isImplicit());
9337 // Extract implicit firstprivates from uses_allocators clauses.
9338 for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
9339 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
9340 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
9341 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: D.AllocatorTraits))
9342 FirstPrivateDecls.try_emplace(Key: cast<VarDecl>(Val: DRE->getDecl()),
9343 /*Implicit=*/Args: true);
9344 else if (const auto *VD = dyn_cast<VarDecl>(
9345 Val: cast<DeclRefExpr>(Val: D.Allocator->IgnoreParenImpCasts())
9346 ->getDecl()))
9347 FirstPrivateDecls.try_emplace(Key: VD, /*Implicit=*/Args: true);
9348 }
9349 }
9350 // Extract defaultmap clause information.
9351 for (const auto *C : Dir.getClausesOfKind<OMPDefaultmapClause>())
9352 if (C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate)
9353 DefaultmapFirstprivateKinds.insert(V: C->getDefaultmapKind());
9354 // Extract device pointer clause information.
9355 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9356 for (auto L : C->component_lists())
9357 DevPointersMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9358 // Extract device addr clause information.
9359 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9360 for (auto L : C->component_lists())
9361 HasDevAddrsMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9362 // Extract map information.
9363 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) {
9364 if (C->getMapType() != OMPC_MAP_to)
9365 continue;
9366 for (auto L : C->component_lists()) {
9367 const ValueDecl *VD = std::get<0>(t&: L);
9368 const auto *RD = VD ? VD->getType()
9369 .getCanonicalType()
9370 .getNonReferenceType()
9371 ->getAsCXXRecordDecl()
9372 : nullptr;
9373 if (RD && RD->isLambda())
9374 LambdasMap.try_emplace(Key: std::get<0>(t&: L), Args&: C);
9375 }
9376 }
9377
9378 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9379 for (auto L : C->component_lists()) {
9380 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9381 std::get<1>(L);
9382 if (!Components.empty())
9383 collectAttachPtrExprInfo(Components, CurDir);
9384 }
9385 };
9386
9387 // Populate the AttachPtrExprMap for all component lists from map-related
9388 // clauses.
9389 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>())
9390 CollectAttachPtrExprsForClauseComponents(C);
9391 for (const auto *C : Dir.getClausesOfKind<OMPToClause>())
9392 CollectAttachPtrExprsForClauseComponents(C);
9393 for (const auto *C : Dir.getClausesOfKind<OMPFromClause>())
9394 CollectAttachPtrExprsForClauseComponents(C);
9395 for (const auto *C : Dir.getClausesOfKind<OMPUseDevicePtrClause>())
9396 CollectAttachPtrExprsForClauseComponents(C);
9397 for (const auto *C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>())
9398 CollectAttachPtrExprsForClauseComponents(C);
9399 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9400 CollectAttachPtrExprsForClauseComponents(C);
9401 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9402 CollectAttachPtrExprsForClauseComponents(C);
9403 }
9404
9405 /// Constructor for the declare mapper directive.
9406 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
9407 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {}
9408
9409 /// Generate code for the combined entry if we have a partially mapped struct
9410 /// and take care of the mapping flags of the arguments corresponding to
9411 /// individual struct members.
9412 /// If a valid \p AttachInfo exists, its pointee addr will be updated to point
9413 /// to the combined-entry's begin address, if emitted.
9414 /// \p PartialStruct contains attach base-pointer information.
9415 /// \returns The index of the combined entry if one was added, std::nullopt
9416 /// otherwise.
9417 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9418 MapFlagsArrayTy &CurTypes,
9419 const StructRangeInfoTy &PartialStruct,
9420 AttachInfoTy &AttachInfo, bool IsMapThis,
9421 llvm::OpenMPIRBuilder &OMPBuilder, const ValueDecl *VD,
9422 unsigned OffsetForMemberOfFlag,
9423 bool NotTargetParams) const {
9424 if (CurTypes.size() == 1 &&
9425 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9426 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9427 !PartialStruct.IsArraySection)
9428 return;
9429 Address LBAddr = PartialStruct.LowestElem.second;
9430 Address HBAddr = PartialStruct.HighestElem.second;
9431 if (PartialStruct.HasCompleteRecord) {
9432 LBAddr = PartialStruct.LB;
9433 HBAddr = PartialStruct.LB;
9434 }
9435 CombinedInfo.Exprs.push_back(Elt: VD);
9436 // Base is the base of the struct
9437 CombinedInfo.BasePointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9438 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9439 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9440 // Pointer is the address of the lowest element
9441 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
9442 const CXXMethodDecl *MD =
9443 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(Val: CGF.CurFuncDecl) : nullptr;
9444 const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
9445 bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false;
9446 // There should not be a mapper for a combined entry.
9447 if (HasBaseClass) {
9448 // OpenMP 5.2 148:21:
9449 // If the target construct is within a class non-static member function,
9450 // and a variable is an accessible data member of the object for which the
9451 // non-static data member function is invoked, the variable is treated as
9452 // if the this[:1] expression had appeared in a map clause with a map-type
9453 // of tofrom.
9454 // Emit this[:1]
9455 CombinedInfo.Pointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9456 QualType Ty = MD->getFunctionObjectParameterType();
9457 llvm::Value *Size =
9458 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty), DestTy: CGF.Int64Ty,
9459 /*isSigned=*/true);
9460 CombinedInfo.Sizes.push_back(Elt: Size);
9461 } else {
9462 CombinedInfo.Pointers.push_back(Elt: LB);
9463 // Size is (addr of {highest+1} element) - (addr of lowest element)
9464 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
9465 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
9466 Ty: HBAddr.getElementType(), Ptr: HB, /*Idx0=*/1);
9467 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(V: LB, DestTy: CGF.VoidPtrTy);
9468 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(V: HAddr, DestTy: CGF.VoidPtrTy);
9469 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(LHS: CHAddr, RHS: CLAddr);
9470 llvm::Value *Size = CGF.Builder.CreateIntCast(V: Diff, DestTy: CGF.Int64Ty,
9471 /*isSigned=*/false);
9472 CombinedInfo.Sizes.push_back(Elt: Size);
9473 }
9474 CombinedInfo.Mappers.push_back(Elt: nullptr);
9475 // Map type is always TARGET_PARAM, if generate info for captures.
9476 CombinedInfo.Types.push_back(
9477 Elt: NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9478 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9479 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9480 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9481 // If any element has the present modifier, then make sure the runtime
9482 // doesn't attempt to allocate the struct.
9483 if (CurTypes.end() !=
9484 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9485 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9486 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9487 }))
9488 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9489 // Remove TARGET_PARAM flag from the first element
9490 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9491 // If any element has the ompx_hold modifier, then make sure the runtime
9492 // uses the hold reference count for the struct as a whole so that it won't
9493 // be unmapped by an extra dynamic reference count decrement. Add it to all
9494 // elements as well so the runtime knows which reference count to check
9495 // when determining whether it's time for device-to-host transfers of
9496 // individual elements.
9497 if (CurTypes.end() !=
9498 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9499 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9500 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9501 })) {
9502 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9503 for (auto &M : CurTypes)
9504 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9505 }
9506
9507 // All other current entries will be MEMBER_OF the combined entry
9508 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9509 // 0xFFFF in the MEMBER_OF field, or ATTACH entries since they are expected
9510 // to be handled by themselves, after all other maps).
9511 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9512 Position: OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9513 for (auto &M : CurTypes)
9514 OMPBuilder.setCorrectMemberOfFlag(Flags&: M, MemberOfFlag);
9515
9516 // When we are emitting a combined entry. If there were any pending
9517 // attachments to be done, we do them to the begin address of the combined
9518 // entry. Note that this means only one attachment per combined-entry will
9519 // be done. So, for instance, if we have:
9520 // S *ps;
9521 // ... map(ps->a, ps->b)
9522 // When we are emitting a combined entry. If AttachInfo is valid,
9523 // update the pointee address to point to the begin address of the combined
9524 // entry. This ensures that if we have multiple maps like:
9525 // `map(ps->a, ps->b)`, we still get a single ATTACH entry, like:
9526 //
9527 // &ps[0], &ps->a, sizeof(ps->a to ps->b), ALLOC // combined-entry
9528 // &ps[0], &ps->a, sizeof(ps->a), TO | FROM
9529 // &ps[0], &ps->b, sizeof(ps->b), TO | FROM
9530 // &ps, &ps->a, sizeof(void*), ATTACH // Use combined-entry's LB
9531 if (AttachInfo.isValid())
9532 AttachInfo.AttachPteeAddr = LBAddr;
9533 }
9534
9535 /// Generate all the base pointers, section pointers, sizes, map types, and
9536 /// mappers for the extracted mappable expressions (all included in \a
9537 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9538 /// pair of the relevant declaration and index where it occurs is appended to
9539 /// the device pointers info array.
9540 void generateAllInfo(
9541 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9542 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9543 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9544 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9545 "Expect a executable directive");
9546 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9547 generateAllInfoForClauses(Clauses: CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9548 SkipVarSet);
9549 }
9550
9551 /// Generate all the base pointers, section pointers, sizes, map types, and
9552 /// mappers for the extracted map clauses of user-defined mapper (all included
9553 /// in \a CombinedInfo).
9554 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9555 llvm::OpenMPIRBuilder &OMPBuilder) const {
9556 assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&
9557 "Expect a declare mapper directive");
9558 const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(Val: CurDir);
9559 generateAllInfoForClauses(Clauses: CurMapperDir->clauses(), CombinedInfo,
9560 OMPBuilder);
9561 }
9562
9563 /// Emit capture info for lambdas for variables captured by reference.
9564 void generateInfoForLambdaCaptures(
9565 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9566 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
9567 QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();
9568 const auto *RD = VDType->getAsCXXRecordDecl();
9569 if (!RD || !RD->isLambda())
9570 return;
9571 Address VDAddr(Arg, CGF.ConvertTypeForMem(T: VDType),
9572 CGF.getContext().getDeclAlign(D: VD));
9573 LValue VDLVal = CGF.MakeAddrLValue(Addr: VDAddr, T: VDType);
9574 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9575 FieldDecl *ThisCapture = nullptr;
9576 RD->getCaptureFields(Captures, ThisCapture);
9577 if (ThisCapture) {
9578 LValue ThisLVal =
9579 CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: ThisCapture);
9580 LValue ThisLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: ThisCapture);
9581 LambdaPointers.try_emplace(Key: ThisLVal.getPointer(CGF),
9582 Args: VDLVal.getPointer(CGF));
9583 CombinedInfo.Exprs.push_back(Elt: VD);
9584 CombinedInfo.BasePointers.push_back(Elt: ThisLVal.getPointer(CGF));
9585 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9586 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9587 CombinedInfo.Pointers.push_back(Elt: ThisLValVal.getPointer(CGF));
9588 CombinedInfo.Sizes.push_back(
9589 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy),
9590 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9591 CombinedInfo.Types.push_back(
9592 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9593 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9594 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9595 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9596 CombinedInfo.Mappers.push_back(Elt: nullptr);
9597 }
9598 for (const LambdaCapture &LC : RD->captures()) {
9599 if (!LC.capturesVariable())
9600 continue;
9601 const VarDecl *VD = cast<VarDecl>(Val: LC.getCapturedVar());
9602 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
9603 continue;
9604 auto It = Captures.find(Val: VD);
9605 assert(It != Captures.end() && "Found lambda capture without field.");
9606 LValue VarLVal = CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: It->second);
9607 if (LC.getCaptureKind() == LCK_ByRef) {
9608 LValue VarLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: It->second);
9609 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9610 Args: VDLVal.getPointer(CGF));
9611 CombinedInfo.Exprs.push_back(Elt: VD);
9612 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9613 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9614 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9615 CombinedInfo.Pointers.push_back(Elt: VarLValVal.getPointer(CGF));
9616 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9617 V: CGF.getTypeSize(
9618 Ty: VD->getType().getCanonicalType().getNonReferenceType()),
9619 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9620 } else {
9621 RValue VarRVal = CGF.EmitLoadOfLValue(V: VarLVal, Loc: RD->getLocation());
9622 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9623 Args: VDLVal.getPointer(CGF));
9624 CombinedInfo.Exprs.push_back(Elt: VD);
9625 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9626 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9627 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9628 CombinedInfo.Pointers.push_back(Elt: VarRVal.getScalarVal());
9629 CombinedInfo.Sizes.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0));
9630 }
9631 CombinedInfo.Types.push_back(
9632 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9633 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9634 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9635 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9636 CombinedInfo.Mappers.push_back(Elt: nullptr);
9637 }
9638 }
9639
9640 /// Set correct indices for lambdas captures.
9641 void adjustMemberOfForLambdaCaptures(
9642 llvm::OpenMPIRBuilder &OMPBuilder,
9643 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9644 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9645 MapFlagsArrayTy &Types) const {
9646 for (unsigned I = 0, E = Types.size(); I < E; ++I) {
9647 // Set correct member_of idx for all implicit lambda captures.
9648 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9649 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9650 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9651 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9652 continue;
9653 llvm::Value *BasePtr = LambdaPointers.lookup(Val: BasePointers[I]);
9654 assert(BasePtr && "Unable to find base lambda address.");
9655 int TgtIdx = -1;
9656 for (unsigned J = I; J > 0; --J) {
9657 unsigned Idx = J - 1;
9658 if (Pointers[Idx] != BasePtr)
9659 continue;
9660 TgtIdx = Idx;
9661 break;
9662 }
9663 assert(TgtIdx != -1 && "Unable to find parent lambda.");
9664 // All other current entries will be MEMBER_OF the combined entry
9665 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9666 // 0xFFFF in the MEMBER_OF field).
9667 OpenMPOffloadMappingFlags MemberOfFlag =
9668 OMPBuilder.getMemberOfFlag(Position: TgtIdx);
9669 OMPBuilder.setCorrectMemberOfFlag(Flags&: Types[I], MemberOfFlag);
9670 }
9671 }
9672
9673 /// Populate component lists for non-lambda captured variables from map,
9674 /// is_device_ptr and has_device_addr clause info.
9675 void populateComponentListsForNonLambdaCaptureFromClauses(
9676 const ValueDecl *VD, MapDataArrayTy &DeclComponentLists,
9677 SmallVectorImpl<
9678 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9679 &StorageForImplicitlyAddedComponentLists) const {
9680 if (VD && LambdasMap.count(Val: VD))
9681 return;
9682
9683 // For member fields list in is_device_ptr, store it in
9684 // DeclComponentLists for generating components info.
9685 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9686 auto It = DevPointersMap.find(Val: VD);
9687 if (It != DevPointersMap.end())
9688 for (const auto &MCL : It->second)
9689 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_to, Args: Unknown,
9690 /*IsImpicit = */ Args: true, Args: nullptr,
9691 Args: nullptr);
9692 auto I = HasDevAddrsMap.find(Val: VD);
9693 if (I != HasDevAddrsMap.end())
9694 for (const auto &MCL : I->second)
9695 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_tofrom, Args: Unknown,
9696 /*IsImpicit = */ Args: true, Args: nullptr,
9697 Args: nullptr);
9698 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9699 "Expect a executable directive");
9700 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9701 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
9702 const auto *EI = C->getVarRefs().begin();
9703 for (const auto L : C->decl_component_lists(VD)) {
9704 const ValueDecl *VDecl, *Mapper;
9705 // The Expression is not correct if the mapping is implicit
9706 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
9707 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
9708 std::tie(args&: VDecl, args&: Components, args&: Mapper) = L;
9709 assert(VDecl == VD && "We got information for the wrong declaration??");
9710 assert(!Components.empty() &&
9711 "Not expecting declaration with no component lists.");
9712 DeclComponentLists.emplace_back(Args&: Components, Args: C->getMapType(),
9713 Args: C->getMapTypeModifiers(),
9714 Args: C->isImplicit(), Args&: Mapper, Args&: E);
9715 ++EI;
9716 }
9717 }
9718
9719 // For the target construct, if there's a map with a base-pointer that's
9720 // a member of an implicitly captured struct, of the current class,
9721 // we need to emit an implicit map on the pointer.
9722 if (isOpenMPTargetExecutionDirective(DKind: CurExecDir->getDirectiveKind()))
9723 addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
9724 CapturedVD: VD, DeclComponentLists, ComponentVectorStorage&: StorageForImplicitlyAddedComponentLists);
9725
9726 llvm::stable_sort(Range&: DeclComponentLists, C: [](const MapData &LHS,
9727 const MapData &RHS) {
9728 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(t: LHS);
9729 OpenMPMapClauseKind MapType = std::get<1>(t: RHS);
9730 bool HasPresent =
9731 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
9732 bool HasAllocs = MapType == OMPC_MAP_alloc;
9733 MapModifiers = std::get<2>(t: RHS);
9734 MapType = std::get<1>(t: LHS);
9735 bool HasPresentR =
9736 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
9737 bool HasAllocsR = MapType == OMPC_MAP_alloc;
9738 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
9739 });
9740 }
9741
9742 /// On a target construct, if there's an implicit map on a struct, or that of
9743 /// this[:], and an explicit map with a member of that struct/class as the
9744 /// base-pointer, we need to make sure that base-pointer is implicitly mapped,
9745 /// to make sure we don't map the full struct/class. For example:
9746 ///
9747 /// \code
9748 /// struct S {
9749 /// int dummy[10000];
9750 /// int *p;
9751 /// void f1() {
9752 /// #pragma omp target map(p[0:1])
9753 /// (void)this;
9754 /// }
9755 /// }; S s;
9756 ///
9757 /// void f2() {
9758 /// #pragma omp target map(s.p[0:10])
9759 /// (void)s;
9760 /// }
9761 /// \endcode
9762 ///
9763 /// Only `this-p` and `s.p` should be mapped in the two cases above.
9764 //
9765 // OpenMP 6.0: 7.9.6 map clause, pg 285
9766 // If a list item with an implicitly determined data-mapping attribute does
9767 // not have any corresponding storage in the device data environment prior to
9768 // a task encountering the construct associated with the map clause, and one
9769 // or more contiguous parts of the original storage are either list items or
9770 // base pointers to list items that are explicitly mapped on the construct,
9771 // only those parts of the original storage will have corresponding storage in
9772 // the device data environment as a result of the map clauses on the
9773 // construct.
9774 void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
9775 const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists,
9776 SmallVectorImpl<
9777 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9778 &ComponentVectorStorage) const {
9779 bool IsThisCapture = CapturedVD == nullptr;
9780
9781 for (const auto &ComponentsAndAttachPtr : AttachPtrExprMap) {
9782 OMPClauseMappableExprCommon::MappableExprComponentListRef
9783 ComponentsWithAttachPtr = ComponentsAndAttachPtr.first;
9784 const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second;
9785 if (!AttachPtrExpr)
9786 continue;
9787
9788 const auto *ME = dyn_cast<MemberExpr>(Val: AttachPtrExpr);
9789 if (!ME)
9790 continue;
9791
9792 const Expr *Base = ME->getBase()->IgnoreParenImpCasts();
9793
9794 // If we are handling a "this" capture, then we are looking for
9795 // attach-ptrs of form `this->p`, either explicitly or implicitly.
9796 if (IsThisCapture && !ME->isImplicitCXXThis() && !isa<CXXThisExpr>(Val: Base))
9797 continue;
9798
9799 if (!IsThisCapture && (!isa<DeclRefExpr>(Val: Base) ||
9800 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
9801 continue;
9802
9803 // For non-this captures, we are looking for attach-ptrs of form
9804 // `s.p`.
9805 // For non-this captures, we are looking for attach-ptrs like `s.p`.
9806 if (!IsThisCapture && (ME->isArrow() || !isa<DeclRefExpr>(Val: Base) ||
9807 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
9808 continue;
9809
9810 // Check if we have an existing map on either:
9811 // this[:], s, this->p, or s.p, in which case, we don't need to add
9812 // an implicit one for the attach-ptr s.p/this->p.
9813 bool FoundExistingMap = false;
9814 for (const MapData &ExistingL : DeclComponentLists) {
9815 OMPClauseMappableExprCommon::MappableExprComponentListRef
9816 ExistingComponents = std::get<0>(t: ExistingL);
9817
9818 if (ExistingComponents.empty())
9819 continue;
9820
9821 // First check if we have a map like map(this->p) or map(s.p).
9822 const auto &FirstComponent = ExistingComponents.front();
9823 const Expr *FirstExpr = FirstComponent.getAssociatedExpression();
9824
9825 if (!FirstExpr)
9826 continue;
9827
9828 // First check if we have a map like map(this->p) or map(s.p).
9829 if (AttachPtrComparator.areEqual(LHS: FirstExpr, RHS: AttachPtrExpr)) {
9830 FoundExistingMap = true;
9831 break;
9832 }
9833
9834 // Check if we have a map like this[0:1]
9835 if (IsThisCapture) {
9836 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: FirstExpr)) {
9837 if (isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts())) {
9838 FoundExistingMap = true;
9839 break;
9840 }
9841 }
9842 continue;
9843 }
9844
9845 // When the attach-ptr is something like `s.p`, check if
9846 // `s` itself is mapped explicitly.
9847 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: FirstExpr)) {
9848 if (DRE->getDecl() == CapturedVD) {
9849 FoundExistingMap = true;
9850 break;
9851 }
9852 }
9853 }
9854
9855 if (FoundExistingMap)
9856 continue;
9857
9858 // If no base map is found, we need to create an implicit map for the
9859 // attach-pointer expr.
9860
9861 ComponentVectorStorage.emplace_back();
9862 auto &AttachPtrComponents = ComponentVectorStorage.back();
9863
9864 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9865 bool SeenAttachPtrComponent = false;
9866 // For creating a map on the attach-ptr `s.p/this->p`, we copy all
9867 // components from the component-list which has `s.p/this->p`
9868 // as the attach-ptr, starting from the component which matches
9869 // `s.p/this->p`. This way, we'll have component-lists of
9870 // `s.p` -> `s`, and `this->p` -> `this`.
9871 for (size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) {
9872 const auto &Component = ComponentsWithAttachPtr[i];
9873 const Expr *ComponentExpr = Component.getAssociatedExpression();
9874
9875 if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr)
9876 continue;
9877 SeenAttachPtrComponent = true;
9878
9879 AttachPtrComponents.emplace_back(Args: Component.getAssociatedExpression(),
9880 Args: Component.getAssociatedDeclaration(),
9881 Args: Component.isNonContiguous());
9882 }
9883 assert(!AttachPtrComponents.empty() &&
9884 "Could not populate component-lists for mapping attach-ptr");
9885
9886 DeclComponentLists.emplace_back(
9887 Args&: AttachPtrComponents, Args: OMPC_MAP_tofrom, Args: Unknown,
9888 /*IsImplicit=*/Args: true, /*mapper=*/Args: nullptr, Args&: AttachPtrExpr);
9889 }
9890 }
9891
9892 /// For a capture that has an associated clause, generate the base pointers,
9893 /// section pointers, sizes, map types, and mappers (all included in
9894 /// \a CurCaptureVarInfo).
9895 void generateInfoForCaptureFromClauseInfo(
9896 const MapDataArrayTy &DeclComponentListsFromClauses,
9897 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
9898 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9899 unsigned OffsetForMemberOfFlag) const {
9900 assert(!Cap->capturesVariableArrayType() &&
9901 "Not expecting to generate map info for a variable array type!");
9902
9903 // We need to know when we generating information for the first component
9904 const ValueDecl *VD = Cap->capturesThis()
9905 ? nullptr
9906 : Cap->getCapturedVar()->getCanonicalDecl();
9907
9908 // for map(to: lambda): skip here, processing it in
9909 // generateDefaultMapInfo
9910 if (LambdasMap.count(Val: VD))
9911 return;
9912
9913 // If this declaration appears in a is_device_ptr clause we just have to
9914 // pass the pointer by value. If it is a reference to a declaration, we just
9915 // pass its value.
9916 if (VD && (DevPointersMap.count(Val: VD) || HasDevAddrsMap.count(Val: VD))) {
9917 CurCaptureVarInfo.Exprs.push_back(Elt: VD);
9918 CurCaptureVarInfo.BasePointers.emplace_back(Args&: Arg);
9919 CurCaptureVarInfo.DevicePtrDecls.emplace_back(Args&: VD);
9920 CurCaptureVarInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::Pointer);
9921 CurCaptureVarInfo.Pointers.push_back(Elt: Arg);
9922 CurCaptureVarInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9923 V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy), DestTy: CGF.Int64Ty,
9924 /*isSigned=*/true));
9925 CurCaptureVarInfo.Types.push_back(
9926 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9927 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9928 CurCaptureVarInfo.Mappers.push_back(Elt: nullptr);
9929 return;
9930 }
9931
9932 auto GenerateInfoForComponentLists =
9933 [&](ArrayRef<MapData> DeclComponentListsFromClauses,
9934 bool IsEligibleForTargetParamFlag) {
9935 MapCombinedInfoTy CurInfoForComponentLists;
9936 StructRangeInfoTy PartialStruct;
9937 AttachInfoTy AttachInfo;
9938
9939 if (DeclComponentListsFromClauses.empty())
9940 return;
9941
9942 generateInfoForCaptureFromComponentLists(
9943 VD, DeclComponentLists: DeclComponentListsFromClauses, CurComponentListInfo&: CurInfoForComponentLists,
9944 PartialStruct, AttachInfo, IsListEligibleForTargetParamFlag: IsEligibleForTargetParamFlag);
9945
9946 // If there is an entry in PartialStruct it means we have a
9947 // struct with individual members mapped. Emit an extra combined
9948 // entry.
9949 if (PartialStruct.Base.isValid()) {
9950 CurCaptureVarInfo.append(CurInfo&: PartialStruct.PreliminaryMapData);
9951 emitCombinedEntry(
9952 CombinedInfo&: CurCaptureVarInfo, CurTypes&: CurInfoForComponentLists.Types,
9953 PartialStruct, AttachInfo, IsMapThis: Cap->capturesThis(), OMPBuilder,
9954 /*VD=*/nullptr, OffsetForMemberOfFlag,
9955 /*NotTargetParams*/ !IsEligibleForTargetParamFlag);
9956 }
9957
9958 // We do the appends to get the entries in the following order:
9959 // combined-entry -> individual-field-entries -> attach-entry,
9960 CurCaptureVarInfo.append(CurInfo&: CurInfoForComponentLists);
9961 if (AttachInfo.isValid())
9962 emitAttachEntry(CGF, CombinedInfo&: CurCaptureVarInfo, AttachInfo);
9963 };
9964
9965 // Group component lists by their AttachPtrExpr and process them in order
9966 // of increasing complexity (nullptr first, then simple expressions like p,
9967 // then more complex ones like p[0], etc.)
9968 //
9969 // This ensure that we:
9970 // * handle maps that can contribute towards setting the kernel argument,
9971 // (e.g. map(ps), or map(ps[0])), before any that cannot (e.g. ps->pt->d).
9972 // * allocate a single contiguous storage for all exprs with the same
9973 // captured var and having the same attach-ptr.
9974 //
9975 // Example: The map clauses below should be handled grouped together based
9976 // on their attachable-base-pointers:
9977 // map-clause | attachable-base-pointer
9978 // --------------------------+------------------------
9979 // map(p, ps) | nullptr
9980 // map(p[0]) | p
9981 // map(p[0]->b, p[0]->c) | p[0]
9982 // map(ps->d, ps->e, ps->pt) | ps
9983 // map(ps->pt->d, ps->pt->e) | ps->pt
9984
9985 // First, collect all MapData entries with their attach-ptr exprs.
9986 SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs;
9987
9988 for (const MapData &L : DeclComponentListsFromClauses) {
9989 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9990 std::get<0>(t: L);
9991 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
9992 AttachPtrMapDataPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
9993 }
9994
9995 // Next, sort by increasing order of their complexity.
9996 llvm::stable_sort(Range&: AttachPtrMapDataPairs,
9997 C: [this](const auto &LHS, const auto &RHS) {
9998 return AttachPtrComparator(LHS.first, RHS.first);
9999 });
10000
10001 bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty();
10002 bool IsFirstGroup = true;
10003
10004 // And finally, process them all in order, grouping those with
10005 // equivalent attach-ptr exprs together.
10006 auto *It = AttachPtrMapDataPairs.begin();
10007 while (It != AttachPtrMapDataPairs.end()) {
10008 const Expr *AttachPtrExpr = It->first;
10009
10010 MapDataArrayTy GroupLists;
10011 while (It != AttachPtrMapDataPairs.end() &&
10012 (It->first == AttachPtrExpr ||
10013 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
10014 GroupLists.push_back(Elt: It->second);
10015 ++It;
10016 }
10017 assert(!GroupLists.empty() && "GroupLists should not be empty");
10018
10019 // Determine if this group of component-lists is eligible for TARGET_PARAM
10020 // flag. Only the first group processed should be eligible, and only if no
10021 // default mapping was done.
10022 bool IsEligibleForTargetParamFlag =
10023 IsFirstGroup && NoDefaultMappingDoneForVD;
10024
10025 GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag);
10026 IsFirstGroup = false;
10027 }
10028 }
10029
10030 /// Generate the base pointers, section pointers, sizes, map types, and
10031 /// mappers associated to \a DeclComponentLists for a given capture
10032 /// \a VD (all included in \a CurComponentListInfo).
10033 void generateInfoForCaptureFromComponentLists(
10034 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
10035 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
10036 AttachInfoTy &AttachInfo, bool IsListEligibleForTargetParamFlag) const {
10037 // Find overlapping elements (including the offset from the base element).
10038 llvm::SmallDenseMap<
10039 const MapData *,
10040 llvm::SmallVector<
10041 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
10042 4>
10043 OverlappedData;
10044 size_t Count = 0;
10045 for (const MapData &L : DeclComponentLists) {
10046 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10047 OpenMPMapClauseKind MapType;
10048 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10049 bool IsImplicit;
10050 const ValueDecl *Mapper;
10051 const Expr *VarRef;
10052 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10053 L;
10054 ++Count;
10055 for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(N: Count)) {
10056 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
10057 std::tie(args&: Components1, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper,
10058 args&: VarRef) = L1;
10059 auto CI = Components.rbegin();
10060 auto CE = Components.rend();
10061 auto SI = Components1.rbegin();
10062 auto SE = Components1.rend();
10063 for (; CI != CE && SI != SE; ++CI, ++SI) {
10064 if (CI->getAssociatedExpression()->getStmtClass() !=
10065 SI->getAssociatedExpression()->getStmtClass())
10066 break;
10067 // Are we dealing with different variables/fields?
10068 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
10069 break;
10070 }
10071 // Found overlapping if, at least for one component, reached the head
10072 // of the components list.
10073 if (CI == CE || SI == SE) {
10074 // Ignore it if it is the same component.
10075 if (CI == CE && SI == SE)
10076 continue;
10077 const auto It = (SI == SE) ? CI : SI;
10078 // If one component is a pointer and another one is a kind of
10079 // dereference of this pointer (array subscript, section, dereference,
10080 // etc.), it is not an overlapping.
10081 // Same, if one component is a base and another component is a
10082 // dereferenced pointer memberexpr with the same base.
10083 if (!isa<MemberExpr>(Val: It->getAssociatedExpression()) ||
10084 (std::prev(x: It)->getAssociatedDeclaration() &&
10085 std::prev(x: It)
10086 ->getAssociatedDeclaration()
10087 ->getType()
10088 ->isPointerType()) ||
10089 (It->getAssociatedDeclaration() &&
10090 It->getAssociatedDeclaration()->getType()->isPointerType() &&
10091 std::next(x: It) != CE && std::next(x: It) != SE))
10092 continue;
10093 const MapData &BaseData = CI == CE ? L : L1;
10094 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
10095 SI == SE ? Components : Components1;
10096 OverlappedData[&BaseData].push_back(Elt: SubData);
10097 }
10098 }
10099 }
10100 // Sort the overlapped elements for each item.
10101 llvm::SmallVector<const FieldDecl *, 4> Layout;
10102 if (!OverlappedData.empty()) {
10103 const Type *BaseType = VD->getType().getCanonicalType().getTypePtr();
10104 const Type *OrigType = BaseType->getPointeeOrArrayElementType();
10105 while (BaseType != OrigType) {
10106 BaseType = OrigType->getCanonicalTypeInternal().getTypePtr();
10107 OrigType = BaseType->getPointeeOrArrayElementType();
10108 }
10109
10110 if (const auto *CRD = BaseType->getAsCXXRecordDecl())
10111 getPlainLayout(RD: CRD, Layout, /*AsBase=*/false);
10112 else {
10113 const auto *RD = BaseType->getAsRecordDecl();
10114 Layout.append(in_start: RD->field_begin(), in_end: RD->field_end());
10115 }
10116 }
10117 for (auto &Pair : OverlappedData) {
10118 llvm::stable_sort(
10119 Range&: Pair.getSecond(),
10120 C: [&Layout](
10121 OMPClauseMappableExprCommon::MappableExprComponentListRef First,
10122 OMPClauseMappableExprCommon::MappableExprComponentListRef
10123 Second) {
10124 auto CI = First.rbegin();
10125 auto CE = First.rend();
10126 auto SI = Second.rbegin();
10127 auto SE = Second.rend();
10128 for (; CI != CE && SI != SE; ++CI, ++SI) {
10129 if (CI->getAssociatedExpression()->getStmtClass() !=
10130 SI->getAssociatedExpression()->getStmtClass())
10131 break;
10132 // Are we dealing with different variables/fields?
10133 if (CI->getAssociatedDeclaration() !=
10134 SI->getAssociatedDeclaration())
10135 break;
10136 }
10137
10138 // Lists contain the same elements.
10139 if (CI == CE && SI == SE)
10140 return false;
10141
10142 // List with less elements is less than list with more elements.
10143 if (CI == CE || SI == SE)
10144 return CI == CE;
10145
10146 const auto *FD1 = cast<FieldDecl>(Val: CI->getAssociatedDeclaration());
10147 const auto *FD2 = cast<FieldDecl>(Val: SI->getAssociatedDeclaration());
10148 if (FD1->getParent() == FD2->getParent())
10149 return FD1->getFieldIndex() < FD2->getFieldIndex();
10150 const auto *It =
10151 llvm::find_if(Range&: Layout, P: [FD1, FD2](const FieldDecl *FD) {
10152 return FD == FD1 || FD == FD2;
10153 });
10154 return *It == FD1;
10155 });
10156 }
10157
10158 // Associated with a capture, because the mapping flags depend on it.
10159 // Go through all of the elements with the overlapped elements.
10160 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
10161 MapCombinedInfoTy StructBaseCombinedInfo;
10162 for (const auto &Pair : OverlappedData) {
10163 const MapData &L = *Pair.getFirst();
10164 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10165 OpenMPMapClauseKind MapType;
10166 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10167 bool IsImplicit;
10168 const ValueDecl *Mapper;
10169 const Expr *VarRef;
10170 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10171 L;
10172 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
10173 OverlappedComponents = Pair.getSecond();
10174 generateInfoForComponentList(
10175 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10176 StructBaseCombinedInfo, PartialStruct, AttachInfo, IsFirstComponentList: AddTargetParamFlag,
10177 IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
10178 /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef, OverlappedElements: OverlappedComponents);
10179 AddTargetParamFlag = false;
10180 }
10181 // Go through other elements without overlapped elements.
10182 for (const MapData &L : DeclComponentLists) {
10183 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10184 OpenMPMapClauseKind MapType;
10185 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10186 bool IsImplicit;
10187 const ValueDecl *Mapper;
10188 const Expr *VarRef;
10189 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10190 L;
10191 auto It = OverlappedData.find(Val: &L);
10192 if (It == OverlappedData.end())
10193 generateInfoForComponentList(
10194 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10195 StructBaseCombinedInfo, PartialStruct, AttachInfo,
10196 IsFirstComponentList: AddTargetParamFlag, IsImplicit, /*GenerateAllInfoForClauses*/ false,
10197 Mapper, /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef,
10198 /*OverlappedElements*/ {});
10199 AddTargetParamFlag = false;
10200 }
10201 }
10202
10203 /// Check if a variable should be treated as firstprivate due to explicit
10204 /// firstprivate clause or defaultmap(firstprivate:...).
10205 bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const {
10206 // Check explicit firstprivate clauses (not implicit from defaultmap)
10207 auto I = FirstPrivateDecls.find(Val: VD);
10208 if (I != FirstPrivateDecls.end() && !I->getSecond())
10209 return true; // Explicit firstprivate only
10210
10211 // Check defaultmap(firstprivate:scalar) for scalar types
10212 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_scalar)) {
10213 if (Type->isScalarType())
10214 return true;
10215 }
10216
10217 // Check defaultmap(firstprivate:pointer) for pointer types
10218 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_pointer)) {
10219 if (Type->isAnyPointerType())
10220 return true;
10221 }
10222
10223 // Check defaultmap(firstprivate:aggregate) for aggregate types
10224 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_aggregate)) {
10225 if (Type->isAggregateType())
10226 return true;
10227 }
10228
10229 // Check defaultmap(firstprivate:all) for all types
10230 return DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_all);
10231 }
10232
10233 /// Generate the default map information for a given capture \a CI,
10234 /// record field declaration \a RI and captured value \a CV.
10235 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
10236 const FieldDecl &RI, llvm::Value *CV,
10237 MapCombinedInfoTy &CombinedInfo) const {
10238 bool IsImplicit = true;
10239 // Do the default mapping.
10240 if (CI.capturesThis()) {
10241 CombinedInfo.Exprs.push_back(Elt: nullptr);
10242 CombinedInfo.BasePointers.push_back(Elt: CV);
10243 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10244 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10245 CombinedInfo.Pointers.push_back(Elt: CV);
10246 const auto *PtrTy = cast<PointerType>(Val: RI.getType().getTypePtr());
10247 CombinedInfo.Sizes.push_back(
10248 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: PtrTy->getPointeeType()),
10249 DestTy: CGF.Int64Ty, /*isSigned=*/true));
10250 // Default map type.
10251 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TO |
10252 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
10253 } else if (CI.capturesVariableByCopy()) {
10254 const VarDecl *VD = CI.getCapturedVar();
10255 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10256 CombinedInfo.BasePointers.push_back(Elt: CV);
10257 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10258 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10259 CombinedInfo.Pointers.push_back(Elt: CV);
10260 bool IsFirstprivate =
10261 isEffectivelyFirstprivate(VD, Type: RI.getType().getNonReferenceType());
10262
10263 if (!RI.getType()->isAnyPointerType()) {
10264 // We have to signal to the runtime captures passed by value that are
10265 // not pointers.
10266 CombinedInfo.Types.push_back(
10267 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10268 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10269 V: CGF.getTypeSize(Ty: RI.getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10270 } else if (IsFirstprivate) {
10271 // Firstprivate pointers should be passed by value (as literals)
10272 // without performing a present table lookup at runtime.
10273 CombinedInfo.Types.push_back(
10274 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10275 // Use zero size for pointer literals (just passing the pointer value)
10276 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10277 } else {
10278 // Pointers are implicitly mapped with a zero size and no flags
10279 // (other than first map that is added for all implicit maps).
10280 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_NONE);
10281 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10282 }
10283 auto I = FirstPrivateDecls.find(Val: VD);
10284 if (I != FirstPrivateDecls.end())
10285 IsImplicit = I->getSecond();
10286 } else {
10287 assert(CI.capturesVariable() && "Expected captured reference.");
10288 const auto *PtrTy = cast<ReferenceType>(Val: RI.getType().getTypePtr());
10289 QualType ElementType = PtrTy->getPointeeType();
10290 const VarDecl *VD = CI.getCapturedVar();
10291 bool IsFirstprivate = isEffectivelyFirstprivate(VD, Type: ElementType);
10292 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10293 CombinedInfo.BasePointers.push_back(Elt: CV);
10294 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10295 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10296
10297 // For firstprivate pointers, pass by value instead of dereferencing
10298 if (IsFirstprivate && ElementType->isAnyPointerType()) {
10299 // Treat as a literal value (pass the pointer value itself)
10300 CombinedInfo.Pointers.push_back(Elt: CV);
10301 // Use zero size for pointer literals
10302 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10303 CombinedInfo.Types.push_back(
10304 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10305 } else {
10306 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10307 V: CGF.getTypeSize(Ty: ElementType), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10308 // The default map type for a scalar/complex type is 'to' because by
10309 // default the value doesn't have to be retrieved. For an aggregate
10310 // type, the default is 'tofrom'.
10311 CombinedInfo.Types.push_back(Elt: getMapModifiersForPrivateClauses(Cap: CI));
10312 CombinedInfo.Pointers.push_back(Elt: CV);
10313 }
10314 auto I = FirstPrivateDecls.find(Val: VD);
10315 if (I != FirstPrivateDecls.end())
10316 IsImplicit = I->getSecond();
10317 }
10318 // Every default map produces a single argument which is a target parameter.
10319 CombinedInfo.Types.back() |=
10320 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
10321
10322 // Add flag stating this is an implicit map.
10323 if (IsImplicit)
10324 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
10325
10326 // No user-defined mapper for default mapping.
10327 CombinedInfo.Mappers.push_back(Elt: nullptr);
10328 }
10329};
10330} // anonymous namespace
10331
10332// Try to extract the base declaration from a `this->x` expression if possible.
10333static ValueDecl *getDeclFromThisExpr(const Expr *E) {
10334 if (!E)
10335 return nullptr;
10336
10337 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenCasts()))
10338 if (const MemberExpr *ME =
10339 dyn_cast<MemberExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))
10340 return ME->getMemberDecl();
10341 return nullptr;
10342}
10343
10344/// Emit a string constant containing the names of the values mapped to the
10345/// offloading runtime library.
10346static llvm::Constant *
10347emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
10348 MappableExprsHandler::MappingExprInfo &MapExprs) {
10349
10350 uint32_t SrcLocStrSize;
10351 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
10352 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10353
10354 SourceLocation Loc;
10355 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
10356 if (const ValueDecl *VD = getDeclFromThisExpr(E: MapExprs.getMapExpr()))
10357 Loc = VD->getLocation();
10358 else
10359 Loc = MapExprs.getMapExpr()->getExprLoc();
10360 } else {
10361 Loc = MapExprs.getMapDecl()->getLocation();
10362 }
10363
10364 std::string ExprName;
10365 if (MapExprs.getMapExpr()) {
10366 PrintingPolicy P(CGF.getContext().getLangOpts());
10367 llvm::raw_string_ostream OS(ExprName);
10368 MapExprs.getMapExpr()->printPretty(OS, Helper: nullptr, Policy: P);
10369 } else {
10370 ExprName = MapExprs.getMapDecl()->getNameAsString();
10371 }
10372
10373 std::string FileName;
10374 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
10375 if (auto *DbgInfo = CGF.getDebugInfo())
10376 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
10377 else
10378 FileName = PLoc.getFilename();
10379 return OMPBuilder.getOrCreateSrcLocStr(FunctionName: FileName, FileName: ExprName, Line: PLoc.getLine(),
10380 Column: PLoc.getColumn(), SrcLocStrSize);
10381}
10382/// Emit the arrays used to pass the captures and map information to the
10383/// offloading runtime library. If there is no map or capture information,
10384/// return nullptr by reference.
10385static void emitOffloadingArraysAndArgs(
10386 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10387 CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder,
10388 bool IsNonContiguous = false, bool ForEndCall = false) {
10389 CodeGenModule &CGM = CGF.CGM;
10390
10391 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10392 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
10393 CGF.AllocaInsertPt->getIterator());
10394 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
10395 CGF.Builder.GetInsertPoint());
10396
10397 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
10398 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10399 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
10400 }
10401 };
10402
10403 auto CustomMapperCB = [&](unsigned int I) {
10404 llvm::Function *MFunc = nullptr;
10405 if (CombinedInfo.Mappers[I]) {
10406 Info.HasMapper = true;
10407 MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
10408 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10409 }
10410 return MFunc;
10411 };
10412 cantFail(Err: OMPBuilder.emitOffloadingArraysAndArgs(
10413 AllocaIP, CodeGenIP, Info, RTArgs&: Info.RTArgs, CombinedInfo, CustomMapperCB,
10414 IsNonContiguous, ForEndCall, DeviceAddrCB));
10415}
10416
10417/// Check for inner distribute directive.
10418static const OMPExecutableDirective *
10419getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
10420 const auto *CS = D.getInnermostCapturedStmt();
10421 const auto *Body =
10422 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
10423 const Stmt *ChildStmt =
10424 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10425
10426 if (const auto *NestedDir =
10427 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10428 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
10429 switch (D.getDirectiveKind()) {
10430 case OMPD_target:
10431 // For now, treat 'target' with nested 'teams loop' as if it's
10432 // distributed (target teams distribute).
10433 if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop)
10434 return NestedDir;
10435 if (DKind == OMPD_teams) {
10436 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
10437 /*IgnoreCaptured=*/true);
10438 if (!Body)
10439 return nullptr;
10440 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10441 if (const auto *NND =
10442 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10443 DKind = NND->getDirectiveKind();
10444 if (isOpenMPDistributeDirective(DKind))
10445 return NND;
10446 }
10447 }
10448 return nullptr;
10449 case OMPD_target_teams:
10450 if (isOpenMPDistributeDirective(DKind))
10451 return NestedDir;
10452 return nullptr;
10453 case OMPD_target_parallel:
10454 case OMPD_target_simd:
10455 case OMPD_target_parallel_for:
10456 case OMPD_target_parallel_for_simd:
10457 return nullptr;
10458 case OMPD_target_teams_distribute:
10459 case OMPD_target_teams_distribute_simd:
10460 case OMPD_target_teams_distribute_parallel_for:
10461 case OMPD_target_teams_distribute_parallel_for_simd:
10462 case OMPD_parallel:
10463 case OMPD_for:
10464 case OMPD_parallel_for:
10465 case OMPD_parallel_master:
10466 case OMPD_parallel_sections:
10467 case OMPD_for_simd:
10468 case OMPD_parallel_for_simd:
10469 case OMPD_cancel:
10470 case OMPD_cancellation_point:
10471 case OMPD_ordered:
10472 case OMPD_threadprivate:
10473 case OMPD_allocate:
10474 case OMPD_task:
10475 case OMPD_simd:
10476 case OMPD_tile:
10477 case OMPD_unroll:
10478 case OMPD_sections:
10479 case OMPD_section:
10480 case OMPD_single:
10481 case OMPD_master:
10482 case OMPD_critical:
10483 case OMPD_taskyield:
10484 case OMPD_barrier:
10485 case OMPD_taskwait:
10486 case OMPD_taskgroup:
10487 case OMPD_atomic:
10488 case OMPD_flush:
10489 case OMPD_depobj:
10490 case OMPD_scan:
10491 case OMPD_teams:
10492 case OMPD_target_data:
10493 case OMPD_target_exit_data:
10494 case OMPD_target_enter_data:
10495 case OMPD_distribute:
10496 case OMPD_distribute_simd:
10497 case OMPD_distribute_parallel_for:
10498 case OMPD_distribute_parallel_for_simd:
10499 case OMPD_teams_distribute:
10500 case OMPD_teams_distribute_simd:
10501 case OMPD_teams_distribute_parallel_for:
10502 case OMPD_teams_distribute_parallel_for_simd:
10503 case OMPD_target_update:
10504 case OMPD_declare_simd:
10505 case OMPD_declare_variant:
10506 case OMPD_begin_declare_variant:
10507 case OMPD_end_declare_variant:
10508 case OMPD_declare_target:
10509 case OMPD_end_declare_target:
10510 case OMPD_declare_reduction:
10511 case OMPD_declare_mapper:
10512 case OMPD_taskloop:
10513 case OMPD_taskloop_simd:
10514 case OMPD_master_taskloop:
10515 case OMPD_master_taskloop_simd:
10516 case OMPD_parallel_master_taskloop:
10517 case OMPD_parallel_master_taskloop_simd:
10518 case OMPD_requires:
10519 case OMPD_metadirective:
10520 case OMPD_unknown:
10521 default:
10522 llvm_unreachable("Unexpected directive.");
10523 }
10524 }
10525
10526 return nullptr;
10527}
10528
10529/// Emit the user-defined mapper function. The code generation follows the
10530/// pattern in the example below.
10531/// \code
10532/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
10533/// void *base, void *begin,
10534/// int64_t size, int64_t type,
10535/// void *name = nullptr) {
10536/// // Allocate space for an array section first.
10537/// if ((size > 1 || (base != begin)) && !maptype.IsDelete)
10538/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10539/// size*sizeof(Ty), clearToFromMember(type));
10540/// // Map members.
10541/// for (unsigned i = 0; i < size; i++) {
10542/// // For each component specified by this mapper:
10543/// for (auto c : begin[i]->all_components) {
10544/// if (c.hasMapper())
10545/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
10546/// c.arg_type, c.arg_name);
10547/// else
10548/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
10549/// c.arg_begin, c.arg_size, c.arg_type,
10550/// c.arg_name);
10551/// }
10552/// }
10553/// // Delete the array section.
10554/// if (size > 1 && maptype.IsDelete)
10555/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10556/// size*sizeof(Ty), clearToFromMember(type));
10557/// }
10558/// \endcode
10559void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
10560 CodeGenFunction *CGF) {
10561 if (UDMMap.count(Val: D) > 0)
10562 return;
10563 ASTContext &C = CGM.getContext();
10564 QualType Ty = D->getType();
10565 auto *MapperVarDecl =
10566 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getMapperVarRef())->getDecl());
10567 CharUnits ElementSize = C.getTypeSizeInChars(T: Ty);
10568 llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
10569
10570 CodeGenFunction MapperCGF(CGM);
10571 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10572 auto PrivatizeAndGenMapInfoCB =
10573 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10574 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10575 MapperCGF.Builder.restoreIP(IP: CodeGenIP);
10576
10577 // Privatize the declared variable of mapper to be the current array
10578 // element.
10579 Address PtrCurrent(
10580 PtrPHI, ElemTy,
10581 Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign())
10582 .getAlignment()
10583 .alignmentOfArrayElement(elementSize: ElementSize));
10584 CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
10585 Scope.addPrivate(LocalVD: MapperVarDecl, Addr: PtrCurrent);
10586 (void)Scope.Privatize();
10587
10588 // Get map clause information.
10589 MappableExprsHandler MEHandler(*D, MapperCGF);
10590 MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder);
10591
10592 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10593 return emitMappingInformation(CGF&: MapperCGF, OMPBuilder, MapExprs&: MapExpr);
10594 };
10595 if (CGM.getCodeGenOpts().getDebugInfo() !=
10596 llvm::codegenoptions::NoDebugInfo) {
10597 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10598 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10599 F: FillInfoMap);
10600 }
10601
10602 return CombinedInfo;
10603 };
10604
10605 auto CustomMapperCB = [&](unsigned I) {
10606 llvm::Function *MapperFunc = nullptr;
10607 if (CombinedInfo.Mappers[I]) {
10608 // Call the corresponding mapper function.
10609 MapperFunc = getOrCreateUserDefinedMapperFunc(
10610 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10611 assert(MapperFunc && "Expect a valid mapper function is available.");
10612 }
10613 return MapperFunc;
10614 };
10615
10616 SmallString<64> TyStr;
10617 llvm::raw_svector_ostream Out(TyStr);
10618 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out);
10619 std::string Name = getName(Parts: {"omp_mapper", TyStr, D->getName()});
10620
10621 llvm::Function *NewFn = cantFail(ValOrErr: OMPBuilder.emitUserDefinedMapper(
10622 PrivAndGenMapInfoCB: PrivatizeAndGenMapInfoCB, ElemTy, FuncName: Name, CustomMapperCB));
10623 UDMMap.try_emplace(Key: D, Args&: NewFn);
10624 if (CGF)
10625 FunctionUDMMap[CGF->CurFn].push_back(Elt: D);
10626}
10627
10628llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc(
10629 const OMPDeclareMapperDecl *D) {
10630 auto I = UDMMap.find(Val: D);
10631 if (I != UDMMap.end())
10632 return I->second;
10633 emitUserDefinedMapper(D);
10634 return UDMMap.lookup(Val: D);
10635}
10636
10637llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall(
10638 CodeGenFunction &CGF, const OMPExecutableDirective &D,
10639 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10640 const OMPLoopDirective &D)>
10641 SizeEmitter) {
10642 OpenMPDirectiveKind Kind = D.getDirectiveKind();
10643 const OMPExecutableDirective *TD = &D;
10644 // Get nested teams distribute kind directive, if any. For now, treat
10645 // 'target_teams_loop' as if it's really a target_teams_distribute.
10646 if ((!isOpenMPDistributeDirective(DKind: Kind) || !isOpenMPTeamsDirective(DKind: Kind)) &&
10647 Kind != OMPD_target_teams_loop)
10648 TD = getNestedDistributeDirective(Ctx&: CGM.getContext(), D);
10649 if (!TD)
10650 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10651
10652 const auto *LD = cast<OMPLoopDirective>(Val: TD);
10653 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
10654 return NumIterations;
10655 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10656}
10657
10658static void
10659emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10660 const OMPExecutableDirective &D,
10661 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10662 bool RequiresOuterTask, const CapturedStmt &CS,
10663 bool OffloadingMandatory, CodeGenFunction &CGF) {
10664 if (OffloadingMandatory) {
10665 CGF.Builder.CreateUnreachable();
10666 } else {
10667 if (RequiresOuterTask) {
10668 CapturedVars.clear();
10669 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
10670 }
10671 llvm::SmallVector<llvm::Value *, 16> Args(CapturedVars.begin(),
10672 CapturedVars.end());
10673 Args.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy()));
10674 OMPRuntime->emitOutlinedFunctionCall(CGF, Loc: D.getBeginLoc(), OutlinedFn,
10675 Args);
10676 }
10677}
10678
10679static llvm::Value *emitDeviceID(
10680 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
10681 CodeGenFunction &CGF) {
10682 // Emit device ID if any.
10683 llvm::Value *DeviceID;
10684 if (Device.getPointer()) {
10685 assert((Device.getInt() == OMPC_DEVICE_unknown ||
10686 Device.getInt() == OMPC_DEVICE_device_num) &&
10687 "Expected device_num modifier.");
10688 llvm::Value *DevVal = CGF.EmitScalarExpr(E: Device.getPointer());
10689 DeviceID =
10690 CGF.Builder.CreateIntCast(V: DevVal, DestTy: CGF.Int64Ty, /*isSigned=*/true);
10691 } else {
10692 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
10693 }
10694 return DeviceID;
10695}
10696
10697static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>
10698emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) {
10699 llvm::Value *DynGP = CGF.Builder.getInt32(C: 0);
10700 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
10701
10702 if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) {
10703 CodeGenFunction::RunCleanupsScope DynGPScope(CGF);
10704 llvm::Value *DynGPVal =
10705 CGF.EmitScalarExpr(E: DynGPClause->getSize(), /*IgnoreResultAssign=*/true);
10706 DynGP = CGF.Builder.CreateIntCast(V: DynGPVal, DestTy: CGF.Int32Ty,
10707 /*isSigned=*/false);
10708 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();
10709 switch (FallbackModifier) {
10710 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:
10711 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
10712 break;
10713 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:
10714 DynGPFallback = OMPDynGroupprivateFallbackType::Null;
10715 break;
10716 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:
10717 case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown:
10718 // This is the default for dyn_groupprivate.
10719 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;
10720 break;
10721 default:
10722 llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate");
10723 }
10724 } else if (auto *OMPXDynCGClause =
10725 D.getSingleClause<OMPXDynCGroupMemClause>()) {
10726 CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF);
10727 llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(E: OMPXDynCGClause->getSize(),
10728 /*IgnoreResultAssign=*/true);
10729 DynGP = CGF.Builder.CreateIntCast(V: DynCGMemVal, DestTy: CGF.Int32Ty,
10730 /*isSigned=*/false);
10731 }
10732 return {DynGP, DynGPFallback};
10733}
10734
10735static void genMapInfoForCaptures(
10736 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
10737 const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10738 llvm::OpenMPIRBuilder &OMPBuilder,
10739 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet,
10740 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10741
10742 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
10743 auto RI = CS.getCapturedRecordDecl()->field_begin();
10744 auto *CV = CapturedVars.begin();
10745 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
10746 CE = CS.capture_end();
10747 CI != CE; ++CI, ++RI, ++CV) {
10748 MappableExprsHandler::MapCombinedInfoTy CurInfo;
10749
10750 // VLA sizes are passed to the outlined region by copy and do not have map
10751 // information associated.
10752 if (CI->capturesVariableArrayType()) {
10753 CurInfo.Exprs.push_back(Elt: nullptr);
10754 CurInfo.BasePointers.push_back(Elt: *CV);
10755 CurInfo.DevicePtrDecls.push_back(Elt: nullptr);
10756 CurInfo.DevicePointers.push_back(
10757 Elt: MappableExprsHandler::DeviceInfoTy::None);
10758 CurInfo.Pointers.push_back(Elt: *CV);
10759 CurInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10760 V: CGF.getTypeSize(Ty: RI->getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10761 // Copy to the device as an argument. No need to retrieve it.
10762 CurInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10763 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
10764 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
10765 CurInfo.Mappers.push_back(Elt: nullptr);
10766 } else {
10767 const ValueDecl *CapturedVD =
10768 CI->capturesThis() ? nullptr
10769 : CI->getCapturedVar()->getCanonicalDecl();
10770 bool HasEntryWithCVAsAttachPtr = false;
10771 if (CapturedVD)
10772 HasEntryWithCVAsAttachPtr =
10773 MEHandler.hasAttachEntryForCapturedVar(VD: CapturedVD);
10774
10775 // Populate component lists for the captured variable from clauses.
10776 MappableExprsHandler::MapDataArrayTy DeclComponentLists;
10777 SmallVector<
10778 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>, 4>
10779 StorageForImplicitlyAddedComponentLists;
10780 MEHandler.populateComponentListsForNonLambdaCaptureFromClauses(
10781 VD: CapturedVD, DeclComponentLists,
10782 StorageForImplicitlyAddedComponentLists);
10783
10784 // OpenMP 6.0, 15.8, target construct, restrictions:
10785 // * A list item in a map clause that is specified on a target construct
10786 // must have a base variable or base pointer.
10787 //
10788 // Map clauses on a target construct must either have a base pointer, or a
10789 // base-variable. So, if we don't have a base-pointer, that means that it
10790 // must have a base-variable, i.e. we have a map like `map(s)`, `map(s.x)`
10791 // etc. In such cases, we do not need to handle default map generation
10792 // for `s`.
10793 bool HasEntryWithoutAttachPtr =
10794 llvm::any_of(Range&: DeclComponentLists, P: [&](const auto &MapData) {
10795 OMPClauseMappableExprCommon::MappableExprComponentListRef
10796 Components = std::get<0>(MapData);
10797 return !MEHandler.getAttachPtrExpr(Components);
10798 });
10799
10800 // Generate default map info first if there's no direct map with CV as
10801 // the base-variable, or attach pointer.
10802 if (DeclComponentLists.empty() ||
10803 (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr))
10804 MEHandler.generateDefaultMapInfo(CI: *CI, RI: **RI, CV: *CV, CombinedInfo&: CurInfo);
10805
10806 // If we have any information in the map clause, we use it, otherwise we
10807 // just do a default mapping.
10808 MEHandler.generateInfoForCaptureFromClauseInfo(
10809 DeclComponentListsFromClauses: DeclComponentLists, Cap: CI, Arg: *CV, CurCaptureVarInfo&: CurInfo, OMPBuilder,
10810 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());
10811
10812 if (!CI->capturesThis())
10813 MappedVarSet.insert(V: CI->getCapturedVar());
10814 else
10815 MappedVarSet.insert(V: nullptr);
10816
10817 // Generate correct mapping for variables captured by reference in
10818 // lambdas.
10819 if (CI->capturesVariable())
10820 MEHandler.generateInfoForLambdaCaptures(VD: CI->getCapturedVar(), Arg: *CV,
10821 CombinedInfo&: CurInfo, LambdaPointers);
10822 }
10823 // We expect to have at least an element of information for this capture.
10824 assert(!CurInfo.BasePointers.empty() &&
10825 "Non-existing map pointer for capture!");
10826 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
10827 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
10828 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
10829 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
10830 "Inconsistent map information sizes!");
10831
10832 // We need to append the results of this capture to what we already have.
10833 CombinedInfo.append(CurInfo);
10834 }
10835 // Adjust MEMBER_OF flags for the lambdas captures.
10836 MEHandler.adjustMemberOfForLambdaCaptures(
10837 OMPBuilder, LambdaPointers, BasePointers&: CombinedInfo.BasePointers,
10838 Pointers&: CombinedInfo.Pointers, Types&: CombinedInfo.Types);
10839}
10840static void
10841genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
10842 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10843 llvm::OpenMPIRBuilder &OMPBuilder,
10844 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet =
10845 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) {
10846
10847 CodeGenModule &CGM = CGF.CGM;
10848 // Map any list items in a map clause that were not captures because they
10849 // weren't referenced within the construct.
10850 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkipVarSet: SkippedVarSet);
10851
10852 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10853 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
10854 };
10855 if (CGM.getCodeGenOpts().getDebugInfo() !=
10856 llvm::codegenoptions::NoDebugInfo) {
10857 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10858 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10859 F: FillInfoMap);
10860 }
10861}
10862
10863static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
10864 const CapturedStmt &CS,
10865 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10866 llvm::OpenMPIRBuilder &OMPBuilder,
10867 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10868 // Get mappable expression information.
10869 MappableExprsHandler MEHandler(D, CGF);
10870 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
10871
10872 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,
10873 MappedVarSet, CombinedInfo);
10874 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, SkippedVarSet: MappedVarSet);
10875}
10876
10877template <typename ClauseTy>
10878static void
10879emitClauseForBareTargetDirective(CodeGenFunction &CGF,
10880 const OMPExecutableDirective &D,
10881 llvm::SmallVectorImpl<llvm::Value *> &Values) {
10882 const auto *C = D.getSingleClause<ClauseTy>();
10883 assert(!C->varlist_empty() &&
10884 "ompx_bare requires explicit num_teams and thread_limit");
10885 CodeGenFunction::RunCleanupsScope Scope(CGF);
10886 for (auto *E : C->varlist()) {
10887 llvm::Value *V = CGF.EmitScalarExpr(E);
10888 Values.push_back(
10889 Elt: CGF.Builder.CreateIntCast(V, DestTy: CGF.Int32Ty, /*isSigned=*/true));
10890 }
10891}
10892
10893static void emitTargetCallKernelLaunch(
10894 CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10895 const OMPExecutableDirective &D,
10896 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask,
10897 const CapturedStmt &CS, bool OffloadingMandatory,
10898 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
10899 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
10900 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
10901 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10902 const OMPLoopDirective &D)>
10903 SizeEmitter,
10904 CodeGenFunction &CGF, CodeGenModule &CGM) {
10905 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();
10906
10907 // Fill up the arrays with all the captured variables.
10908 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10909 CGOpenMPRuntime::TargetDataInfo Info;
10910 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
10911
10912 // Append a null entry for the implicit dyn_ptr argument.
10913 using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
10914 auto *NullPtr = llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy());
10915 CombinedInfo.BasePointers.push_back(Elt: NullPtr);
10916 CombinedInfo.Pointers.push_back(Elt: NullPtr);
10917 CombinedInfo.DevicePointers.push_back(
10918 Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
10919 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.getInt64(C: 0));
10920 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
10921 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10922 if (!CombinedInfo.Names.empty())
10923 CombinedInfo.Names.push_back(Elt: NullPtr);
10924 CombinedInfo.Exprs.push_back(Elt: nullptr);
10925 CombinedInfo.Mappers.push_back(Elt: nullptr);
10926 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10927
10928 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
10929 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
10930
10931 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
10932 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
10933 CGF.VoidPtrTy, CGM.getPointerAlign());
10934 InputInfo.PointersArray =
10935 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
10936 InputInfo.SizesArray =
10937 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
10938 InputInfo.MappersArray =
10939 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
10940 MapTypesArray = Info.RTArgs.MapTypesArray;
10941 MapNamesArray = Info.RTArgs.MapNamesArray;
10942
10943 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
10944 RequiresOuterTask, &CS, OffloadingMandatory, Device,
10945 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
10946 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
10947 bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
10948
10949 if (IsReverseOffloading) {
10950 // Reverse offloading is not supported, so just execute on the host.
10951 // FIXME: This fallback solution is incorrect since it ignores the
10952 // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to
10953 // assert here and ensure SEMA emits an error.
10954 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
10955 RequiresOuterTask, CS, OffloadingMandatory, CGF);
10956 return;
10957 }
10958
10959 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
10960 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
10961
10962 llvm::Value *BasePointersArray =
10963 InputInfo.BasePointersArray.emitRawPointer(CGF);
10964 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
10965 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
10966 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
10967
10968 auto &&EmitTargetCallFallbackCB =
10969 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
10970 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
10971 -> llvm::OpenMPIRBuilder::InsertPointTy {
10972 CGF.Builder.restoreIP(IP);
10973 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
10974 RequiresOuterTask, CS, OffloadingMandatory, CGF);
10975 return CGF.Builder.saveIP();
10976 };
10977
10978 bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
10979 SmallVector<llvm::Value *, 3> NumTeams;
10980 SmallVector<llvm::Value *, 3> NumThreads;
10981 if (IsBare) {
10982 emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, Values&: NumTeams);
10983 emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
10984 Values&: NumThreads);
10985 } else {
10986 NumTeams.push_back(Elt: OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
10987 NumThreads.push_back(
10988 Elt: OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
10989 }
10990
10991 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
10992 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, Loc: D.getBeginLoc());
10993 llvm::Value *NumIterations =
10994 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
10995 auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF);
10996 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
10997 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
10998
10999 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
11000 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
11001 nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray);
11002
11003 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
11004 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
11005 DynCGroupMem, HasNoWait, /*StrictBlocksAndThreads=*/IsBare,
11006 DynCGroupMemFallback);
11007
11008 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11009 cantFail(ValOrErr: OMPRuntime->getOMPBuilder().emitKernelLaunch(
11010 Loc: CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
11011 RTLoc, AllocaIP));
11012 CGF.Builder.restoreIP(IP: AfterIP);
11013 };
11014
11015 if (RequiresOuterTask)
11016 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11017 else
11018 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11019}
11020
11021static void
11022emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11023 const OMPExecutableDirective &D,
11024 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11025 bool RequiresOuterTask, const CapturedStmt &CS,
11026 bool OffloadingMandatory, CodeGenFunction &CGF) {
11027
11028 // Notify that the host version must be executed.
11029 auto &&ElseGen =
11030 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11031 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11032 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11033 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11034 };
11035
11036 if (RequiresOuterTask) {
11037 CodeGenFunction::OMPTargetDataInfo InputInfo;
11038 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ElseGen, InputInfo);
11039 } else {
11040 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ElseGen);
11041 }
11042}
11043
11044void CGOpenMPRuntime::emitTargetCall(
11045 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11046 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11047 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11048 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11049 const OMPLoopDirective &D)>
11050 SizeEmitter) {
11051 if (!CGF.HaveInsertPoint())
11052 return;
11053
11054 const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice &&
11055 CGM.getLangOpts().OpenMPOffloadMandatory;
11056
11057 assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
11058
11059 const bool RequiresOuterTask =
11060 D.hasClausesOfKind<OMPDependClause>() ||
11061 D.hasClausesOfKind<OMPNowaitClause>() ||
11062 D.hasClausesOfKind<OMPInReductionClause>() ||
11063 (CGM.getLangOpts().OpenMP >= 51 &&
11064 needsTaskBasedThreadLimit(DKind: D.getDirectiveKind()) &&
11065 D.hasClausesOfKind<OMPThreadLimitClause>());
11066 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
11067 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
11068 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
11069 PrePostActionTy &) {
11070 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
11071 };
11072 emitInlinedDirective(CGF, InnerKind: OMPD_unknown, CodeGen: ArgsCodegen);
11073
11074 CodeGenFunction::OMPTargetDataInfo InputInfo;
11075 llvm::Value *MapTypesArray = nullptr;
11076 llvm::Value *MapNamesArray = nullptr;
11077
11078 auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,
11079 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11080 OutlinedFnID, &InputInfo, &MapTypesArray,
11081 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
11082 PrePostActionTy &) {
11083 emitTargetCallKernelLaunch(OMPRuntime: this, OutlinedFn, D, CapturedVars,
11084 RequiresOuterTask, CS, OffloadingMandatory,
11085 Device, OutlinedFnID, InputInfo, MapTypesArray,
11086 MapNamesArray, SizeEmitter, CGF, CGM);
11087 };
11088
11089 auto &&TargetElseGen =
11090 [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11091 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11092 emitTargetCallElse(OMPRuntime: this, OutlinedFn, D, CapturedVars, RequiresOuterTask,
11093 CS, OffloadingMandatory, CGF);
11094 };
11095
11096 // If we have a target function ID it means that we need to support
11097 // offloading, otherwise, just execute on the host. We need to execute on host
11098 // regardless of the conditional in the if clause if, e.g., the user do not
11099 // specify target triples.
11100 if (OutlinedFnID) {
11101 if (IfCond) {
11102 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen, ElseGen: TargetElseGen);
11103 } else {
11104 RegionCodeGenTy ThenRCG(TargetThenGen);
11105 ThenRCG(CGF);
11106 }
11107 } else {
11108 RegionCodeGenTy ElseRCG(TargetElseGen);
11109 ElseRCG(CGF);
11110 }
11111}
11112
11113void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
11114 StringRef ParentName) {
11115 if (!S)
11116 return;
11117
11118 // Register vtable from device for target data and target directives.
11119 // Add this block here since scanForTargetRegionsFunctions ignores
11120 // target data by checking if S is a executable directive (target).
11121 if (auto *E = dyn_cast<OMPExecutableDirective>(Val: S);
11122 E && isOpenMPTargetDataManagementDirective(DKind: E->getDirectiveKind())) {
11123 // Don't need to check if it's device compile
11124 // since scanForTargetRegionsFunctions currently only called
11125 // in device compilation.
11126 registerVTable(D: *E);
11127 }
11128
11129 // Codegen OMP target directives that offload compute to the device.
11130 bool RequiresDeviceCodegen =
11131 isa<OMPExecutableDirective>(Val: S) &&
11132 isOpenMPTargetExecutionDirective(
11133 DKind: cast<OMPExecutableDirective>(Val: S)->getDirectiveKind());
11134
11135 if (RequiresDeviceCodegen) {
11136 const auto &E = *cast<OMPExecutableDirective>(Val: S);
11137
11138 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
11139 CGM, OMPBuilder, BeginLoc: E.getBeginLoc(), ParentName);
11140
11141 // Is this a target region that should not be emitted as an entry point? If
11142 // so just signal we are done with this target region.
11143 if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
11144 return;
11145
11146 switch (E.getDirectiveKind()) {
11147 case OMPD_target:
11148 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
11149 S: cast<OMPTargetDirective>(Val: E));
11150 break;
11151 case OMPD_target_parallel:
11152 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
11153 CGM, ParentName, S: cast<OMPTargetParallelDirective>(Val: E));
11154 break;
11155 case OMPD_target_teams:
11156 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
11157 CGM, ParentName, S: cast<OMPTargetTeamsDirective>(Val: E));
11158 break;
11159 case OMPD_target_teams_distribute:
11160 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
11161 CGM, ParentName, S: cast<OMPTargetTeamsDistributeDirective>(Val: E));
11162 break;
11163 case OMPD_target_teams_distribute_simd:
11164 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
11165 CGM, ParentName, S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: E));
11166 break;
11167 case OMPD_target_parallel_for:
11168 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
11169 CGM, ParentName, S: cast<OMPTargetParallelForDirective>(Val: E));
11170 break;
11171 case OMPD_target_parallel_for_simd:
11172 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
11173 CGM, ParentName, S: cast<OMPTargetParallelForSimdDirective>(Val: E));
11174 break;
11175 case OMPD_target_simd:
11176 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
11177 CGM, ParentName, S: cast<OMPTargetSimdDirective>(Val: E));
11178 break;
11179 case OMPD_target_teams_distribute_parallel_for:
11180 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
11181 CGM, ParentName,
11182 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: E));
11183 break;
11184 case OMPD_target_teams_distribute_parallel_for_simd:
11185 CodeGenFunction::
11186 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
11187 CGM, ParentName,
11188 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: E));
11189 break;
11190 case OMPD_target_teams_loop:
11191 CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
11192 CGM, ParentName, S: cast<OMPTargetTeamsGenericLoopDirective>(Val: E));
11193 break;
11194 case OMPD_target_parallel_loop:
11195 CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
11196 CGM, ParentName, S: cast<OMPTargetParallelGenericLoopDirective>(Val: E));
11197 break;
11198 case OMPD_parallel:
11199 case OMPD_for:
11200 case OMPD_parallel_for:
11201 case OMPD_parallel_master:
11202 case OMPD_parallel_sections:
11203 case OMPD_for_simd:
11204 case OMPD_parallel_for_simd:
11205 case OMPD_cancel:
11206 case OMPD_cancellation_point:
11207 case OMPD_ordered:
11208 case OMPD_threadprivate:
11209 case OMPD_allocate:
11210 case OMPD_task:
11211 case OMPD_simd:
11212 case OMPD_tile:
11213 case OMPD_unroll:
11214 case OMPD_sections:
11215 case OMPD_section:
11216 case OMPD_single:
11217 case OMPD_master:
11218 case OMPD_critical:
11219 case OMPD_taskyield:
11220 case OMPD_barrier:
11221 case OMPD_taskwait:
11222 case OMPD_taskgroup:
11223 case OMPD_atomic:
11224 case OMPD_flush:
11225 case OMPD_depobj:
11226 case OMPD_scan:
11227 case OMPD_teams:
11228 case OMPD_target_data:
11229 case OMPD_target_exit_data:
11230 case OMPD_target_enter_data:
11231 case OMPD_distribute:
11232 case OMPD_distribute_simd:
11233 case OMPD_distribute_parallel_for:
11234 case OMPD_distribute_parallel_for_simd:
11235 case OMPD_teams_distribute:
11236 case OMPD_teams_distribute_simd:
11237 case OMPD_teams_distribute_parallel_for:
11238 case OMPD_teams_distribute_parallel_for_simd:
11239 case OMPD_target_update:
11240 case OMPD_declare_simd:
11241 case OMPD_declare_variant:
11242 case OMPD_begin_declare_variant:
11243 case OMPD_end_declare_variant:
11244 case OMPD_declare_target:
11245 case OMPD_end_declare_target:
11246 case OMPD_declare_reduction:
11247 case OMPD_declare_mapper:
11248 case OMPD_taskloop:
11249 case OMPD_taskloop_simd:
11250 case OMPD_master_taskloop:
11251 case OMPD_master_taskloop_simd:
11252 case OMPD_parallel_master_taskloop:
11253 case OMPD_parallel_master_taskloop_simd:
11254 case OMPD_requires:
11255 case OMPD_metadirective:
11256 case OMPD_unknown:
11257 default:
11258 llvm_unreachable("Unknown target directive for OpenMP device codegen.");
11259 }
11260 return;
11261 }
11262
11263 if (const auto *E = dyn_cast<OMPExecutableDirective>(Val: S)) {
11264 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
11265 return;
11266
11267 scanForTargetRegionsFunctions(S: E->getRawStmt(), ParentName);
11268 return;
11269 }
11270
11271 // If this is a lambda function, look into its body.
11272 if (const auto *L = dyn_cast<LambdaExpr>(Val: S))
11273 S = L->getBody();
11274
11275 // Keep looking for target regions recursively.
11276 for (const Stmt *II : S->children())
11277 scanForTargetRegionsFunctions(S: II, ParentName);
11278}
11279
11280static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) {
11281 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
11282 OMPDeclareTargetDeclAttr::getDeviceType(VD);
11283 if (!DevTy)
11284 return false;
11285 // Do not emit device_type(nohost) functions for the host.
11286 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
11287 return true;
11288 // Do not emit device_type(host) functions for the device.
11289 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
11290 return true;
11291 return false;
11292}
11293
11294bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
11295 // If emitting code for the host, we do not process FD here. Instead we do
11296 // the normal code generation.
11297 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
11298 if (const auto *FD = dyn_cast<FunctionDecl>(Val: GD.getDecl()))
11299 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11300 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11301 return true;
11302 return false;
11303 }
11304
11305 const ValueDecl *VD = cast<ValueDecl>(Val: GD.getDecl());
11306 // Try to detect target regions in the function.
11307 if (const auto *FD = dyn_cast<FunctionDecl>(Val: VD)) {
11308 StringRef Name = CGM.getMangledName(GD);
11309 scanForTargetRegionsFunctions(S: FD->getBody(), ParentName: Name);
11310 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11311 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11312 return true;
11313 }
11314
11315 // Do not emit function if it is not marked as declare target.
11316 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
11317 AlreadyEmittedTargetDecls.count(V: VD) == 0;
11318}
11319
11320bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
11321 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: GD.getDecl()),
11322 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11323 return true;
11324
11325 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
11326 return false;
11327
11328 // Check if there are Ctors/Dtors in this declaration and look for target
11329 // regions in it. We use the complete variant to produce the kernel name
11330 // mangling.
11331 QualType RDTy = cast<VarDecl>(Val: GD.getDecl())->getType();
11332 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
11333 for (const CXXConstructorDecl *Ctor : RD->ctors()) {
11334 StringRef ParentName =
11335 CGM.getMangledName(GD: GlobalDecl(Ctor, Ctor_Complete));
11336 scanForTargetRegionsFunctions(S: Ctor->getBody(), ParentName);
11337 }
11338 if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
11339 StringRef ParentName =
11340 CGM.getMangledName(GD: GlobalDecl(Dtor, Dtor_Complete));
11341 scanForTargetRegionsFunctions(S: Dtor->getBody(), ParentName);
11342 }
11343 }
11344
11345 // Do not emit variable if it is not marked as declare target.
11346 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11347 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
11348 VD: cast<VarDecl>(Val: GD.getDecl()));
11349 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
11350 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11351 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11352 HasRequiresUnifiedSharedMemory)) {
11353 DeferredGlobalVariables.insert(V: cast<VarDecl>(Val: GD.getDecl()));
11354 return true;
11355 }
11356 return false;
11357}
11358
11359void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
11360 llvm::Constant *Addr) {
11361 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
11362 !CGM.getLangOpts().OpenMPIsTargetDevice)
11363 return;
11364
11365 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11366 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11367
11368 // If this is an 'extern' declaration we defer to the canonical definition and
11369 // do not emit an offloading entry.
11370 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
11371 VD->hasExternalStorage())
11372 return;
11373
11374 // MT_Local variables use direct access with no host-device mapping.
11375 // No offload entry needed — the device global keeps its own initializer.
11376 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local)
11377 return;
11378
11379 if (!Res) {
11380 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11381 // Register non-target variables being emitted in device code (debug info
11382 // may cause this).
11383 StringRef VarName = CGM.getMangledName(GD: VD);
11384 EmittedNonTargetVariables.try_emplace(Key: VarName, Args&: Addr);
11385 }
11386 return;
11387 }
11388
11389 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
11390 auto LinkageForVariable = [&VD, this]() {
11391 return CGM.getLLVMLinkageVarDefinition(VD);
11392 };
11393
11394 std::vector<llvm::GlobalVariable *> GeneratedRefs;
11395 OMPBuilder.registerTargetGlobalVariable(
11396 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
11397 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
11398 IsExternallyVisible: VD->isExternallyVisible(),
11399 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
11400 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
11401 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
11402 TargetTriple: CGM.getLangOpts().OMPTargetTriples, GlobalInitializer: AddrOfGlobal, VariableLinkage: LinkageForVariable,
11403 LlvmPtrTy: CGM.getTypes().ConvertTypeForMem(
11404 T: CGM.getContext().getPointerType(T: VD->getType())),
11405 Addr);
11406
11407 for (auto *ref : GeneratedRefs)
11408 CGM.addCompilerUsedGlobal(GV: ref);
11409}
11410
11411bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
11412 if (isa<FunctionDecl>(Val: GD.getDecl()) ||
11413 isa<OMPDeclareReductionDecl>(Val: GD.getDecl()))
11414 return emitTargetFunctions(GD);
11415
11416 return emitTargetGlobalVariable(GD);
11417}
11418
11419void CGOpenMPRuntime::emitDeferredTargetDecls() const {
11420 for (const VarDecl *VD : DeferredGlobalVariables) {
11421 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11422 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11423 if (!Res)
11424 continue;
11425 // MT_Local and MT_To/MT_Enter without USM are always emitted.
11426 if (*Res == OMPDeclareTargetDeclAttr::MT_Local ||
11427 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11428 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11429 !HasRequiresUnifiedSharedMemory)) {
11430 CGM.EmitGlobal(D: VD);
11431 } else {
11432 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
11433 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11434 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
11435 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
11436 HasRequiresUnifiedSharedMemory)) &&
11437 "Expected link clause or to clause with unified memory.");
11438 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
11439 }
11440 }
11441}
11442
11443void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
11444 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
11445 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
11446 " Expected target-based directive.");
11447}
11448
11449void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) {
11450 for (const OMPClause *Clause : D->clauselists()) {
11451 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
11452 HasRequiresUnifiedSharedMemory = true;
11453 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
11454 } else if (const auto *AC =
11455 dyn_cast<OMPAtomicDefaultMemOrderClause>(Val: Clause)) {
11456 switch (AC->getAtomicDefaultMemOrderKind()) {
11457 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
11458 RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;
11459 break;
11460 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
11461 RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
11462 break;
11463 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
11464 RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
11465 break;
11466 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown:
11467 break;
11468 }
11469 }
11470 }
11471}
11472
11473llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {
11474 return RequiresAtomicOrdering;
11475}
11476
11477bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
11478 LangAS &AS) {
11479 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
11480 return false;
11481 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
11482 switch(A->getAllocatorType()) {
11483 case OMPAllocateDeclAttr::OMPNullMemAlloc:
11484 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
11485 // Not supported, fallback to the default mem space.
11486 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
11487 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
11488 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
11489 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
11490 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
11491 case OMPAllocateDeclAttr::OMPConstMemAlloc:
11492 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
11493 AS = LangAS::Default;
11494 return true;
11495 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
11496 llvm_unreachable("Expected predefined allocator for the variables with the "
11497 "static storage.");
11498 }
11499 return false;
11500}
11501
11502bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {
11503 return HasRequiresUnifiedSharedMemory;
11504}
11505
11506CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
11507 CodeGenModule &CGM)
11508 : CGM(CGM) {
11509 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11510 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
11511 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
11512 }
11513}
11514
11515CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
11516 if (CGM.getLangOpts().OpenMPIsTargetDevice)
11517 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
11518}
11519
11520bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
11521 if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal)
11522 return true;
11523
11524 const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
11525 // Do not emit function if it is marked as declare target as it was already
11526 // emitted.
11527 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: D)) {
11528 if (D->hasBody() && AlreadyEmittedTargetDecls.count(V: D) == 0) {
11529 if (auto *F = dyn_cast_or_null<llvm::Function>(
11530 Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD))))
11531 return !F->isDeclaration();
11532 return false;
11533 }
11534 return true;
11535 }
11536
11537 return !AlreadyEmittedTargetDecls.insert(V: D).second;
11538}
11539
11540void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
11541 const OMPExecutableDirective &D,
11542 SourceLocation Loc,
11543 llvm::Function *OutlinedFn,
11544 ArrayRef<llvm::Value *> CapturedVars) {
11545 if (!CGF.HaveInsertPoint())
11546 return;
11547
11548 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11549 CodeGenFunction::RunCleanupsScope Scope(CGF);
11550
11551 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
11552 llvm::Value *Args[] = {
11553 RTLoc,
11554 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
11555 OutlinedFn};
11556 llvm::SmallVector<llvm::Value *, 16> RealArgs;
11557 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
11558 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
11559
11560 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
11561 M&: CGM.getModule(), FnID: OMPRTL___kmpc_fork_teams);
11562 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
11563}
11564
11565void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11566 const Expr *NumTeams,
11567 const Expr *ThreadLimit,
11568 SourceLocation Loc) {
11569 if (!CGF.HaveInsertPoint())
11570 return;
11571
11572 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11573
11574 llvm::Value *NumTeamsVal =
11575 NumTeams
11576 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: NumTeams),
11577 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11578 : CGF.Builder.getInt32(C: 0);
11579
11580 llvm::Value *ThreadLimitVal =
11581 ThreadLimit
11582 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11583 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11584 : CGF.Builder.getInt32(C: 0);
11585
11586 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
11587 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
11588 ThreadLimitVal};
11589 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11590 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_num_teams),
11591 args: PushNumTeamsArgs);
11592}
11593
11594void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
11595 const Expr *ThreadLimit,
11596 SourceLocation Loc) {
11597 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11598 llvm::Value *ThreadLimitVal =
11599 ThreadLimit
11600 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11601 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11602 : CGF.Builder.getInt32(C: 0);
11603
11604 // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit)
11605 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
11606 ThreadLimitVal};
11607 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11608 M&: CGM.getModule(), FnID: OMPRTL___kmpc_set_thread_limit),
11609 args: ThreadLimitArgs);
11610}
11611
11612void CGOpenMPRuntime::emitTargetDataCalls(
11613 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11614 const Expr *Device, const RegionCodeGenTy &CodeGen,
11615 CGOpenMPRuntime::TargetDataInfo &Info) {
11616 if (!CGF.HaveInsertPoint())
11617 return;
11618
11619 // Action used to replace the default codegen action and turn privatization
11620 // off.
11621 PrePostActionTy NoPrivAction;
11622
11623 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
11624
11625 llvm::Value *IfCondVal = nullptr;
11626 if (IfCond)
11627 IfCondVal = CGF.EvaluateExprAsBool(E: IfCond);
11628
11629 // Emit device ID if any.
11630 llvm::Value *DeviceID = nullptr;
11631 if (Device) {
11632 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
11633 DestTy: CGF.Int64Ty, /*isSigned=*/true);
11634 } else {
11635 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11636 }
11637
11638 // Fill up the arrays with all the mapped variables.
11639 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11640 auto GenMapInfoCB =
11641 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
11642 CGF.Builder.restoreIP(IP: CodeGenIP);
11643 // Get map clause information.
11644 MappableExprsHandler MEHandler(D, CGF);
11645 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);
11646
11647 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11648 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
11649 };
11650 if (CGM.getCodeGenOpts().getDebugInfo() !=
11651 llvm::codegenoptions::NoDebugInfo) {
11652 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
11653 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
11654 F: FillInfoMap);
11655 }
11656
11657 return CombinedInfo;
11658 };
11659 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
11660 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
11661 CGF.Builder.restoreIP(IP: CodeGenIP);
11662 switch (BodyGenType) {
11663 case BodyGenTy::Priv:
11664 if (!Info.CaptureDeviceAddrMap.empty())
11665 CodeGen(CGF);
11666 break;
11667 case BodyGenTy::DupNoPriv:
11668 if (!Info.CaptureDeviceAddrMap.empty()) {
11669 CodeGen.setAction(NoPrivAction);
11670 CodeGen(CGF);
11671 }
11672 break;
11673 case BodyGenTy::NoPriv:
11674 if (Info.CaptureDeviceAddrMap.empty()) {
11675 CodeGen.setAction(NoPrivAction);
11676 CodeGen(CGF);
11677 }
11678 break;
11679 }
11680 return InsertPointTy(CGF.Builder.GetInsertBlock(),
11681 CGF.Builder.GetInsertPoint());
11682 };
11683
11684 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
11685 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
11686 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
11687 }
11688 };
11689
11690 auto CustomMapperCB = [&](unsigned int I) {
11691 llvm::Function *MFunc = nullptr;
11692 if (CombinedInfo.Mappers[I]) {
11693 Info.HasMapper = true;
11694 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
11695 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
11696 }
11697 return MFunc;
11698 };
11699
11700 // Source location for the ident struct
11701 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11702
11703 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
11704 CGF.AllocaInsertPt->getIterator());
11705 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
11706 CGF.Builder.GetInsertPoint());
11707 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
11708 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11709 cantFail(ValOrErr: OMPBuilder.createTargetData(
11710 Loc: OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
11711 IfCond: IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
11712 /*MapperFunc=*/nullptr, BodyGenCB: BodyCB, DeviceAddrCB, SrcLocInfo: RTLoc));
11713 CGF.Builder.restoreIP(IP: AfterIP);
11714}
11715
11716void CGOpenMPRuntime::emitTargetDataStandAloneCall(
11717 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11718 const Expr *Device) {
11719 if (!CGF.HaveInsertPoint())
11720 return;
11721
11722 assert((isa<OMPTargetEnterDataDirective>(D) ||
11723 isa<OMPTargetExitDataDirective>(D) ||
11724 isa<OMPTargetUpdateDirective>(D)) &&
11725 "Expecting either target enter, exit data, or update directives.");
11726
11727 CodeGenFunction::OMPTargetDataInfo InputInfo;
11728 llvm::Value *MapTypesArray = nullptr;
11729 llvm::Value *MapNamesArray = nullptr;
11730 // Generate the code for the opening of the data environment.
11731 auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,
11732 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
11733 // Emit device ID if any.
11734 llvm::Value *DeviceID = nullptr;
11735 if (Device) {
11736 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
11737 DestTy: CGF.Int64Ty, /*isSigned=*/true);
11738 } else {
11739 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11740 }
11741
11742 // Emit the number of elements in the offloading arrays.
11743 llvm::Constant *PointerNum =
11744 CGF.Builder.getInt32(C: InputInfo.NumberOfTargetItems);
11745
11746 // Source location for the ident struct
11747 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11748
11749 SmallVector<llvm::Value *, 13> OffloadingArgs(
11750 {RTLoc, DeviceID, PointerNum,
11751 InputInfo.BasePointersArray.emitRawPointer(CGF),
11752 InputInfo.PointersArray.emitRawPointer(CGF),
11753 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
11754 InputInfo.MappersArray.emitRawPointer(CGF)});
11755
11756 // Select the right runtime function call for each standalone
11757 // directive.
11758 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
11759 RuntimeFunction RTLFn;
11760 switch (D.getDirectiveKind()) {
11761 case OMPD_target_enter_data:
11762 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
11763 : OMPRTL___tgt_target_data_begin_mapper;
11764 break;
11765 case OMPD_target_exit_data:
11766 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
11767 : OMPRTL___tgt_target_data_end_mapper;
11768 break;
11769 case OMPD_target_update:
11770 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
11771 : OMPRTL___tgt_target_data_update_mapper;
11772 break;
11773 case OMPD_parallel:
11774 case OMPD_for:
11775 case OMPD_parallel_for:
11776 case OMPD_parallel_master:
11777 case OMPD_parallel_sections:
11778 case OMPD_for_simd:
11779 case OMPD_parallel_for_simd:
11780 case OMPD_cancel:
11781 case OMPD_cancellation_point:
11782 case OMPD_ordered:
11783 case OMPD_threadprivate:
11784 case OMPD_allocate:
11785 case OMPD_task:
11786 case OMPD_simd:
11787 case OMPD_tile:
11788 case OMPD_unroll:
11789 case OMPD_sections:
11790 case OMPD_section:
11791 case OMPD_single:
11792 case OMPD_master:
11793 case OMPD_critical:
11794 case OMPD_taskyield:
11795 case OMPD_barrier:
11796 case OMPD_taskwait:
11797 case OMPD_taskgroup:
11798 case OMPD_atomic:
11799 case OMPD_flush:
11800 case OMPD_depobj:
11801 case OMPD_scan:
11802 case OMPD_teams:
11803 case OMPD_target_data:
11804 case OMPD_distribute:
11805 case OMPD_distribute_simd:
11806 case OMPD_distribute_parallel_for:
11807 case OMPD_distribute_parallel_for_simd:
11808 case OMPD_teams_distribute:
11809 case OMPD_teams_distribute_simd:
11810 case OMPD_teams_distribute_parallel_for:
11811 case OMPD_teams_distribute_parallel_for_simd:
11812 case OMPD_declare_simd:
11813 case OMPD_declare_variant:
11814 case OMPD_begin_declare_variant:
11815 case OMPD_end_declare_variant:
11816 case OMPD_declare_target:
11817 case OMPD_end_declare_target:
11818 case OMPD_declare_reduction:
11819 case OMPD_declare_mapper:
11820 case OMPD_taskloop:
11821 case OMPD_taskloop_simd:
11822 case OMPD_master_taskloop:
11823 case OMPD_master_taskloop_simd:
11824 case OMPD_parallel_master_taskloop:
11825 case OMPD_parallel_master_taskloop_simd:
11826 case OMPD_target:
11827 case OMPD_target_simd:
11828 case OMPD_target_teams_distribute:
11829 case OMPD_target_teams_distribute_simd:
11830 case OMPD_target_teams_distribute_parallel_for:
11831 case OMPD_target_teams_distribute_parallel_for_simd:
11832 case OMPD_target_teams:
11833 case OMPD_target_parallel:
11834 case OMPD_target_parallel_for:
11835 case OMPD_target_parallel_for_simd:
11836 case OMPD_requires:
11837 case OMPD_metadirective:
11838 case OMPD_unknown:
11839 default:
11840 llvm_unreachable("Unexpected standalone target data directive.");
11841 break;
11842 }
11843 if (HasNowait) {
11844 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
11845 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
11846 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
11847 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
11848 }
11849 CGF.EmitRuntimeCall(
11850 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID: RTLFn),
11851 args: OffloadingArgs);
11852 };
11853
11854 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
11855 &MapNamesArray](CodeGenFunction &CGF,
11856 PrePostActionTy &) {
11857 // Fill up the arrays with all the mapped variables.
11858 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11859 CGOpenMPRuntime::TargetDataInfo Info;
11860 MappableExprsHandler MEHandler(D, CGF);
11861 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
11862 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11863 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11864
11865 bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
11866 D.hasClausesOfKind<OMPNowaitClause>();
11867
11868 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11869 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11870 CGF.VoidPtrTy, CGM.getPointerAlign());
11871 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
11872 CGM.getPointerAlign());
11873 InputInfo.SizesArray =
11874 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11875 InputInfo.MappersArray =
11876 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11877 MapTypesArray = Info.RTArgs.MapTypesArray;
11878 MapNamesArray = Info.RTArgs.MapNamesArray;
11879 if (RequiresOuterTask)
11880 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11881 else
11882 emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11883 };
11884
11885 if (IfCond) {
11886 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen,
11887 ElseGen: [](CodeGenFunction &CGF, PrePostActionTy &) {});
11888 } else {
11889 RegionCodeGenTy ThenRCG(TargetThenGen);
11890 ThenRCG(CGF);
11891 }
11892}
11893
11894static unsigned
11895evaluateCDTSize(const FunctionDecl *FD,
11896 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
11897 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
11898 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
11899 // of that clause. The VLEN value must be power of 2.
11900 // In other case the notion of the function`s "characteristic data type" (CDT)
11901 // is used to compute the vector length.
11902 // CDT is defined in the following order:
11903 // a) For non-void function, the CDT is the return type.
11904 // b) If the function has any non-uniform, non-linear parameters, then the
11905 // CDT is the type of the first such parameter.
11906 // c) If the CDT determined by a) or b) above is struct, union, or class
11907 // type which is pass-by-value (except for the type that maps to the
11908 // built-in complex data type), the characteristic data type is int.
11909 // d) If none of the above three cases is applicable, the CDT is int.
11910 // The VLEN is then determined based on the CDT and the size of vector
11911 // register of that ISA for which current vector version is generated. The
11912 // VLEN is computed using the formula below:
11913 // VLEN = sizeof(vector_register) / sizeof(CDT),
11914 // where vector register size specified in section 3.2.1 Registers and the
11915 // Stack Frame of original AMD64 ABI document.
11916 QualType RetType = FD->getReturnType();
11917 if (RetType.isNull())
11918 return 0;
11919 ASTContext &C = FD->getASTContext();
11920 QualType CDT;
11921 if (!RetType.isNull() && !RetType->isVoidType()) {
11922 CDT = RetType;
11923 } else {
11924 unsigned Offset = 0;
11925 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD)) {
11926 if (ParamAttrs[Offset].Kind ==
11927 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector)
11928 CDT = C.getPointerType(T: C.getCanonicalTagType(TD: MD->getParent()));
11929 ++Offset;
11930 }
11931 if (CDT.isNull()) {
11932 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
11933 if (ParamAttrs[I + Offset].Kind ==
11934 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) {
11935 CDT = FD->getParamDecl(i: I)->getType();
11936 break;
11937 }
11938 }
11939 }
11940 }
11941 if (CDT.isNull())
11942 CDT = C.IntTy;
11943 CDT = CDT->getCanonicalTypeUnqualified();
11944 if (CDT->isRecordType() || CDT->isUnionType())
11945 CDT = C.IntTy;
11946 return C.getTypeSize(T: CDT);
11947}
11948
11949// This are the Functions that are needed to mangle the name of the
11950// vector functions generated by the compiler, according to the rules
11951// defined in the "Vector Function ABI specifications for AArch64",
11952// available at
11953// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
11954
11955/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
11956static bool getAArch64MTV(QualType QT,
11957 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) {
11958 QT = QT.getCanonicalType();
11959
11960 if (QT->isVoidType())
11961 return false;
11962
11963 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform)
11964 return false;
11965
11966 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
11967 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)
11968 return false;
11969
11970 if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
11971 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) &&
11972 !QT->isReferenceType())
11973 return false;
11974
11975 return true;
11976}
11977
11978/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
11979static bool getAArch64PBV(QualType QT, ASTContext &C) {
11980 QT = QT.getCanonicalType();
11981 unsigned Size = C.getTypeSize(T: QT);
11982
11983 // Only scalars and complex within 16 bytes wide set PVB to true.
11984 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
11985 return false;
11986
11987 if (QT->isFloatingType())
11988 return true;
11989
11990 if (QT->isIntegerType())
11991 return true;
11992
11993 if (QT->isPointerType())
11994 return true;
11995
11996 // TODO: Add support for complex types (section 3.1.2, item 2).
11997
11998 return false;
11999}
12000
12001/// Computes the lane size (LS) of a return type or of an input parameter,
12002/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
12003/// TODO: Add support for references, section 3.2.1, item 1.
12004static unsigned getAArch64LS(QualType QT,
12005 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind,
12006 ASTContext &C) {
12007 if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
12008 QualType PTy = QT.getCanonicalType()->getPointeeType();
12009 if (getAArch64PBV(QT: PTy, C))
12010 return C.getTypeSize(T: PTy);
12011 }
12012 if (getAArch64PBV(QT, C))
12013 return C.getTypeSize(T: QT);
12014
12015 return C.getTypeSize(T: C.getUIntPtrType());
12016}
12017
12018// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
12019// signature of the scalar function, as defined in 3.2.2 of the
12020// AAVFABI.
12021static std::tuple<unsigned, unsigned, bool>
12022getNDSWDS(const FunctionDecl *FD,
12023 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12024 QualType RetType = FD->getReturnType().getCanonicalType();
12025
12026 ASTContext &C = FD->getASTContext();
12027
12028 bool OutputBecomesInput = false;
12029
12030 llvm::SmallVector<unsigned, 8> Sizes;
12031 if (!RetType->isVoidType()) {
12032 Sizes.push_back(Elt: getAArch64LS(
12033 QT: RetType, Kind: llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C));
12034 if (!getAArch64PBV(QT: RetType, C) && getAArch64MTV(QT: RetType, Kind: {}))
12035 OutputBecomesInput = true;
12036 }
12037 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12038 QualType QT = FD->getParamDecl(i: I)->getType().getCanonicalType();
12039 Sizes.push_back(Elt: getAArch64LS(QT, Kind: ParamAttrs[I].Kind, C));
12040 }
12041
12042 assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
12043 // The LS of a function parameter / return value can only be a power
12044 // of 2, starting from 8 bits, up to 128.
12045 assert(llvm::all_of(Sizes,
12046 [](unsigned Size) {
12047 return Size == 8 || Size == 16 || Size == 32 ||
12048 Size == 64 || Size == 128;
12049 }) &&
12050 "Invalid size");
12051
12052 return std::make_tuple(args&: *llvm::min_element(Range&: Sizes), args&: *llvm::max_element(Range&: Sizes),
12053 args&: OutputBecomesInput);
12054}
12055
12056static llvm::OpenMPIRBuilder::DeclareSimdBranch
12057convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) {
12058 switch (State) {
12059 case OMPDeclareSimdDeclAttr::BS_Undefined:
12060 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined;
12061 case OMPDeclareSimdDeclAttr::BS_Inbranch:
12062 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch;
12063 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
12064 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch;
12065 }
12066 llvm_unreachable("unexpected declare simd branch state");
12067}
12068
12069// Check the values provided via `simdlen` by the user.
12070static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc,
12071 unsigned UserVLEN, unsigned WDS, char ISA) {
12072 // 1. A `simdlen(1)` doesn't produce vector signatures.
12073 if (UserVLEN == 1) {
12074 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_1_no_effect);
12075 return false;
12076 }
12077
12078 // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced
12079 // SIMD.
12080 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(Value: UserVLEN)) {
12081 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_requires_power_of_2);
12082 return false;
12083 }
12084
12085 // 3. Section 3.4.1: SVE fixed length must obey the architectural limits.
12086 if (ISA == 's' && UserVLEN != 0 &&
12087 ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) {
12088 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_must_fit_lanes) << WDS;
12089 return false;
12090 }
12091
12092 return true;
12093}
12094
12095void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
12096 llvm::Function *Fn) {
12097 ASTContext &C = CGM.getContext();
12098 FD = FD->getMostRecentDecl();
12099 while (FD) {
12100 // Map params to their positions in function decl.
12101 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
12102 if (isa<CXXMethodDecl>(Val: FD))
12103 ParamPositions.try_emplace(Key: FD, Args: 0);
12104 unsigned ParamPos = ParamPositions.size();
12105 for (const ParmVarDecl *P : FD->parameters()) {
12106 ParamPositions.try_emplace(Key: P->getCanonicalDecl(), Args&: ParamPos);
12107 ++ParamPos;
12108 }
12109 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
12110 llvm::SmallVector<llvm::OpenMPIRBuilder::DeclareSimdAttrTy, 8> ParamAttrs(
12111 ParamPositions.size());
12112 // Mark uniform parameters.
12113 for (const Expr *E : Attr->uniforms()) {
12114 E = E->IgnoreParenImpCasts();
12115 unsigned Pos;
12116 if (isa<CXXThisExpr>(Val: E)) {
12117 Pos = ParamPositions[FD];
12118 } else {
12119 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12120 ->getCanonicalDecl();
12121 auto It = ParamPositions.find(Val: PVD);
12122 assert(It != ParamPositions.end() && "Function parameter not found");
12123 Pos = It->second;
12124 }
12125 ParamAttrs[Pos].Kind =
12126 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform;
12127 }
12128 // Get alignment info.
12129 auto *NI = Attr->alignments_begin();
12130 for (const Expr *E : Attr->aligneds()) {
12131 E = E->IgnoreParenImpCasts();
12132 unsigned Pos;
12133 QualType ParmTy;
12134 if (isa<CXXThisExpr>(Val: E)) {
12135 Pos = ParamPositions[FD];
12136 ParmTy = E->getType();
12137 } else {
12138 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12139 ->getCanonicalDecl();
12140 auto It = ParamPositions.find(Val: PVD);
12141 assert(It != ParamPositions.end() && "Function parameter not found");
12142 Pos = It->second;
12143 ParmTy = PVD->getType();
12144 }
12145 ParamAttrs[Pos].Alignment =
12146 (*NI)
12147 ? (*NI)->EvaluateKnownConstInt(Ctx: C)
12148 : llvm::APSInt::getUnsigned(
12149 X: C.toCharUnitsFromBits(BitSize: C.getOpenMPDefaultSimdAlign(T: ParmTy))
12150 .getQuantity());
12151 ++NI;
12152 }
12153 // Mark linear parameters.
12154 auto *SI = Attr->steps_begin();
12155 auto *MI = Attr->modifiers_begin();
12156 for (const Expr *E : Attr->linears()) {
12157 E = E->IgnoreParenImpCasts();
12158 unsigned Pos;
12159 bool IsReferenceType = false;
12160 // Rescaling factor needed to compute the linear parameter
12161 // value in the mangled name.
12162 unsigned PtrRescalingFactor = 1;
12163 if (isa<CXXThisExpr>(Val: E)) {
12164 Pos = ParamPositions[FD];
12165 auto *P = cast<PointerType>(Val: E->getType());
12166 PtrRescalingFactor = CGM.getContext()
12167 .getTypeSizeInChars(T: P->getPointeeType())
12168 .getQuantity();
12169 } else {
12170 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12171 ->getCanonicalDecl();
12172 auto It = ParamPositions.find(Val: PVD);
12173 assert(It != ParamPositions.end() && "Function parameter not found");
12174 Pos = It->second;
12175 if (auto *P = dyn_cast<PointerType>(Val: PVD->getType()))
12176 PtrRescalingFactor = CGM.getContext()
12177 .getTypeSizeInChars(T: P->getPointeeType())
12178 .getQuantity();
12179 else if (PVD->getType()->isReferenceType()) {
12180 IsReferenceType = true;
12181 PtrRescalingFactor =
12182 CGM.getContext()
12183 .getTypeSizeInChars(T: PVD->getType().getNonReferenceType())
12184 .getQuantity();
12185 }
12186 }
12187 llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos];
12188 if (*MI == OMPC_LINEAR_ref)
12189 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef;
12190 else if (*MI == OMPC_LINEAR_uval)
12191 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal;
12192 else if (IsReferenceType)
12193 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal;
12194 else
12195 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear;
12196 // Assuming a stride of 1, for `linear` without modifiers.
12197 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: 1);
12198 if (*SI) {
12199 Expr::EvalResult Result;
12200 if (!(*SI)->EvaluateAsInt(Result, Ctx: C, AllowSideEffects: Expr::SE_AllowSideEffects)) {
12201 if (const auto *DRE =
12202 cast<DeclRefExpr>(Val: (*SI)->IgnoreParenImpCasts())) {
12203 if (const auto *StridePVD =
12204 dyn_cast<ParmVarDecl>(Val: DRE->getDecl())) {
12205 ParamAttr.HasVarStride = true;
12206 auto It = ParamPositions.find(Val: StridePVD->getCanonicalDecl());
12207 assert(It != ParamPositions.end() &&
12208 "Function parameter not found");
12209 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: It->second);
12210 }
12211 }
12212 } else {
12213 ParamAttr.StrideOrArg = Result.Val.getInt();
12214 }
12215 }
12216 // If we are using a linear clause on a pointer, we need to
12217 // rescale the value of linear_step with the byte size of the
12218 // pointee type.
12219 if (!ParamAttr.HasVarStride &&
12220 (ParamAttr.Kind ==
12221 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12222 ParamAttr.Kind ==
12223 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef))
12224 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
12225 ++SI;
12226 ++MI;
12227 }
12228 llvm::APSInt VLENVal;
12229 SourceLocation ExprLoc;
12230 const Expr *VLENExpr = Attr->getSimdlen();
12231 if (VLENExpr) {
12232 VLENVal = VLENExpr->EvaluateKnownConstInt(Ctx: C);
12233 ExprLoc = VLENExpr->getExprLoc();
12234 }
12235 llvm::OpenMPIRBuilder::DeclareSimdBranch State =
12236 convertDeclareSimdBranch(State: Attr->getBranchState());
12237 if (CGM.getTriple().isX86()) {
12238 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
12239 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12240 OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElements: NumElts, VLENVal, ParamAttrs,
12241 Branch: State);
12242 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
12243 unsigned VLEN = VLENVal.getExtValue();
12244 // Get basic data for building the vector signature.
12245 const auto Data = getNDSWDS(FD, ParamAttrs);
12246 const unsigned NDS = std::get<0>(t: Data);
12247 const unsigned WDS = std::get<1>(t: Data);
12248 const bool OutputBecomesInput = std::get<2>(t: Data);
12249 if (CGM.getTarget().hasFeature(Feature: "sve")) {
12250 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 's'))
12251 OMPBuilder.emitAArch64DeclareSimdFunction(
12252 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 's', NarrowestDataSize: NDS, OutputBecomesInput);
12253 } else if (CGM.getTarget().hasFeature(Feature: "neon")) {
12254 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 'n'))
12255 OMPBuilder.emitAArch64DeclareSimdFunction(
12256 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 'n', NarrowestDataSize: NDS, OutputBecomesInput);
12257 }
12258 }
12259 }
12260 FD = FD->getPreviousDecl();
12261 }
12262}
12263
12264namespace {
12265/// Cleanup action for doacross support.
12266class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
12267public:
12268 static const int DoacrossFinArgs = 2;
12269
12270private:
12271 llvm::FunctionCallee RTLFn;
12272 llvm::Value *Args[DoacrossFinArgs];
12273
12274public:
12275 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
12276 ArrayRef<llvm::Value *> CallArgs)
12277 : RTLFn(RTLFn) {
12278 assert(CallArgs.size() == DoacrossFinArgs);
12279 std::copy(first: CallArgs.begin(), last: CallArgs.end(), result: std::begin(arr&: Args));
12280 }
12281 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12282 if (!CGF.HaveInsertPoint())
12283 return;
12284 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12285 }
12286};
12287} // namespace
12288
12289void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
12290 const OMPLoopDirective &D,
12291 ArrayRef<Expr *> NumIterations) {
12292 if (!CGF.HaveInsertPoint())
12293 return;
12294
12295 ASTContext &C = CGM.getContext();
12296 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
12297 RecordDecl *RD;
12298 if (KmpDimTy.isNull()) {
12299 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
12300 // kmp_int64 lo; // lower
12301 // kmp_int64 up; // upper
12302 // kmp_int64 st; // stride
12303 // };
12304 RD = C.buildImplicitRecord(Name: "kmp_dim");
12305 RD->startDefinition();
12306 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12307 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12308 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12309 RD->completeDefinition();
12310 KmpDimTy = C.getCanonicalTagType(TD: RD);
12311 } else {
12312 RD = KmpDimTy->castAsRecordDecl();
12313 }
12314 llvm::APInt Size(/*numBits=*/32, NumIterations.size());
12315 QualType ArrayTy = C.getConstantArrayType(EltTy: KmpDimTy, ArySize: Size, SizeExpr: nullptr,
12316 ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12317
12318 Address DimsAddr = CGF.CreateMemTemp(T: ArrayTy, Name: "dims");
12319 CGF.EmitNullInitialization(DestPtr: DimsAddr, Ty: ArrayTy);
12320 enum { LowerFD = 0, UpperFD, StrideFD };
12321 // Fill dims with data.
12322 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
12323 LValue DimsLVal = CGF.MakeAddrLValue(
12324 Addr: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: I), T: KmpDimTy);
12325 // dims.upper = num_iterations;
12326 LValue UpperLVal = CGF.EmitLValueForField(
12327 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: UpperFD));
12328 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
12329 Src: CGF.EmitScalarExpr(E: NumIterations[I]), SrcTy: NumIterations[I]->getType(),
12330 DstTy: Int64Ty, Loc: NumIterations[I]->getExprLoc());
12331 CGF.EmitStoreOfScalar(value: NumIterVal, lvalue: UpperLVal);
12332 // dims.stride = 1;
12333 LValue StrideLVal = CGF.EmitLValueForField(
12334 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: StrideFD));
12335 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::getSigned(Ty: CGM.Int64Ty, /*V=*/1),
12336 lvalue: StrideLVal);
12337 }
12338
12339 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
12340 // kmp_int32 num_dims, struct kmp_dim * dims);
12341 llvm::Value *Args[] = {
12342 emitUpdateLocation(CGF, Loc: D.getBeginLoc()),
12343 getThreadID(CGF, Loc: D.getBeginLoc()),
12344 llvm::ConstantInt::getSigned(Ty: CGM.Int32Ty, V: NumIterations.size()),
12345 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12346 V: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: 0).emitRawPointer(CGF),
12347 DestTy: CGM.VoidPtrTy)};
12348
12349 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12350 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_init);
12351 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12352 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
12353 emitUpdateLocation(CGF, Loc: D.getEndLoc()), getThreadID(CGF, Loc: D.getEndLoc())};
12354 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12355 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_fini);
12356 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(Kind: NormalAndEHCleanup, A: FiniRTLFn,
12357 A: llvm::ArrayRef(FiniArgs));
12358}
12359
12360template <typename T>
12361static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
12362 const T *C, llvm::Value *ULoc,
12363 llvm::Value *ThreadID) {
12364 QualType Int64Ty =
12365 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
12366 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
12367 QualType ArrayTy = CGM.getContext().getConstantArrayType(
12368 EltTy: Int64Ty, ArySize: Size, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12369 Address CntAddr = CGF.CreateMemTemp(T: ArrayTy, Name: ".cnt.addr");
12370 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
12371 const Expr *CounterVal = C->getLoopData(I);
12372 assert(CounterVal);
12373 llvm::Value *CntVal = CGF.EmitScalarConversion(
12374 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
12375 Loc: CounterVal->getExprLoc());
12376 CGF.EmitStoreOfScalar(Value: CntVal, Addr: CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: I),
12377 /*Volatile=*/false, Ty: Int64Ty);
12378 }
12379 llvm::Value *Args[] = {
12380 ULoc, ThreadID,
12381 CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: 0).emitRawPointer(CGF)};
12382 llvm::FunctionCallee RTLFn;
12383 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
12384 OMPDoacrossKind<T> ODK;
12385 if (ODK.isSource(C)) {
12386 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12387 FnID: OMPRTL___kmpc_doacross_post);
12388 } else {
12389 assert(ODK.isSink(C) && "Expect sink modifier.");
12390 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12391 FnID: OMPRTL___kmpc_doacross_wait);
12392 }
12393 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12394}
12395
12396void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12397 const OMPDependClause *C) {
12398 return EmitDoacrossOrdered<OMPDependClause>(
12399 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12400 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12401}
12402
12403void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12404 const OMPDoacrossClause *C) {
12405 return EmitDoacrossOrdered<OMPDoacrossClause>(
12406 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12407 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12408}
12409
12410void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
12411 llvm::FunctionCallee Callee,
12412 ArrayRef<llvm::Value *> Args) const {
12413 assert(Loc.isValid() && "Outlined function call location must be valid.");
12414 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
12415
12416 if (auto *Fn = dyn_cast<llvm::Function>(Val: Callee.getCallee())) {
12417 if (Fn->doesNotThrow()) {
12418 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
12419 return;
12420 }
12421 }
12422 CGF.EmitRuntimeCall(callee: Callee, args: Args);
12423}
12424
12425void CGOpenMPRuntime::emitOutlinedFunctionCall(
12426 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
12427 ArrayRef<llvm::Value *> Args) const {
12428 emitCall(CGF, Loc, Callee: OutlinedFn, Args);
12429}
12430
12431void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
12432 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D))
12433 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: FD))
12434 HasEmittedDeclareTargetRegion = true;
12435}
12436
12437Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
12438 const VarDecl *NativeParam,
12439 const VarDecl *TargetParam) const {
12440 return CGF.GetAddrOfLocalVar(VD: NativeParam);
12441}
12442
12443/// Return allocator value from expression, or return a null allocator (default
12444/// when no allocator specified).
12445static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
12446 const Expr *Allocator) {
12447 llvm::Value *AllocVal;
12448 if (Allocator) {
12449 AllocVal = CGF.EmitScalarExpr(E: Allocator);
12450 // According to the standard, the original allocator type is a enum
12451 // (integer). Convert to pointer type, if required.
12452 AllocVal = CGF.EmitScalarConversion(Src: AllocVal, SrcTy: Allocator->getType(),
12453 DstTy: CGF.getContext().VoidPtrTy,
12454 Loc: Allocator->getExprLoc());
12455 } else {
12456 // If no allocator specified, it defaults to the null allocator.
12457 AllocVal = llvm::Constant::getNullValue(
12458 Ty: CGF.CGM.getTypes().ConvertType(T: CGF.getContext().VoidPtrTy));
12459 }
12460 return AllocVal;
12461}
12462
12463/// Return the alignment from an allocate directive if present.
12464static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {
12465 std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);
12466
12467 if (!AllocateAlignment)
12468 return nullptr;
12469
12470 return llvm::ConstantInt::get(Ty: CGM.SizeTy, V: AllocateAlignment->getQuantity());
12471}
12472
12473Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
12474 const VarDecl *VD) {
12475 if (!VD)
12476 return Address::invalid();
12477 Address UntiedAddr = Address::invalid();
12478 Address UntiedRealAddr = Address::invalid();
12479 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12480 if (It != FunctionToUntiedTaskStackMap.end()) {
12481 const UntiedLocalVarsAddressesMap &UntiedData =
12482 UntiedLocalVarsStack[It->second];
12483 auto I = UntiedData.find(Key: VD);
12484 if (I != UntiedData.end()) {
12485 UntiedAddr = I->second.first;
12486 UntiedRealAddr = I->second.second;
12487 }
12488 }
12489 const VarDecl *CVD = VD->getCanonicalDecl();
12490 if (CVD->hasAttr<OMPAllocateDeclAttr>()) {
12491 // Use the default allocation.
12492 if (!isAllocatableDecl(VD))
12493 return UntiedAddr;
12494 llvm::Value *Size;
12495 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
12496 if (CVD->getType()->isVariablyModifiedType()) {
12497 Size = CGF.getTypeSize(Ty: CVD->getType());
12498 // Align the size: ((size + align - 1) / align) * align
12499 Size = CGF.Builder.CreateNUWAdd(
12500 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
12501 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
12502 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
12503 } else {
12504 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
12505 Size = CGM.getSize(numChars: Sz.alignTo(Align));
12506 }
12507 llvm::Value *ThreadID = getThreadID(CGF, Loc: CVD->getBeginLoc());
12508 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
12509 const Expr *Allocator = AA->getAllocator();
12510 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
12511 llvm::Value *Alignment = getAlignmentValue(CGM, VD: CVD);
12512 SmallVector<llvm::Value *, 4> Args;
12513 Args.push_back(Elt: ThreadID);
12514 if (Alignment)
12515 Args.push_back(Elt: Alignment);
12516 Args.push_back(Elt: Size);
12517 Args.push_back(Elt: AllocVal);
12518 llvm::omp::RuntimeFunction FnID =
12519 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
12520 llvm::Value *Addr = CGF.EmitRuntimeCall(
12521 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args,
12522 name: getName(Parts: {CVD->getName(), ".void.addr"}));
12523 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12524 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free);
12525 QualType Ty = CGM.getContext().getPointerType(T: CVD->getType());
12526 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12527 V: Addr, DestTy: CGF.ConvertTypeForMem(T: Ty), Name: getName(Parts: {CVD->getName(), ".addr"}));
12528 if (UntiedAddr.isValid())
12529 CGF.EmitStoreOfScalar(Value: Addr, Addr: UntiedAddr, /*Volatile=*/false, Ty);
12530
12531 // Cleanup action for allocate support.
12532 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
12533 llvm::FunctionCallee RTLFn;
12534 SourceLocation::UIntTy LocEncoding;
12535 Address Addr;
12536 const Expr *AllocExpr;
12537
12538 public:
12539 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
12540 SourceLocation::UIntTy LocEncoding, Address Addr,
12541 const Expr *AllocExpr)
12542 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
12543 AllocExpr(AllocExpr) {}
12544 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12545 if (!CGF.HaveInsertPoint())
12546 return;
12547 llvm::Value *Args[3];
12548 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
12549 CGF, Loc: SourceLocation::getFromRawEncoding(Encoding: LocEncoding));
12550 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12551 V: Addr.emitRawPointer(CGF), DestTy: CGF.VoidPtrTy);
12552 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator: AllocExpr);
12553 Args[2] = AllocVal;
12554 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12555 }
12556 };
12557 Address VDAddr =
12558 UntiedRealAddr.isValid()
12559 ? UntiedRealAddr
12560 : Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
12561 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
12562 Kind: NormalAndEHCleanup, A: FiniRTLFn, A: CVD->getLocation().getRawEncoding(),
12563 A: VDAddr, A: Allocator);
12564 if (UntiedRealAddr.isValid())
12565 if (auto *Region =
12566 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
12567 Region->emitUntiedSwitch(CGF);
12568 return VDAddr;
12569 }
12570 return UntiedAddr;
12571}
12572
12573bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,
12574 const VarDecl *VD) const {
12575 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12576 if (It == FunctionToUntiedTaskStackMap.end())
12577 return false;
12578 return UntiedLocalVarsStack[It->second].count(Key: VD) > 0;
12579}
12580
12581CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII(
12582 CodeGenModule &CGM, const OMPLoopDirective &S)
12583 : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {
12584 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12585 if (!NeedToPush)
12586 return;
12587 NontemporalDeclsSet &DS =
12588 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12589 for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) {
12590 for (const Stmt *Ref : C->private_refs()) {
12591 const auto *SimpleRefExpr = cast<Expr>(Val: Ref)->IgnoreParenImpCasts();
12592 const ValueDecl *VD;
12593 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: SimpleRefExpr)) {
12594 VD = DRE->getDecl();
12595 } else {
12596 const auto *ME = cast<MemberExpr>(Val: SimpleRefExpr);
12597 assert((ME->isImplicitCXXThis() ||
12598 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
12599 "Expected member of current class.");
12600 VD = ME->getMemberDecl();
12601 }
12602 DS.insert(V: VD);
12603 }
12604 }
12605}
12606
12607CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() {
12608 if (!NeedToPush)
12609 return;
12610 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
12611}
12612
12613CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII(
12614 CodeGenFunction &CGF,
12615 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
12616 std::pair<Address, Address>> &LocalVars)
12617 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
12618 if (!NeedToPush)
12619 return;
12620 CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace(
12621 Key: CGF.CurFn, Args: CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
12622 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(Elt: LocalVars);
12623}
12624
12625CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() {
12626 if (!NeedToPush)
12627 return;
12628 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
12629}
12630
12631bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const {
12632 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12633
12634 return llvm::any_of(
12635 Range&: CGM.getOpenMPRuntime().NontemporalDeclsStack,
12636 P: [VD](const NontemporalDeclsSet &Set) { return Set.contains(V: VD); });
12637}
12638
12639void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
12640 const OMPExecutableDirective &S,
12641 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled)
12642 const {
12643 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
12644 // Vars in target/task regions must be excluded completely.
12645 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()) ||
12646 isOpenMPTaskingDirective(Kind: S.getDirectiveKind())) {
12647 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
12648 getOpenMPCaptureRegions(CaptureRegions, DKind: S.getDirectiveKind());
12649 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CaptureRegions.front());
12650 for (const CapturedStmt::Capture &Cap : CS->captures()) {
12651 if (Cap.capturesVariable() || Cap.capturesVariableByCopy())
12652 NeedToCheckForLPCs.insert(V: Cap.getCapturedVar());
12653 }
12654 }
12655 // Exclude vars in private clauses.
12656 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
12657 for (const Expr *Ref : C->varlist()) {
12658 if (!Ref->getType()->isScalarType())
12659 continue;
12660 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12661 if (!DRE)
12662 continue;
12663 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12664 }
12665 }
12666 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
12667 for (const Expr *Ref : C->varlist()) {
12668 if (!Ref->getType()->isScalarType())
12669 continue;
12670 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12671 if (!DRE)
12672 continue;
12673 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12674 }
12675 }
12676 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
12677 for (const Expr *Ref : C->varlist()) {
12678 if (!Ref->getType()->isScalarType())
12679 continue;
12680 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12681 if (!DRE)
12682 continue;
12683 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12684 }
12685 }
12686 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
12687 for (const Expr *Ref : C->varlist()) {
12688 if (!Ref->getType()->isScalarType())
12689 continue;
12690 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12691 if (!DRE)
12692 continue;
12693 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12694 }
12695 }
12696 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
12697 for (const Expr *Ref : C->varlist()) {
12698 if (!Ref->getType()->isScalarType())
12699 continue;
12700 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12701 if (!DRE)
12702 continue;
12703 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12704 }
12705 }
12706 for (const Decl *VD : NeedToCheckForLPCs) {
12707 for (const LastprivateConditionalData &Data :
12708 llvm::reverse(C&: CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
12709 if (Data.DeclToUniqueName.count(Key: VD) > 0) {
12710 if (!Data.Disabled)
12711 NeedToAddForLPCsAsDisabled.insert(V: VD);
12712 break;
12713 }
12714 }
12715 }
12716}
12717
12718CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12719 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
12720 : CGM(CGF.CGM),
12721 Action((CGM.getLangOpts().OpenMP >= 50 &&
12722 llvm::any_of(Range: S.getClausesOfKind<OMPLastprivateClause>(),
12723 P: [](const OMPLastprivateClause *C) {
12724 return C->getKind() ==
12725 OMPC_LASTPRIVATE_conditional;
12726 }))
12727 ? ActionToDo::PushAsLastprivateConditional
12728 : ActionToDo::DoNotPush) {
12729 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12730 if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
12731 return;
12732 assert(Action == ActionToDo::PushAsLastprivateConditional &&
12733 "Expected a push action.");
12734 LastprivateConditionalData &Data =
12735 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
12736 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
12737 if (C->getKind() != OMPC_LASTPRIVATE_conditional)
12738 continue;
12739
12740 for (const Expr *Ref : C->varlist()) {
12741 Data.DeclToUniqueName.insert(KV: std::make_pair(
12742 x: cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts())->getDecl(),
12743 y: SmallString<16>(generateUniqueName(CGM, Prefix: "pl_cond", Ref))));
12744 }
12745 }
12746 Data.IVLVal = IVLVal;
12747 Data.Fn = CGF.CurFn;
12748}
12749
12750CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12751 CodeGenFunction &CGF, const OMPExecutableDirective &S)
12752 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
12753 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12754 if (CGM.getLangOpts().OpenMP < 50)
12755 return;
12756 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
12757 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
12758 if (!NeedToAddForLPCsAsDisabled.empty()) {
12759 Action = ActionToDo::DisableLastprivateConditional;
12760 LastprivateConditionalData &Data =
12761 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
12762 for (const Decl *VD : NeedToAddForLPCsAsDisabled)
12763 Data.DeclToUniqueName.try_emplace(Key: VD);
12764 Data.Fn = CGF.CurFn;
12765 Data.Disabled = true;
12766 }
12767}
12768
12769CGOpenMPRuntime::LastprivateConditionalRAII
12770CGOpenMPRuntime::LastprivateConditionalRAII::disable(
12771 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
12772 return LastprivateConditionalRAII(CGF, S);
12773}
12774
12775CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() {
12776 if (CGM.getLangOpts().OpenMP < 50)
12777 return;
12778 if (Action == ActionToDo::DisableLastprivateConditional) {
12779 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12780 "Expected list of disabled private vars.");
12781 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12782 }
12783 if (Action == ActionToDo::PushAsLastprivateConditional) {
12784 assert(
12785 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12786 "Expected list of lastprivate conditional vars.");
12787 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12788 }
12789}
12790
12791Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
12792 const VarDecl *VD) {
12793 ASTContext &C = CGM.getContext();
12794 auto I = LastprivateConditionalToTypes.try_emplace(Key: CGF.CurFn).first;
12795 QualType NewType;
12796 const FieldDecl *VDField;
12797 const FieldDecl *FiredField;
12798 LValue BaseLVal;
12799 auto VI = I->getSecond().find(Val: VD);
12800 if (VI == I->getSecond().end()) {
12801 RecordDecl *RD = C.buildImplicitRecord(Name: "lasprivate.conditional");
12802 RD->startDefinition();
12803 VDField = addFieldToRecordDecl(C, DC: RD, FieldTy: VD->getType().getNonReferenceType());
12804 FiredField = addFieldToRecordDecl(C, DC: RD, FieldTy: C.CharTy);
12805 RD->completeDefinition();
12806 NewType = C.getCanonicalTagType(TD: RD);
12807 Address Addr = CGF.CreateMemTemp(T: NewType, Align: C.getDeclAlign(D: VD), Name: VD->getName());
12808 BaseLVal = CGF.MakeAddrLValue(Addr, T: NewType, Source: AlignmentSource::Decl);
12809 I->getSecond().try_emplace(Key: VD, Args&: NewType, Args&: VDField, Args&: FiredField, Args&: BaseLVal);
12810 } else {
12811 NewType = std::get<0>(t&: VI->getSecond());
12812 VDField = std::get<1>(t&: VI->getSecond());
12813 FiredField = std::get<2>(t&: VI->getSecond());
12814 BaseLVal = std::get<3>(t&: VI->getSecond());
12815 }
12816 LValue FiredLVal =
12817 CGF.EmitLValueForField(Base: BaseLVal, Field: FiredField);
12818 CGF.EmitStoreOfScalar(
12819 value: llvm::ConstantInt::getNullValue(Ty: CGF.ConvertTypeForMem(T: C.CharTy)),
12820 lvalue: FiredLVal);
12821 return CGF.EmitLValueForField(Base: BaseLVal, Field: VDField).getAddress();
12822}
12823
12824namespace {
12825/// Checks if the lastprivate conditional variable is referenced in LHS.
12826class LastprivateConditionalRefChecker final
12827 : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {
12828 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM;
12829 const Expr *FoundE = nullptr;
12830 const Decl *FoundD = nullptr;
12831 StringRef UniqueDeclName;
12832 LValue IVLVal;
12833 llvm::Function *FoundFn = nullptr;
12834 SourceLocation Loc;
12835
12836public:
12837 bool VisitDeclRefExpr(const DeclRefExpr *E) {
12838 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
12839 llvm::reverse(C&: LPM)) {
12840 auto It = D.DeclToUniqueName.find(Key: E->getDecl());
12841 if (It == D.DeclToUniqueName.end())
12842 continue;
12843 if (D.Disabled)
12844 return false;
12845 FoundE = E;
12846 FoundD = E->getDecl()->getCanonicalDecl();
12847 UniqueDeclName = It->second;
12848 IVLVal = D.IVLVal;
12849 FoundFn = D.Fn;
12850 break;
12851 }
12852 return FoundE == E;
12853 }
12854 bool VisitMemberExpr(const MemberExpr *E) {
12855 if (!CodeGenFunction::IsWrappedCXXThis(E: E->getBase()))
12856 return false;
12857 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
12858 llvm::reverse(C&: LPM)) {
12859 auto It = D.DeclToUniqueName.find(Key: E->getMemberDecl());
12860 if (It == D.DeclToUniqueName.end())
12861 continue;
12862 if (D.Disabled)
12863 return false;
12864 FoundE = E;
12865 FoundD = E->getMemberDecl()->getCanonicalDecl();
12866 UniqueDeclName = It->second;
12867 IVLVal = D.IVLVal;
12868 FoundFn = D.Fn;
12869 break;
12870 }
12871 return FoundE == E;
12872 }
12873 bool VisitStmt(const Stmt *S) {
12874 for (const Stmt *Child : S->children()) {
12875 if (!Child)
12876 continue;
12877 if (const auto *E = dyn_cast<Expr>(Val: Child))
12878 if (!E->isGLValue())
12879 continue;
12880 if (Visit(S: Child))
12881 return true;
12882 }
12883 return false;
12884 }
12885 explicit LastprivateConditionalRefChecker(
12886 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
12887 : LPM(LPM) {}
12888 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
12889 getFoundData() const {
12890 return std::make_tuple(args: FoundE, args: FoundD, args: UniqueDeclName, args: IVLVal, args: FoundFn);
12891 }
12892};
12893} // namespace
12894
12895void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
12896 LValue IVLVal,
12897 StringRef UniqueDeclName,
12898 LValue LVal,
12899 SourceLocation Loc) {
12900 // Last updated loop counter for the lastprivate conditional var.
12901 // int<xx> last_iv = 0;
12902 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(T: IVLVal.getType());
12903 llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable(
12904 Ty: LLIVTy, Name: getName(Parts: {UniqueDeclName, "iv"}));
12905 cast<llvm::GlobalVariable>(Val: LastIV)->setAlignment(
12906 IVLVal.getAlignment().getAsAlign());
12907 LValue LastIVLVal =
12908 CGF.MakeNaturalAlignRawAddrLValue(V: LastIV, T: IVLVal.getType());
12909
12910 // Last value of the lastprivate conditional.
12911 // decltype(priv_a) last_a;
12912 llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
12913 Ty: CGF.ConvertTypeForMem(T: LVal.getType()), Name: UniqueDeclName);
12914 cast<llvm::GlobalVariable>(Val: Last)->setAlignment(
12915 LVal.getAlignment().getAsAlign());
12916 LValue LastLVal =
12917 CGF.MakeRawAddrLValue(V: Last, T: LVal.getType(), Alignment: LVal.getAlignment());
12918
12919 // Global loop counter. Required to handle inner parallel-for regions.
12920 // iv
12921 llvm::Value *IVVal = CGF.EmitLoadOfScalar(lvalue: IVLVal, Loc);
12922
12923 // #pragma omp critical(a)
12924 // if (last_iv <= iv) {
12925 // last_iv = iv;
12926 // last_a = priv_a;
12927 // }
12928 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
12929 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
12930 Action.Enter(CGF);
12931 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(lvalue: LastIVLVal, Loc);
12932 // (last_iv <= iv) ? Check if the variable is updated and store new
12933 // value in global var.
12934 llvm::Value *CmpRes;
12935 if (IVLVal.getType()->isSignedIntegerType()) {
12936 CmpRes = CGF.Builder.CreateICmpSLE(LHS: LastIVVal, RHS: IVVal);
12937 } else {
12938 assert(IVLVal.getType()->isUnsignedIntegerType() &&
12939 "Loop iteration variable must be integer.");
12940 CmpRes = CGF.Builder.CreateICmpULE(LHS: LastIVVal, RHS: IVVal);
12941 }
12942 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lp_cond_then");
12943 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "lp_cond_exit");
12944 CGF.Builder.CreateCondBr(Cond: CmpRes, True: ThenBB, False: ExitBB);
12945 // {
12946 CGF.EmitBlock(BB: ThenBB);
12947
12948 // last_iv = iv;
12949 CGF.EmitStoreOfScalar(value: IVVal, lvalue: LastIVLVal);
12950
12951 // last_a = priv_a;
12952 switch (CGF.getEvaluationKind(T: LVal.getType())) {
12953 case TEK_Scalar: {
12954 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
12955 CGF.EmitStoreOfScalar(value: PrivVal, lvalue: LastLVal);
12956 break;
12957 }
12958 case TEK_Complex: {
12959 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(src: LVal, loc: Loc);
12960 CGF.EmitStoreOfComplex(V: PrivVal, dest: LastLVal, /*isInit=*/false);
12961 break;
12962 }
12963 case TEK_Aggregate:
12964 llvm_unreachable(
12965 "Aggregates are not supported in lastprivate conditional.");
12966 }
12967 // }
12968 CGF.EmitBranch(Block: ExitBB);
12969 // There is no need to emit line number for unconditional branch.
12970 (void)ApplyDebugLocation::CreateEmpty(CGF);
12971 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
12972 };
12973
12974 if (CGM.getLangOpts().OpenMPSimd) {
12975 // Do not emit as a critical region as no parallel region could be emitted.
12976 RegionCodeGenTy ThenRCG(CodeGen);
12977 ThenRCG(CGF);
12978 } else {
12979 emitCriticalRegion(CGF, CriticalName: UniqueDeclName, CriticalOpGen: CodeGen, Loc);
12980 }
12981}
12982
12983void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
12984 const Expr *LHS) {
12985 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
12986 return;
12987 LastprivateConditionalRefChecker Checker(LastprivateConditionalStack);
12988 if (!Checker.Visit(S: LHS))
12989 return;
12990 const Expr *FoundE;
12991 const Decl *FoundD;
12992 StringRef UniqueDeclName;
12993 LValue IVLVal;
12994 llvm::Function *FoundFn;
12995 std::tie(args&: FoundE, args&: FoundD, args&: UniqueDeclName, args&: IVLVal, args&: FoundFn) =
12996 Checker.getFoundData();
12997 if (FoundFn != CGF.CurFn) {
12998 // Special codegen for inner parallel regions.
12999 // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1;
13000 auto It = LastprivateConditionalToTypes[FoundFn].find(Val: FoundD);
13001 assert(It != LastprivateConditionalToTypes[FoundFn].end() &&
13002 "Lastprivate conditional is not found in outer region.");
13003 QualType StructTy = std::get<0>(t&: It->getSecond());
13004 const FieldDecl* FiredDecl = std::get<2>(t&: It->getSecond());
13005 LValue PrivLVal = CGF.EmitLValue(E: FoundE);
13006 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
13007 Addr: PrivLVal.getAddress(),
13008 Ty: CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: StructTy)),
13009 ElementTy: CGF.ConvertTypeForMem(T: StructTy));
13010 LValue BaseLVal =
13011 CGF.MakeAddrLValue(Addr: StructAddr, T: StructTy, Source: AlignmentSource::Decl);
13012 LValue FiredLVal = CGF.EmitLValueForField(Base: BaseLVal, Field: FiredDecl);
13013 CGF.EmitAtomicStore(rvalue: RValue::get(V: llvm::ConstantInt::get(
13014 Ty: CGF.ConvertTypeForMem(T: FiredDecl->getType()), V: 1)),
13015 lvalue: FiredLVal, AO: llvm::AtomicOrdering::Unordered,
13016 /*IsVolatile=*/true, /*isInit=*/false);
13017 return;
13018 }
13019
13020 // Private address of the lastprivate conditional in the current context.
13021 // priv_a
13022 LValue LVal = CGF.EmitLValue(E: FoundE);
13023 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
13024 Loc: FoundE->getExprLoc());
13025}
13026
13027void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional(
13028 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13029 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) {
13030 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13031 return;
13032 auto Range = llvm::reverse(C&: LastprivateConditionalStack);
13033 auto It = llvm::find_if(
13034 Range, P: [](const LastprivateConditionalData &D) { return !D.Disabled; });
13035 if (It == Range.end() || It->Fn != CGF.CurFn)
13036 return;
13037 auto LPCI = LastprivateConditionalToTypes.find(Val: It->Fn);
13038 assert(LPCI != LastprivateConditionalToTypes.end() &&
13039 "Lastprivates must be registered already.");
13040 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
13041 getOpenMPCaptureRegions(CaptureRegions, DKind: D.getDirectiveKind());
13042 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: CaptureRegions.back());
13043 for (const auto &Pair : It->DeclToUniqueName) {
13044 const auto *VD = cast<VarDecl>(Val: Pair.first->getCanonicalDecl());
13045 if (!CS->capturesVariable(Var: VD) || IgnoredDecls.contains(V: VD))
13046 continue;
13047 auto I = LPCI->getSecond().find(Val: Pair.first);
13048 assert(I != LPCI->getSecond().end() &&
13049 "Lastprivate must be rehistered already.");
13050 // bool Cmp = priv_a.Fired != 0;
13051 LValue BaseLVal = std::get<3>(t&: I->getSecond());
13052 LValue FiredLVal =
13053 CGF.EmitLValueForField(Base: BaseLVal, Field: std::get<2>(t&: I->getSecond()));
13054 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: FiredLVal, Loc: D.getBeginLoc());
13055 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Res);
13056 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lpc.then");
13057 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "lpc.done");
13058 // if (Cmp) {
13059 CGF.Builder.CreateCondBr(Cond: Cmp, True: ThenBB, False: DoneBB);
13060 CGF.EmitBlock(BB: ThenBB);
13061 Address Addr = CGF.GetAddrOfLocalVar(VD);
13062 LValue LVal;
13063 if (VD->getType()->isReferenceType())
13064 LVal = CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
13065 Source: AlignmentSource::Decl);
13066 else
13067 LVal = CGF.MakeAddrLValue(Addr, T: VD->getType().getNonReferenceType(),
13068 Source: AlignmentSource::Decl);
13069 emitLastprivateConditionalUpdate(CGF, IVLVal: It->IVLVal, UniqueDeclName: Pair.second, LVal,
13070 Loc: D.getBeginLoc());
13071 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
13072 CGF.EmitBlock(BB: DoneBB, /*IsFinal=*/IsFinished: true);
13073 // }
13074 }
13075}
13076
13077void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
13078 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
13079 SourceLocation Loc) {
13080 if (CGF.getLangOpts().OpenMP < 50)
13081 return;
13082 auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(Key: VD);
13083 assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() &&
13084 "Unknown lastprivate conditional variable.");
13085 StringRef UniqueName = It->second;
13086 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: UniqueName);
13087 // The variable was not updated in the region - exit.
13088 if (!GV)
13089 return;
13090 LValue LPLVal = CGF.MakeRawAddrLValue(
13091 V: GV, T: PrivLVal.getType().getNonReferenceType(), Alignment: PrivLVal.getAlignment());
13092 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: LPLVal, Loc);
13093 CGF.EmitStoreOfScalar(value: Res, lvalue: PrivLVal);
13094}
13095
13096llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
13097 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13098 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13099 const RegionCodeGenTy &CodeGen) {
13100 llvm_unreachable("Not supported in SIMD-only mode");
13101}
13102
13103llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
13104 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13105 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13106 const RegionCodeGenTy &CodeGen) {
13107 llvm_unreachable("Not supported in SIMD-only mode");
13108}
13109
13110llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
13111 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
13112 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
13113 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
13114 bool Tied, unsigned &NumberOfParts) {
13115 llvm_unreachable("Not supported in SIMD-only mode");
13116}
13117
13118void CGOpenMPSIMDRuntime::emitParallelCall(
13119 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
13120 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
13121 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
13122 OpenMPSeverityClauseKind Severity, const Expr *Message) {
13123 llvm_unreachable("Not supported in SIMD-only mode");
13124}
13125
13126void CGOpenMPSIMDRuntime::emitCriticalRegion(
13127 CodeGenFunction &CGF, StringRef CriticalName,
13128 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
13129 const Expr *Hint) {
13130 llvm_unreachable("Not supported in SIMD-only mode");
13131}
13132
13133void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
13134 const RegionCodeGenTy &MasterOpGen,
13135 SourceLocation Loc) {
13136 llvm_unreachable("Not supported in SIMD-only mode");
13137}
13138
13139void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,
13140 const RegionCodeGenTy &MasterOpGen,
13141 SourceLocation Loc,
13142 const Expr *Filter) {
13143 llvm_unreachable("Not supported in SIMD-only mode");
13144}
13145
13146void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
13147 SourceLocation Loc) {
13148 llvm_unreachable("Not supported in SIMD-only mode");
13149}
13150
13151void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
13152 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
13153 SourceLocation Loc) {
13154 llvm_unreachable("Not supported in SIMD-only mode");
13155}
13156
13157void CGOpenMPSIMDRuntime::emitSingleRegion(
13158 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
13159 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
13160 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
13161 ArrayRef<const Expr *> AssignmentOps) {
13162 llvm_unreachable("Not supported in SIMD-only mode");
13163}
13164
13165void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
13166 const RegionCodeGenTy &OrderedOpGen,
13167 SourceLocation Loc,
13168 bool IsThreads) {
13169 llvm_unreachable("Not supported in SIMD-only mode");
13170}
13171
13172void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
13173 SourceLocation Loc,
13174 OpenMPDirectiveKind Kind,
13175 bool EmitChecks,
13176 bool ForceSimpleCall) {
13177 llvm_unreachable("Not supported in SIMD-only mode");
13178}
13179
13180void CGOpenMPSIMDRuntime::emitForDispatchInit(
13181 CodeGenFunction &CGF, SourceLocation Loc,
13182 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
13183 bool Ordered, const DispatchRTInput &DispatchValues) {
13184 llvm_unreachable("Not supported in SIMD-only mode");
13185}
13186
13187void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
13188 SourceLocation Loc) {
13189 llvm_unreachable("Not supported in SIMD-only mode");
13190}
13191
13192void CGOpenMPSIMDRuntime::emitForStaticInit(
13193 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
13194 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
13195 llvm_unreachable("Not supported in SIMD-only mode");
13196}
13197
13198void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
13199 CodeGenFunction &CGF, SourceLocation Loc,
13200 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
13201 llvm_unreachable("Not supported in SIMD-only mode");
13202}
13203
13204void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
13205 SourceLocation Loc,
13206 unsigned IVSize,
13207 bool IVSigned) {
13208 llvm_unreachable("Not supported in SIMD-only mode");
13209}
13210
13211void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
13212 SourceLocation Loc,
13213 OpenMPDirectiveKind DKind) {
13214 llvm_unreachable("Not supported in SIMD-only mode");
13215}
13216
13217llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
13218 SourceLocation Loc,
13219 unsigned IVSize, bool IVSigned,
13220 Address IL, Address LB,
13221 Address UB, Address ST) {
13222 llvm_unreachable("Not supported in SIMD-only mode");
13223}
13224
13225void CGOpenMPSIMDRuntime::emitNumThreadsClause(
13226 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
13227 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
13228 SourceLocation SeverityLoc, const Expr *Message,
13229 SourceLocation MessageLoc) {
13230 llvm_unreachable("Not supported in SIMD-only mode");
13231}
13232
13233void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
13234 ProcBindKind ProcBind,
13235 SourceLocation Loc) {
13236 llvm_unreachable("Not supported in SIMD-only mode");
13237}
13238
13239Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
13240 const VarDecl *VD,
13241 Address VDAddr,
13242 SourceLocation Loc) {
13243 llvm_unreachable("Not supported in SIMD-only mode");
13244}
13245
13246llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
13247 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
13248 CodeGenFunction *CGF) {
13249 llvm_unreachable("Not supported in SIMD-only mode");
13250}
13251
13252Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
13253 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
13254 llvm_unreachable("Not supported in SIMD-only mode");
13255}
13256
13257void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
13258 ArrayRef<const Expr *> Vars,
13259 SourceLocation Loc,
13260 llvm::AtomicOrdering AO) {
13261 llvm_unreachable("Not supported in SIMD-only mode");
13262}
13263
13264void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
13265 const OMPExecutableDirective &D,
13266 llvm::Function *TaskFunction,
13267 QualType SharedsTy, Address Shareds,
13268 const Expr *IfCond,
13269 const OMPTaskDataTy &Data) {
13270 llvm_unreachable("Not supported in SIMD-only mode");
13271}
13272
13273void CGOpenMPSIMDRuntime::emitTaskLoopCall(
13274 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
13275 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
13276 const Expr *IfCond, const OMPTaskDataTy &Data) {
13277 llvm_unreachable("Not supported in SIMD-only mode");
13278}
13279
13280void CGOpenMPSIMDRuntime::emitReduction(
13281 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
13282 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
13283 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
13284 assert(Options.SimpleReduction && "Only simple reduction is expected.");
13285 CGOpenMPRuntime::emitReduction(CGF, Loc, OrgPrivates: Privates, OrgLHSExprs: LHSExprs, OrgRHSExprs: RHSExprs,
13286 OrgReductionOps: ReductionOps, Options);
13287}
13288
13289llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
13290 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
13291 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
13292 llvm_unreachable("Not supported in SIMD-only mode");
13293}
13294
13295void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
13296 SourceLocation Loc,
13297 bool IsWorksharingReduction) {
13298 llvm_unreachable("Not supported in SIMD-only mode");
13299}
13300
13301void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
13302 SourceLocation Loc,
13303 ReductionCodeGen &RCG,
13304 unsigned N) {
13305 llvm_unreachable("Not supported in SIMD-only mode");
13306}
13307
13308Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
13309 SourceLocation Loc,
13310 llvm::Value *ReductionsPtr,
13311 LValue SharedLVal) {
13312 llvm_unreachable("Not supported in SIMD-only mode");
13313}
13314
13315void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
13316 SourceLocation Loc,
13317 const OMPTaskDataTy &Data) {
13318 llvm_unreachable("Not supported in SIMD-only mode");
13319}
13320
13321void CGOpenMPSIMDRuntime::emitCancellationPointCall(
13322 CodeGenFunction &CGF, SourceLocation Loc,
13323 OpenMPDirectiveKind CancelRegion) {
13324 llvm_unreachable("Not supported in SIMD-only mode");
13325}
13326
13327void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
13328 SourceLocation Loc, const Expr *IfCond,
13329 OpenMPDirectiveKind CancelRegion) {
13330 llvm_unreachable("Not supported in SIMD-only mode");
13331}
13332
13333void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
13334 const OMPExecutableDirective &D, StringRef ParentName,
13335 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
13336 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
13337 llvm_unreachable("Not supported in SIMD-only mode");
13338}
13339
13340void CGOpenMPSIMDRuntime::emitTargetCall(
13341 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13342 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
13343 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
13344 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
13345 const OMPLoopDirective &D)>
13346 SizeEmitter) {
13347 llvm_unreachable("Not supported in SIMD-only mode");
13348}
13349
13350bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
13351 llvm_unreachable("Not supported in SIMD-only mode");
13352}
13353
13354bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
13355 llvm_unreachable("Not supported in SIMD-only mode");
13356}
13357
13358bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
13359 return false;
13360}
13361
13362void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
13363 const OMPExecutableDirective &D,
13364 SourceLocation Loc,
13365 llvm::Function *OutlinedFn,
13366 ArrayRef<llvm::Value *> CapturedVars) {
13367 llvm_unreachable("Not supported in SIMD-only mode");
13368}
13369
13370void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
13371 const Expr *NumTeams,
13372 const Expr *ThreadLimit,
13373 SourceLocation Loc) {
13374 llvm_unreachable("Not supported in SIMD-only mode");
13375}
13376
13377void CGOpenMPSIMDRuntime::emitTargetDataCalls(
13378 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13379 const Expr *Device, const RegionCodeGenTy &CodeGen,
13380 CGOpenMPRuntime::TargetDataInfo &Info) {
13381 llvm_unreachable("Not supported in SIMD-only mode");
13382}
13383
13384void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
13385 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13386 const Expr *Device) {
13387 llvm_unreachable("Not supported in SIMD-only mode");
13388}
13389
13390void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
13391 const OMPLoopDirective &D,
13392 ArrayRef<Expr *> NumIterations) {
13393 llvm_unreachable("Not supported in SIMD-only mode");
13394}
13395
13396void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13397 const OMPDependClause *C) {
13398 llvm_unreachable("Not supported in SIMD-only mode");
13399}
13400
13401void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13402 const OMPDoacrossClause *C) {
13403 llvm_unreachable("Not supported in SIMD-only mode");
13404}
13405
13406const VarDecl *
13407CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
13408 const VarDecl *NativeParam) const {
13409 llvm_unreachable("Not supported in SIMD-only mode");
13410}
13411
13412Address
13413CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
13414 const VarDecl *NativeParam,
13415 const VarDecl *TargetParam) const {
13416 llvm_unreachable("Not supported in SIMD-only mode");
13417}
13418