1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "ABIInfoImpl.h"
15#include "CGCXXABI.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGRecordLayout.h"
19#include "CodeGenFunction.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/Attr.h"
23#include "clang/AST/Decl.h"
24#include "clang/AST/OpenMPClause.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/SourceManager.h"
30#include "clang/CodeGen/ConstantInitBuilder.h"
31#include "llvm/ADT/ArrayRef.h"
32#include "llvm/ADT/SmallSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Bitcode/BitcodeReader.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/GlobalValue.h"
39#include "llvm/IR/InstrTypes.h"
40#include "llvm/IR/Value.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/VirtualFileSystem.h"
43#include "llvm/Support/raw_ostream.h"
44#include <cassert>
45#include <cstdint>
46#include <numeric>
47#include <optional>
48
49using namespace clang;
50using namespace CodeGen;
51using namespace llvm::omp;
52
53namespace {
54/// Base class for handling code generation inside OpenMP regions.
55class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
56public:
57 /// Kinds of OpenMP regions used in codegen.
58 enum CGOpenMPRegionKind {
59 /// Region with outlined function for standalone 'parallel'
60 /// directive.
61 ParallelOutlinedRegion,
62 /// Region with outlined function for standalone 'task' directive.
63 TaskOutlinedRegion,
64 /// Region for constructs that do not require function outlining,
65 /// like 'for', 'sections', 'atomic' etc. directives.
66 InlinedRegion,
67 /// Region with outlined function for standalone 'target' directive.
68 TargetRegion,
69 };
70
71 CGOpenMPRegionInfo(const CapturedStmt &CS,
72 const CGOpenMPRegionKind RegionKind,
73 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
74 bool HasCancel)
75 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
76 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
77
78 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
79 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
80 bool HasCancel)
81 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
82 Kind(Kind), HasCancel(HasCancel) {}
83
84 /// Get a variable or parameter for storing global thread id
85 /// inside OpenMP construct.
86 virtual const VarDecl *getThreadIDVariable() const = 0;
87
88 /// Emit the captured statement body.
89 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
90
91 /// Get an LValue for the current ThreadID variable.
92 /// \return LValue for thread id variable. This LValue always has type int32*.
93 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
94
95 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
96
97 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
98
99 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
100
101 bool hasCancel() const { return HasCancel; }
102
103 static bool classof(const CGCapturedStmtInfo *Info) {
104 return Info->getKind() == CR_OpenMP;
105 }
106
107 ~CGOpenMPRegionInfo() override = default;
108
109protected:
110 CGOpenMPRegionKind RegionKind;
111 RegionCodeGenTy CodeGen;
112 OpenMPDirectiveKind Kind;
113 bool HasCancel;
114};
115
116/// API for captured statement code generation in OpenMP constructs.
117class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
118public:
119 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
120 const RegionCodeGenTy &CodeGen,
121 OpenMPDirectiveKind Kind, bool HasCancel,
122 StringRef HelperName)
123 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
124 HasCancel),
125 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
126 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
127 }
128
129 /// Get a variable or parameter for storing global thread id
130 /// inside OpenMP construct.
131 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
132
133 /// Get the name of the capture helper.
134 StringRef getHelperName() const override { return HelperName; }
135
136 static bool classof(const CGCapturedStmtInfo *Info) {
137 return CGOpenMPRegionInfo::classof(Info) &&
138 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
139 ParallelOutlinedRegion;
140 }
141
142private:
143 /// A variable or parameter storing global thread id for OpenMP
144 /// constructs.
145 const VarDecl *ThreadIDVar;
146 StringRef HelperName;
147};
148
149/// API for captured statement code generation in OpenMP constructs.
150class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
151public:
152 class UntiedTaskActionTy final : public PrePostActionTy {
153 bool Untied;
154 const VarDecl *PartIDVar;
155 const RegionCodeGenTy UntiedCodeGen;
156 llvm::SwitchInst *UntiedSwitch = nullptr;
157
158 public:
159 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
160 const RegionCodeGenTy &UntiedCodeGen)
161 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
162 void Enter(CodeGenFunction &CGF) override {
163 if (Untied) {
164 // Emit task switching point.
165 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
166 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
167 PtrTy: PartIDVar->getType()->castAs<PointerType>());
168 llvm::Value *Res =
169 CGF.EmitLoadOfScalar(lvalue: PartIdLVal, Loc: PartIDVar->getLocation());
170 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: ".untied.done.");
171 UntiedSwitch = CGF.Builder.CreateSwitch(V: Res, Dest: DoneBB);
172 CGF.EmitBlock(BB: DoneBB);
173 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
174 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
175 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: 0),
176 Dest: CGF.Builder.GetInsertBlock());
177 emitUntiedSwitch(CGF);
178 }
179 }
180 void emitUntiedSwitch(CodeGenFunction &CGF) const {
181 if (Untied) {
182 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
183 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
184 PtrTy: PartIDVar->getType()->castAs<PointerType>());
185 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
186 lvalue: PartIdLVal);
187 UntiedCodeGen(CGF);
188 CodeGenFunction::JumpDest CurPoint =
189 CGF.getJumpDestInCurrentScope(Name: ".untied.next.");
190 CGF.EmitBranch(Block: CGF.ReturnBlock.getBlock());
191 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
192 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
193 Dest: CGF.Builder.GetInsertBlock());
194 CGF.EmitBranchThroughCleanup(Dest: CurPoint);
195 CGF.EmitBlock(BB: CurPoint.getBlock());
196 }
197 }
198 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
199 };
200 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
201 const VarDecl *ThreadIDVar,
202 const RegionCodeGenTy &CodeGen,
203 OpenMPDirectiveKind Kind, bool HasCancel,
204 const UntiedTaskActionTy &Action)
205 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
206 ThreadIDVar(ThreadIDVar), Action(Action) {
207 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
208 }
209
210 /// Get a variable or parameter for storing global thread id
211 /// inside OpenMP construct.
212 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
213
214 /// Get an LValue for the current ThreadID variable.
215 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
216
217 /// Get the name of the capture helper.
218 StringRef getHelperName() const override { return ".omp_outlined."; }
219
220 void emitUntiedSwitch(CodeGenFunction &CGF) override {
221 Action.emitUntiedSwitch(CGF);
222 }
223
224 static bool classof(const CGCapturedStmtInfo *Info) {
225 return CGOpenMPRegionInfo::classof(Info) &&
226 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
227 TaskOutlinedRegion;
228 }
229
230private:
231 /// A variable or parameter storing global thread id for OpenMP
232 /// constructs.
233 const VarDecl *ThreadIDVar;
234 /// Action for emitting code for untied tasks.
235 const UntiedTaskActionTy &Action;
236};
237
238/// API for inlined captured statement code generation in OpenMP
239/// constructs.
240class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
241public:
242 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
243 const RegionCodeGenTy &CodeGen,
244 OpenMPDirectiveKind Kind, bool HasCancel)
245 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
246 OldCSI(OldCSI),
247 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(Val: OldCSI)) {}
248
249 // Retrieve the value of the context parameter.
250 llvm::Value *getContextValue() const override {
251 if (OuterRegionInfo)
252 return OuterRegionInfo->getContextValue();
253 llvm_unreachable("No context value for inlined OpenMP region");
254 }
255
256 void setContextValue(llvm::Value *V) override {
257 if (OuterRegionInfo) {
258 OuterRegionInfo->setContextValue(V);
259 return;
260 }
261 llvm_unreachable("No context value for inlined OpenMP region");
262 }
263
264 /// Lookup the captured field decl for a variable.
265 const FieldDecl *lookup(const VarDecl *VD) const override {
266 if (OuterRegionInfo)
267 return OuterRegionInfo->lookup(VD);
268 // If there is no outer outlined region,no need to lookup in a list of
269 // captured variables, we can use the original one.
270 return nullptr;
271 }
272
273 FieldDecl *getThisFieldDecl() const override {
274 if (OuterRegionInfo)
275 return OuterRegionInfo->getThisFieldDecl();
276 return nullptr;
277 }
278
279 /// Get a variable or parameter for storing global thread id
280 /// inside OpenMP construct.
281 const VarDecl *getThreadIDVariable() const override {
282 if (OuterRegionInfo)
283 return OuterRegionInfo->getThreadIDVariable();
284 return nullptr;
285 }
286
287 /// Get an LValue for the current ThreadID variable.
288 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
289 if (OuterRegionInfo)
290 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
291 llvm_unreachable("No LValue for inlined OpenMP construct");
292 }
293
294 /// Get the name of the capture helper.
295 StringRef getHelperName() const override {
296 if (auto *OuterRegionInfo = getOldCSI())
297 return OuterRegionInfo->getHelperName();
298 llvm_unreachable("No helper name for inlined OpenMP construct");
299 }
300
301 void emitUntiedSwitch(CodeGenFunction &CGF) override {
302 if (OuterRegionInfo)
303 OuterRegionInfo->emitUntiedSwitch(CGF);
304 }
305
306 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
307
308 static bool classof(const CGCapturedStmtInfo *Info) {
309 return CGOpenMPRegionInfo::classof(Info) &&
310 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == InlinedRegion;
311 }
312
313 ~CGOpenMPInlinedRegionInfo() override = default;
314
315private:
316 /// CodeGen info about outer OpenMP region.
317 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
318 CGOpenMPRegionInfo *OuterRegionInfo;
319};
320
321/// API for captured statement code generation in OpenMP target
322/// constructs. For this captures, implicit parameters are used instead of the
323/// captured fields. The name of the target region has to be unique in a given
324/// application so it is provided by the client, because only the client has
325/// the information to generate that.
326class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
327public:
328 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
329 const RegionCodeGenTy &CodeGen, StringRef HelperName)
330 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
331 /*HasCancel=*/false),
332 HelperName(HelperName) {}
333
334 /// This is unused for target regions because each starts executing
335 /// with a single thread.
336 const VarDecl *getThreadIDVariable() const override { return nullptr; }
337
338 /// Get the name of the capture helper.
339 StringRef getHelperName() const override { return HelperName; }
340
341 static bool classof(const CGCapturedStmtInfo *Info) {
342 return CGOpenMPRegionInfo::classof(Info) &&
343 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == TargetRegion;
344 }
345
346private:
347 StringRef HelperName;
348};
349
350static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
351 llvm_unreachable("No codegen for expressions");
352}
353/// API for generation of expressions captured in a innermost OpenMP
354/// region.
355class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
356public:
357 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
358 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
359 OMPD_unknown,
360 /*HasCancel=*/false),
361 PrivScope(CGF) {
362 // Make sure the globals captured in the provided statement are local by
363 // using the privatization logic. We assume the same variable is not
364 // captured more than once.
365 for (const auto &C : CS.captures()) {
366 if (!C.capturesVariable() && !C.capturesVariableByCopy())
367 continue;
368
369 const VarDecl *VD = C.getCapturedVar();
370 if (VD->isLocalVarDeclOrParm())
371 continue;
372
373 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
374 /*RefersToEnclosingVariableOrCapture=*/false,
375 VD->getType().getNonReferenceType(), VK_LValue,
376 C.getLocation());
377 PrivScope.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
378 }
379 (void)PrivScope.Privatize();
380 }
381
382 /// Lookup the captured field decl for a variable.
383 const FieldDecl *lookup(const VarDecl *VD) const override {
384 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
385 return FD;
386 return nullptr;
387 }
388
389 /// Emit the captured statement body.
390 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
391 llvm_unreachable("No body for expressions");
392 }
393
394 /// Get a variable or parameter for storing global thread id
395 /// inside OpenMP construct.
396 const VarDecl *getThreadIDVariable() const override {
397 llvm_unreachable("No thread id for expressions");
398 }
399
400 /// Get the name of the capture helper.
401 StringRef getHelperName() const override {
402 llvm_unreachable("No helper name for expressions");
403 }
404
405 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
406
407private:
408 /// Private scope to capture global variables.
409 CodeGenFunction::OMPPrivateScope PrivScope;
410};
411
412/// RAII for emitting code of OpenMP constructs.
413class InlinedOpenMPRegionRAII {
414 CodeGenFunction &CGF;
415 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
416 FieldDecl *LambdaThisCaptureField = nullptr;
417 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
418 bool NoInheritance = false;
419
420public:
421 /// Constructs region for combined constructs.
422 /// \param CodeGen Code generation sequence for combined directives. Includes
423 /// a list of functions used for code generation of implicitly inlined
424 /// regions.
425 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
426 OpenMPDirectiveKind Kind, bool HasCancel,
427 bool NoInheritance = true)
428 : CGF(CGF), NoInheritance(NoInheritance) {
429 // Start emission for the construct.
430 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
431 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
432 if (NoInheritance) {
433 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
434 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
435 CGF.LambdaThisCaptureField = nullptr;
436 BlockInfo = CGF.BlockInfo;
437 CGF.BlockInfo = nullptr;
438 }
439 }
440
441 ~InlinedOpenMPRegionRAII() {
442 // Restore original CapturedStmtInfo only if we're done with code emission.
443 auto *OldCSI =
444 cast<CGOpenMPInlinedRegionInfo>(Val: CGF.CapturedStmtInfo)->getOldCSI();
445 delete CGF.CapturedStmtInfo;
446 CGF.CapturedStmtInfo = OldCSI;
447 if (NoInheritance) {
448 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
449 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
450 CGF.BlockInfo = BlockInfo;
451 }
452 }
453};
454
455/// Values for bit flags used in the ident_t to describe the fields.
456/// All enumeric elements are named and described in accordance with the code
457/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
458enum OpenMPLocationFlags : unsigned {
459 /// Use trampoline for internal microtask.
460 OMP_IDENT_IMD = 0x01,
461 /// Use c-style ident structure.
462 OMP_IDENT_KMPC = 0x02,
463 /// Atomic reduction option for kmpc_reduce.
464 OMP_ATOMIC_REDUCE = 0x10,
465 /// Explicit 'barrier' directive.
466 OMP_IDENT_BARRIER_EXPL = 0x20,
467 /// Implicit barrier in code.
468 OMP_IDENT_BARRIER_IMPL = 0x40,
469 /// Implicit barrier in 'for' directive.
470 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
471 /// Implicit barrier in 'sections' directive.
472 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
473 /// Implicit barrier in 'single' directive.
474 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
475 /// Call of __kmp_for_static_init for static loop.
476 OMP_IDENT_WORK_LOOP = 0x200,
477 /// Call of __kmp_for_static_init for sections.
478 OMP_IDENT_WORK_SECTIONS = 0x400,
479 /// Call of __kmp_for_static_init for distribute.
480 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
481 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
482};
483
484/// Describes ident structure that describes a source location.
485/// All descriptions are taken from
486/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
487/// Original structure:
488/// typedef struct ident {
489/// kmp_int32 reserved_1; /**< might be used in Fortran;
490/// see above */
491/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
492/// KMP_IDENT_KMPC identifies this union
493/// member */
494/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
495/// see above */
496///#if USE_ITT_BUILD
497/// /* but currently used for storing
498/// region-specific ITT */
499/// /* contextual information. */
500///#endif /* USE_ITT_BUILD */
501/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
502/// C++ */
503/// char const *psource; /**< String describing the source location.
504/// The string is composed of semi-colon separated
505// fields which describe the source file,
506/// the function and a pair of line numbers that
507/// delimit the construct.
508/// */
509/// } ident_t;
510enum IdentFieldIndex {
511 /// might be used in Fortran
512 IdentField_Reserved_1,
513 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
514 IdentField_Flags,
515 /// Not really used in Fortran any more
516 IdentField_Reserved_2,
517 /// Source[4] in Fortran, do not use for C++
518 IdentField_Reserved_3,
519 /// String describing the source location. The string is composed of
520 /// semi-colon separated fields which describe the source file, the function
521 /// and a pair of line numbers that delimit the construct.
522 IdentField_PSource
523};
524
525/// Schedule types for 'omp for' loops (these enumerators are taken from
526/// the enum sched_type in kmp.h).
527enum OpenMPSchedType {
528 /// Lower bound for default (unordered) versions.
529 OMP_sch_lower = 32,
530 OMP_sch_static_chunked = 33,
531 OMP_sch_static = 34,
532 OMP_sch_dynamic_chunked = 35,
533 OMP_sch_guided_chunked = 36,
534 OMP_sch_runtime = 37,
535 OMP_sch_auto = 38,
536 /// static with chunk adjustment (e.g., simd)
537 OMP_sch_static_balanced_chunked = 45,
538 /// Lower bound for 'ordered' versions.
539 OMP_ord_lower = 64,
540 OMP_ord_static_chunked = 65,
541 OMP_ord_static = 66,
542 OMP_ord_dynamic_chunked = 67,
543 OMP_ord_guided_chunked = 68,
544 OMP_ord_runtime = 69,
545 OMP_ord_auto = 70,
546 OMP_sch_default = OMP_sch_static,
547 /// dist_schedule types
548 OMP_dist_sch_static_chunked = 91,
549 OMP_dist_sch_static = 92,
550 /// Fused distribute+for static schedule (entityId = team*nthreads + tid,
551 /// num_entities = nteams*nthreads). One for_static_init call, no
552 /// surrounding distribute_static_init. Matches
553 /// kmp_sched_distr_static_chunk_sched_static_chunkone in the device RTL
554 /// (openmp/device/include/DeviceTypes.h).
555 OMP_dist_sch_static_chunked_sch_static_chunkone = 93,
556 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
557 /// Set if the monotonic schedule modifier was present.
558 OMP_sch_modifier_monotonic = (1 << 29),
559 /// Set if the nonmonotonic schedule modifier was present.
560 OMP_sch_modifier_nonmonotonic = (1 << 30),
561};
562
563/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
564/// region.
565class CleanupTy final : public EHScopeStack::Cleanup {
566 PrePostActionTy *Action;
567
568public:
569 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
570 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
571 if (!CGF.HaveInsertPoint())
572 return;
573 Action->Exit(CGF);
574 }
575};
576
577} // anonymous namespace
578
579void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
580 CodeGenFunction::RunCleanupsScope Scope(CGF);
581 if (PrePostAction) {
582 CGF.EHStack.pushCleanup<CleanupTy>(Kind: NormalAndEHCleanup, A: PrePostAction);
583 Callback(CodeGen, CGF, *PrePostAction);
584 } else {
585 PrePostActionTy Action;
586 Callback(CodeGen, CGF, Action);
587 }
588}
589
590/// Check if the combiner is a call to UDR combiner and if it is so return the
591/// UDR decl used for reduction.
592static const OMPDeclareReductionDecl *
593getReductionInit(const Expr *ReductionOp) {
594 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
595 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
596 if (const auto *DRE =
597 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
598 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl()))
599 return DRD;
600 return nullptr;
601}
602
603static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
604 const OMPDeclareReductionDecl *DRD,
605 const Expr *InitOp,
606 Address Private, Address Original,
607 QualType Ty) {
608 if (DRD->getInitializer()) {
609 std::pair<llvm::Function *, llvm::Function *> Reduction =
610 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
611 const auto *CE = cast<CallExpr>(Val: InitOp);
612 const auto *OVE = cast<OpaqueValueExpr>(Val: CE->getCallee());
613 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
614 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
615 const auto *LHSDRE =
616 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: LHS)->getSubExpr());
617 const auto *RHSDRE =
618 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: RHS)->getSubExpr());
619 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
620 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: LHSDRE->getDecl()), Addr: Private);
621 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: RHSDRE->getDecl()), Addr: Original);
622 (void)PrivateScope.Privatize();
623 RValue Func = RValue::get(V: Reduction.second);
624 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
625 CGF.EmitIgnoredExpr(E: InitOp);
626 } else {
627 llvm::Constant *Init = CGF.CGM.EmitNullConstant(T: Ty);
628 std::string Name = CGF.CGM.getOpenMPRuntime().getName(Parts: {"init"});
629 auto *GV = new llvm::GlobalVariable(
630 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
631 llvm::GlobalValue::PrivateLinkage, Init, Name);
632 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V: GV, T: Ty);
633 RValue InitRVal;
634 switch (CGF.getEvaluationKind(T: Ty)) {
635 case TEK_Scalar:
636 InitRVal = CGF.EmitLoadOfLValue(V: LV, Loc: DRD->getLocation());
637 break;
638 case TEK_Complex:
639 InitRVal =
640 RValue::getComplex(C: CGF.EmitLoadOfComplex(src: LV, loc: DRD->getLocation()));
641 break;
642 case TEK_Aggregate: {
643 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);
644 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
645 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
646 /*IsInitializer=*/false);
647 return;
648 }
649 }
650 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue);
651 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
652 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
653 /*IsInitializer=*/false);
654 }
655}
656
657/// Emit initialization of arrays of complex types.
658/// \param DestAddr Address of the array.
659/// \param Type Type of array.
660/// \param Init Initial expression of array.
661/// \param SrcAddr Address of the original array.
662static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
663 QualType Type, bool EmitDeclareReductionInit,
664 const Expr *Init,
665 const OMPDeclareReductionDecl *DRD,
666 Address SrcAddr = Address::invalid()) {
667 // Perform element-by-element initialization.
668 QualType ElementTy;
669
670 // Drill down to the base element type on both arrays.
671 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
672 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
673 if (DRD)
674 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
675
676 llvm::Value *SrcBegin = nullptr;
677 if (DRD)
678 SrcBegin = SrcAddr.emitRawPointer(CGF);
679 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
680 // Cast from pointer to array type to pointer to single element.
681 llvm::Value *DestEnd =
682 CGF.Builder.CreateGEP(Ty: DestAddr.getElementType(), Ptr: DestBegin, IdxList: NumElements);
683 // The basic structure here is a while-do loop.
684 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arrayinit.body");
685 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arrayinit.done");
686 llvm::Value *IsEmpty =
687 CGF.Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arrayinit.isempty");
688 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
689
690 // Enter the loop body, making that address the current address.
691 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
692 CGF.EmitBlock(BB: BodyBB);
693
694 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
695
696 llvm::PHINode *SrcElementPHI = nullptr;
697 Address SrcElementCurrent = Address::invalid();
698 if (DRD) {
699 SrcElementPHI = CGF.Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2,
700 Name: "omp.arraycpy.srcElementPast");
701 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
702 SrcElementCurrent =
703 Address(SrcElementPHI, SrcAddr.getElementType(),
704 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
705 }
706 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
707 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
708 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
709 Address DestElementCurrent =
710 Address(DestElementPHI, DestAddr.getElementType(),
711 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
712
713 // Emit copy.
714 {
715 CodeGenFunction::RunCleanupsScope InitScope(CGF);
716 if (EmitDeclareReductionInit) {
717 emitInitWithReductionInitializer(CGF, DRD, InitOp: Init, Private: DestElementCurrent,
718 Original: SrcElementCurrent, Ty: ElementTy);
719 } else
720 CGF.EmitAnyExprToMem(E: Init, Location: DestElementCurrent, Quals: ElementTy.getQualifiers(),
721 /*IsInitializer=*/false);
722 }
723
724 if (DRD) {
725 // Shift the address forward by one element.
726 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
727 Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI, /*Idx0=*/1,
728 Name: "omp.arraycpy.dest.element");
729 SrcElementPHI->addIncoming(V: SrcElementNext, BB: CGF.Builder.GetInsertBlock());
730 }
731
732 // Shift the address forward by one element.
733 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
734 Ty: DestAddr.getElementType(), Ptr: DestElementPHI, /*Idx0=*/1,
735 Name: "omp.arraycpy.dest.element");
736 // Check whether we've reached the end.
737 llvm::Value *Done =
738 CGF.Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
739 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
740 DestElementPHI->addIncoming(V: DestElementNext, BB: CGF.Builder.GetInsertBlock());
741
742 // Done.
743 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
744}
745
746LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
747 return CGF.EmitOMPSharedLValue(E);
748}
749
750LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
751 const Expr *E) {
752 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E))
753 return CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false);
754 return LValue();
755}
756
757void ReductionCodeGen::emitAggregateInitialization(
758 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
759 const OMPDeclareReductionDecl *DRD) {
760 // Emit VarDecl with copy init for arrays.
761 // Get the address of the original variable captured in current
762 // captured region.
763 const auto *PrivateVD =
764 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
765 bool EmitDeclareReductionInit =
766 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
767 EmitOMPAggregateInit(CGF, DestAddr: PrivateAddr, Type: PrivateVD->getType(),
768 EmitDeclareReductionInit,
769 Init: EmitDeclareReductionInit ? ClausesData[N].ReductionOp
770 : PrivateVD->getInit(),
771 DRD, SrcAddr: SharedAddr);
772}
773
774ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
775 ArrayRef<const Expr *> Origs,
776 ArrayRef<const Expr *> Privates,
777 ArrayRef<const Expr *> ReductionOps) {
778 ClausesData.reserve(N: Shareds.size());
779 SharedAddresses.reserve(N: Shareds.size());
780 Sizes.reserve(N: Shareds.size());
781 BaseDecls.reserve(N: Shareds.size());
782 const auto *IOrig = Origs.begin();
783 const auto *IPriv = Privates.begin();
784 const auto *IRed = ReductionOps.begin();
785 for (const Expr *Ref : Shareds) {
786 ClausesData.emplace_back(Args&: Ref, Args: *IOrig, Args: *IPriv, Args: *IRed);
787 std::advance(i&: IOrig, n: 1);
788 std::advance(i&: IPriv, n: 1);
789 std::advance(i&: IRed, n: 1);
790 }
791}
792
793void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
794 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
795 "Number of generated lvalues must be exactly N.");
796 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Shared);
797 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Shared);
798 SharedAddresses.emplace_back(Args&: First, Args&: Second);
799 if (ClausesData[N].Shared == ClausesData[N].Ref) {
800 OrigAddresses.emplace_back(Args&: First, Args&: Second);
801 } else {
802 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Ref);
803 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Ref);
804 OrigAddresses.emplace_back(Args&: First, Args&: Second);
805 }
806}
807
808void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
809 QualType PrivateType = getPrivateType(N);
810 bool AsArraySection = isa<ArraySectionExpr>(Val: ClausesData[N].Ref);
811 if (!PrivateType->isVariablyModifiedType()) {
812 Sizes.emplace_back(
813 Args: CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType()),
814 Args: nullptr);
815 return;
816 }
817 llvm::Value *Size;
818 llvm::Value *SizeInChars;
819 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
820 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(Ty: ElemType);
821 if (AsArraySection) {
822 Size = CGF.Builder.CreatePtrDiff(ElemTy: ElemType,
823 LHS: OrigAddresses[N].second.getPointer(CGF),
824 RHS: OrigAddresses[N].first.getPointer(CGF));
825 Size = CGF.Builder.CreateZExtOrTrunc(V: Size, DestTy: ElemSizeOf->getType());
826 Size = CGF.Builder.CreateNUWAdd(
827 LHS: Size, RHS: llvm::ConstantInt::get(Ty: Size->getType(), /*V=*/1));
828 SizeInChars = CGF.Builder.CreateNUWMul(LHS: Size, RHS: ElemSizeOf);
829 } else {
830 SizeInChars =
831 CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType());
832 Size = CGF.Builder.CreateExactUDiv(LHS: SizeInChars, RHS: ElemSizeOf);
833 }
834 Sizes.emplace_back(Args&: SizeInChars, Args&: Size);
835 CodeGenFunction::OpaqueValueMapping OpaqueMap(
836 CGF,
837 cast<OpaqueValueExpr>(
838 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
839 RValue::get(V: Size));
840 CGF.EmitVariablyModifiedType(Ty: PrivateType);
841}
842
843void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
844 llvm::Value *Size) {
845 QualType PrivateType = getPrivateType(N);
846 if (!PrivateType->isVariablyModifiedType()) {
847 assert(!Size && !Sizes[N].second &&
848 "Size should be nullptr for non-variably modified reduction "
849 "items.");
850 return;
851 }
852 CodeGenFunction::OpaqueValueMapping OpaqueMap(
853 CGF,
854 cast<OpaqueValueExpr>(
855 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
856 RValue::get(V: Size));
857 CGF.EmitVariablyModifiedType(Ty: PrivateType);
858}
859
860void ReductionCodeGen::emitInitialization(
861 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
862 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
863 assert(SharedAddresses.size() > N && "No variable was generated");
864 const auto *PrivateVD =
865 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
866 const OMPDeclareReductionDecl *DRD =
867 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
868 if (CGF.getContext().getAsArrayType(T: PrivateVD->getType())) {
869 if (DRD && DRD->getInitializer())
870 (void)DefaultInit(CGF);
871 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
872 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
873 (void)DefaultInit(CGF);
874 QualType SharedType = SharedAddresses[N].first.getType();
875 emitInitWithReductionInitializer(CGF, DRD, InitOp: ClausesData[N].ReductionOp,
876 Private: PrivateAddr, Original: SharedAddr, Ty: SharedType);
877 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
878 !CGF.isTrivialInitializer(Init: PrivateVD->getInit())) {
879 CGF.EmitAnyExprToMem(E: PrivateVD->getInit(), Location: PrivateAddr,
880 Quals: PrivateVD->getType().getQualifiers(),
881 /*IsInitializer=*/false);
882 }
883}
884
885bool ReductionCodeGen::needCleanups(unsigned N) {
886 QualType PrivateType = getPrivateType(N);
887 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
888 return DTorKind != QualType::DK_none;
889}
890
891void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
892 Address PrivateAddr) {
893 QualType PrivateType = getPrivateType(N);
894 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
895 if (needCleanups(N)) {
896 PrivateAddr =
897 PrivateAddr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: PrivateType));
898 CGF.pushDestroy(dtorKind: DTorKind, addr: PrivateAddr, type: PrivateType);
899 }
900}
901
902static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
903 LValue BaseLV) {
904 BaseTy = BaseTy.getNonReferenceType();
905 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
906 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
907 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
908 BaseLV = CGF.EmitLoadOfPointerLValue(Ptr: BaseLV.getAddress(), PtrTy);
909 } else {
910 LValue RefLVal = CGF.MakeAddrLValue(Addr: BaseLV.getAddress(), T: BaseTy);
911 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
912 }
913 BaseTy = BaseTy->getPointeeType();
914 }
915 return CGF.MakeAddrLValue(
916 Addr: BaseLV.getAddress().withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy)),
917 T: BaseLV.getType(), BaseInfo: BaseLV.getBaseInfo(),
918 TBAAInfo: CGF.CGM.getTBAAInfoForSubobject(Base: BaseLV, AccessType: BaseLV.getType()));
919}
920
921static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
922 Address OriginalBaseAddress, llvm::Value *Addr) {
923 RawAddress Tmp = RawAddress::invalid();
924 Address TopTmp = Address::invalid();
925 Address MostTopTmp = Address::invalid();
926 BaseTy = BaseTy.getNonReferenceType();
927 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
928 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
929 Tmp = CGF.CreateMemTempWithoutCast(T: BaseTy);
930 if (TopTmp.isValid())
931 CGF.Builder.CreateStore(Val: Tmp.getPointer(), Addr: TopTmp);
932 else
933 MostTopTmp = Tmp;
934 TopTmp = Tmp;
935 BaseTy = BaseTy->getPointeeType();
936 }
937
938 if (Tmp.isValid()) {
939 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
940 V: Addr, DestTy: Tmp.getElementType());
941 CGF.Builder.CreateStore(Val: Addr, Addr: Tmp);
942 return MostTopTmp;
943 }
944
945 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
946 V: Addr, DestTy: OriginalBaseAddress.getType());
947 return OriginalBaseAddress.withPointer(NewPointer: Addr, IsKnownNonNull: NotKnownNonNull);
948}
949
950static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
951 const VarDecl *OrigVD = nullptr;
952 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Ref)) {
953 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
954 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Val: Base))
955 Base = TempOASE->getBase()->IgnoreParenImpCasts();
956 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
957 Base = TempASE->getBase()->IgnoreParenImpCasts();
958 DE = cast<DeclRefExpr>(Val: Base);
959 OrigVD = cast<VarDecl>(Val: DE->getDecl());
960 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Ref)) {
961 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
962 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
963 Base = TempASE->getBase()->IgnoreParenImpCasts();
964 DE = cast<DeclRefExpr>(Val: Base);
965 OrigVD = cast<VarDecl>(Val: DE->getDecl());
966 }
967 return OrigVD;
968}
969
970Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
971 Address PrivateAddr) {
972 const DeclRefExpr *DE;
973 if (const VarDecl *OrigVD = ::getBaseDecl(Ref: ClausesData[N].Ref, DE)) {
974 BaseDecls.emplace_back(Args&: OrigVD);
975 LValue OriginalBaseLValue = CGF.EmitLValue(E: DE);
976 LValue BaseLValue =
977 loadToBegin(CGF, BaseTy: OrigVD->getType(), ElTy: SharedAddresses[N].first.getType(),
978 BaseLV: OriginalBaseLValue);
979 Address SharedAddr = SharedAddresses[N].first.getAddress();
980 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
981 ElemTy: SharedAddr.getElementType(), LHS: BaseLValue.getPointer(CGF),
982 RHS: SharedAddr.emitRawPointer(CGF));
983 llvm::Value *PrivatePointer =
984 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
985 V: PrivateAddr.emitRawPointer(CGF), DestTy: SharedAddr.getType());
986 llvm::Value *Ptr = CGF.Builder.CreateGEP(
987 Ty: SharedAddr.getElementType(), Ptr: PrivatePointer, IdxList: Adjustment);
988 return castToBase(CGF, BaseTy: OrigVD->getType(),
989 ElTy: SharedAddresses[N].first.getType(),
990 OriginalBaseAddress: OriginalBaseLValue.getAddress(), Addr: Ptr);
991 }
992 BaseDecls.emplace_back(
993 Args: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Ref)->getDecl()));
994 return PrivateAddr;
995}
996
997bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
998 const OMPDeclareReductionDecl *DRD =
999 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
1000 return DRD && DRD->getInitializer();
1001}
1002
1003LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1004 return CGF.EmitLoadOfPointerLValue(
1005 Ptr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1006 PtrTy: getThreadIDVariable()->getType()->castAs<PointerType>());
1007}
1008
1009void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1010 if (!CGF.HaveInsertPoint())
1011 return;
1012 // 1.2.2 OpenMP Language Terminology
1013 // Structured block - An executable statement with a single entry at the
1014 // top and a single exit at the bottom.
1015 // The point of exit cannot be a branch out of the structured block.
1016 // longjmp() and throw() must not violate the entry/exit criteria.
1017 CGF.EHStack.pushTerminate();
1018 if (S)
1019 CGF.incrementProfileCounter(S);
1020 CodeGen(CGF);
1021 CGF.EHStack.popTerminate();
1022}
1023
1024LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1025 CodeGenFunction &CGF) {
1026 return CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1027 T: getThreadIDVariable()->getType(),
1028 Source: AlignmentSource::Decl);
1029}
1030
1031static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1032 QualType FieldTy) {
1033 auto *Field = FieldDecl::Create(
1034 C, DC, StartLoc: SourceLocation(), IdLoc: SourceLocation(), /*Id=*/nullptr, T: FieldTy,
1035 TInfo: C.getTrivialTypeSourceInfo(T: FieldTy, Loc: SourceLocation()),
1036 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1037 Field->setAccess(AS_public);
1038 DC->addDecl(D: Field);
1039 return Field;
1040}
1041
1042CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
1043 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1044 KmpCriticalNameTy = llvm::ArrayType::get(ElementType: CGM.Int32Ty, /*NumElements*/ 8);
1045 llvm::OpenMPIRBuilderConfig Config(
1046 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),
1047 CGM.getLangOpts().OpenMPOffloadMandatory,
1048 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,
1049 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);
1050 Config.setDefaultTargetAS(
1051 CGM.getContext().getTargetInfo().getTargetAddressSpace(AS: LangAS::Default));
1052 Config.setRuntimeCC(CGM.getRuntimeCC());
1053
1054 OMPBuilder.setConfig(Config);
1055 OMPBuilder.initialize();
1056 OMPBuilder.loadOffloadInfoMetadata(VFS&: *CGM.getFileSystem(),
1057 HostFilePath: CGM.getLangOpts().OpenMPIsTargetDevice
1058 ? CGM.getLangOpts().OMPHostIRFile
1059 : StringRef{});
1060
1061 // The user forces the compiler to behave as if omp requires
1062 // unified_shared_memory was given.
1063 if (CGM.getLangOpts().OpenMPForceUSM) {
1064 HasRequiresUnifiedSharedMemory = true;
1065 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
1066 }
1067}
1068
1069void CGOpenMPRuntime::clear() {
1070 InternalVars.clear();
1071 // Clean non-target variable declarations possibly used only in debug info.
1072 for (const auto &Data : EmittedNonTargetVariables) {
1073 if (!Data.getValue().pointsToAliveValue())
1074 continue;
1075 auto *GV = dyn_cast<llvm::GlobalVariable>(Val: Data.getValue());
1076 if (!GV)
1077 continue;
1078 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1079 continue;
1080 GV->eraseFromParent();
1081 }
1082}
1083
1084std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1085 return OMPBuilder.createPlatformSpecificName(Parts);
1086}
1087
1088static llvm::Function *
1089emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1090 const Expr *CombinerInitializer, const VarDecl *In,
1091 const VarDecl *Out, bool IsCombiner) {
1092 // void .omp_combiner.(Ty *in, Ty *out);
1093 ASTContext &C = CGM.getContext();
1094 QualType PtrTy = C.getPointerType(T: Ty).withRestrict();
1095 auto *OmpOutParm = ImplicitParamDecl::Create(
1096 C, /*DC=*/nullptr, IdLoc: Out->getLocation(),
1097 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1098 auto *OmpInParm = ImplicitParamDecl::Create(
1099 C, /*DC=*/nullptr, IdLoc: In->getLocation(),
1100 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1101 FunctionArgList Args{OmpOutParm, OmpInParm};
1102 const CGFunctionInfo &FnInfo =
1103 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
1104 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1105 std::string Name = CGM.getOpenMPRuntime().getName(
1106 Parts: {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1107 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
1108 N: Name, M: &CGM.getModule());
1109 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
1110 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1111 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
1112 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
1113 Fn->removeFnAttr(Kind: llvm::Attribute::NoInline);
1114 Fn->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
1115 Fn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1116 }
1117 CodeGenFunction CGF(CGM);
1118 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1119 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1120 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc: In->getLocation(),
1121 StartLoc: Out->getLocation());
1122 CodeGenFunction::OMPPrivateScope Scope(CGF);
1123 Address AddrIn = CGF.GetAddrOfLocalVar(VD: OmpInParm);
1124 Scope.addPrivate(
1125 LocalVD: In, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrIn, PtrTy: PtrTy->castAs<PointerType>())
1126 .getAddress());
1127 Address AddrOut = CGF.GetAddrOfLocalVar(VD: OmpOutParm);
1128 Scope.addPrivate(
1129 LocalVD: Out, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrOut, PtrTy: PtrTy->castAs<PointerType>())
1130 .getAddress());
1131 (void)Scope.Privatize();
1132 if (!IsCombiner && Out->hasInit() &&
1133 !CGF.isTrivialInitializer(Init: Out->getInit())) {
1134 CGF.EmitAnyExprToMem(E: Out->getInit(), Location: CGF.GetAddrOfLocalVar(VD: Out),
1135 Quals: Out->getType().getQualifiers(),
1136 /*IsInitializer=*/true);
1137 }
1138 if (CombinerInitializer)
1139 CGF.EmitIgnoredExpr(E: CombinerInitializer);
1140 Scope.ForceCleanup();
1141 CGF.FinishFunction();
1142 return Fn;
1143}
1144
1145void CGOpenMPRuntime::emitUserDefinedReduction(
1146 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1147 if (UDRMap.count(Val: D) > 0)
1148 return;
1149 llvm::Function *Combiner = emitCombinerOrInitializer(
1150 CGM, Ty: D->getType(), CombinerInitializer: D->getCombiner(),
1151 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerIn())->getDecl()),
1152 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerOut())->getDecl()),
1153 /*IsCombiner=*/true);
1154 llvm::Function *Initializer = nullptr;
1155 if (const Expr *Init = D->getInitializer()) {
1156 Initializer = emitCombinerOrInitializer(
1157 CGM, Ty: D->getType(),
1158 CombinerInitializer: D->getInitializerKind() == OMPDeclareReductionInitKind::Call ? Init
1159 : nullptr,
1160 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitOrig())->getDecl()),
1161 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitPriv())->getDecl()),
1162 /*IsCombiner=*/false);
1163 }
1164 UDRMap.try_emplace(Key: D, Args&: Combiner, Args&: Initializer);
1165 if (CGF)
1166 FunctionUDRMap[CGF->CurFn].push_back(Elt: D);
1167}
1168
1169std::pair<llvm::Function *, llvm::Function *>
1170CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1171 auto I = UDRMap.find(Val: D);
1172 if (I != UDRMap.end())
1173 return I->second;
1174 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1175 return UDRMap.lookup(Val: D);
1176}
1177
1178namespace {
1179// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1180// Builder if one is present.
1181struct PushAndPopStackRAII {
1182 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1183 bool HasCancel, llvm::omp::Directive Kind)
1184 : OMPBuilder(OMPBuilder) {
1185 if (!OMPBuilder)
1186 return;
1187
1188 // The following callback is the crucial part of clangs cleanup process.
1189 //
1190 // NOTE:
1191 // Once the OpenMPIRBuilder is used to create parallel regions (and
1192 // similar), the cancellation destination (Dest below) is determined via
1193 // IP. That means if we have variables to finalize we split the block at IP,
1194 // use the new block (=BB) as destination to build a JumpDest (via
1195 // getJumpDestInCurrentScope(BB)) which then is fed to
1196 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1197 // to push & pop an FinalizationInfo object.
1198 // The FiniCB will still be needed but at the point where the
1199 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1200 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1201 assert(IP.getBlock()->end() == IP.getPoint() &&
1202 "Clang CG should cause non-terminated block!");
1203 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1204 CGF.Builder.restoreIP(IP);
1205 CodeGenFunction::JumpDest Dest =
1206 CGF.getOMPCancelDestination(Kind: OMPD_parallel);
1207 CGF.EmitBranchThroughCleanup(Dest);
1208 return llvm::Error::success();
1209 };
1210
1211 // TODO: Remove this once we emit parallel regions through the
1212 // OpenMPIRBuilder as it can do this setup internally.
1213 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1214 OMPBuilder->pushFinalizationCB(FI: std::move(FI));
1215 }
1216 ~PushAndPopStackRAII() {
1217 if (OMPBuilder)
1218 OMPBuilder->popFinalizationCB();
1219 }
1220 llvm::OpenMPIRBuilder *OMPBuilder;
1221};
1222} // namespace
1223
1224static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1225 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1226 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1227 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1228 assert(ThreadIDVar->getType()->isPointerType() &&
1229 "thread id variable must be of type kmp_int32 *");
1230 CodeGenFunction CGF(CGM, true);
1231 bool HasCancel = false;
1232 if (const auto *OPD = dyn_cast<OMPParallelDirective>(Val: &D))
1233 HasCancel = OPD->hasCancel();
1234 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(Val: &D))
1235 HasCancel = OPD->hasCancel();
1236 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &D))
1237 HasCancel = OPSD->hasCancel();
1238 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(Val: &D))
1239 HasCancel = OPFD->hasCancel();
1240 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(Val: &D))
1241 HasCancel = OPFD->hasCancel();
1242 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(Val: &D))
1243 HasCancel = OPFD->hasCancel();
1244 else if (const auto *OPFD =
1245 dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &D))
1246 HasCancel = OPFD->hasCancel();
1247 else if (const auto *OPFD =
1248 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &D))
1249 HasCancel = OPFD->hasCancel();
1250
1251 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1252 // parallel region to make cancellation barriers work properly.
1253 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1254 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1255 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1256 HasCancel, OutlinedHelperName);
1257 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1258 return CGF.GenerateOpenMPCapturedStmtFunction(S: *CS, D);
1259}
1260
1261std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const {
1262 std::string Suffix = getName(Parts: {"omp_outlined"});
1263 return (Name + Suffix).str();
1264}
1265
1266std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {
1267 return getOutlinedHelperName(Name: CGF.CurFn->getName());
1268}
1269
1270std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const {
1271 std::string Suffix = getName(Parts: {"omp", "reduction", "reduction_func"});
1272 return (Name + Suffix).str();
1273}
1274
1275llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1276 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1277 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1278 const RegionCodeGenTy &CodeGen) {
1279 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_parallel);
1280 return emitParallelOrTeamsOutlinedFunction(
1281 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1282 CodeGen);
1283}
1284
1285llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1286 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1287 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1288 const RegionCodeGenTy &CodeGen) {
1289 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_teams);
1290 return emitParallelOrTeamsOutlinedFunction(
1291 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1292 CodeGen);
1293}
1294
1295llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1296 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1297 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1298 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1299 bool Tied, unsigned &NumberOfParts) {
1300 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1301 PrePostActionTy &) {
1302 llvm::Value *ThreadID = getThreadID(CGF, Loc: D.getBeginLoc());
1303 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
1304 llvm::Value *TaskArgs[] = {
1305 UpLoc, ThreadID,
1306 CGF.EmitLoadOfPointerLValue(Ptr: CGF.GetAddrOfLocalVar(VD: TaskTVar),
1307 PtrTy: TaskTVar->getType()->castAs<PointerType>())
1308 .getPointer(CGF)};
1309 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1310 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
1311 args: TaskArgs);
1312 };
1313 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1314 UntiedCodeGen);
1315 CodeGen.setAction(Action);
1316 assert(!ThreadIDVar->getType()->isPointerType() &&
1317 "thread id variable must be of type kmp_int32 for tasks");
1318 const OpenMPDirectiveKind Region =
1319 isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) ? OMPD_taskloop
1320 : OMPD_task;
1321 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: Region);
1322 bool HasCancel = false;
1323 if (const auto *TD = dyn_cast<OMPTaskDirective>(Val: &D))
1324 HasCancel = TD->hasCancel();
1325 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(Val: &D))
1326 HasCancel = TD->hasCancel();
1327 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(Val: &D))
1328 HasCancel = TD->hasCancel();
1329 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(Val: &D))
1330 HasCancel = TD->hasCancel();
1331
1332 CodeGenFunction CGF(CGM, true);
1333 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1334 InnermostKind, HasCancel, Action);
1335 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1336 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(S: *CS);
1337 if (!Tied)
1338 NumberOfParts = Action.getNumberOfParts();
1339 return Res;
1340}
1341
1342void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1343 bool AtCurrentPoint) {
1344 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1345 assert(!Elem.ServiceInsertPt && "Insert point is set already.");
1346
1347 llvm::Value *Undef = llvm::UndefValue::get(T: CGF.Int32Ty);
1348 if (AtCurrentPoint) {
1349 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",
1350 CGF.Builder.GetInsertBlock());
1351 } else {
1352 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1353 Elem.ServiceInsertPt->insertAfter(InsertPos: CGF.AllocaInsertPt->getIterator());
1354 }
1355}
1356
1357void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1358 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1359 if (Elem.ServiceInsertPt) {
1360 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1361 Elem.ServiceInsertPt = nullptr;
1362 Ptr->eraseFromParent();
1363 }
1364}
1365
1366static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1367 SourceLocation Loc,
1368 SmallString<128> &Buffer) {
1369 llvm::raw_svector_ostream OS(Buffer);
1370 // Build debug location
1371 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1372 OS << ";";
1373 if (auto *DbgInfo = CGF.getDebugInfo())
1374 OS << DbgInfo->remapDIPath(PLoc.getFilename());
1375 else
1376 OS << PLoc.getFilename();
1377 OS << ";";
1378 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1379 OS << FD->getQualifiedNameAsString();
1380 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1381 return OS.str();
1382}
1383
1384llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1385 SourceLocation Loc,
1386 unsigned Flags, bool EmitLoc) {
1387 uint32_t SrcLocStrSize;
1388 llvm::Constant *SrcLocStr;
1389 if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() ==
1390 llvm::codegenoptions::NoDebugInfo) ||
1391 Loc.isInvalid()) {
1392 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1393 } else {
1394 std::string FunctionName;
1395 std::string FileName;
1396 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1397 FunctionName = FD->getQualifiedNameAsString();
1398 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1399 if (auto *DbgInfo = CGF.getDebugInfo())
1400 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
1401 else
1402 FileName = PLoc.getFilename();
1403 unsigned Line = PLoc.getLine();
1404 unsigned Column = PLoc.getColumn();
1405 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1406 Column, SrcLocStrSize);
1407 }
1408 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1409 return OMPBuilder.getOrCreateIdent(
1410 SrcLocStr, SrcLocStrSize, Flags: llvm::omp::IdentFlag(Flags), Reserve2Flags: Reserved2Flags);
1411}
1412
1413llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1414 SourceLocation Loc) {
1415 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1416 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1417 // the clang invariants used below might be broken.
1418 if (CGM.getLangOpts().OpenMPIRBuilder) {
1419 SmallString<128> Buffer;
1420 OMPBuilder.updateToLocation(Loc: CGF.Builder.saveIP());
1421 uint32_t SrcLocStrSize;
1422 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1423 LocStr: getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1424 return OMPBuilder.getOrCreateThreadID(
1425 Ident: OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1426 }
1427
1428 llvm::Value *ThreadID = nullptr;
1429 // Check whether we've already cached a load of the thread id in this
1430 // function.
1431 auto I = OpenMPLocThreadIDMap.find(Val: CGF.CurFn);
1432 if (I != OpenMPLocThreadIDMap.end()) {
1433 ThreadID = I->second.ThreadID;
1434 if (ThreadID != nullptr)
1435 return ThreadID;
1436 }
1437 // If exceptions are enabled, do not use parameter to avoid possible crash.
1438 if (auto *OMPRegionInfo =
1439 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
1440 if (OMPRegionInfo->getThreadIDVariable()) {
1441 // Check if this an outlined function with thread id passed as argument.
1442 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1443 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1444 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1445 !CGF.getLangOpts().CXXExceptions ||
1446 CGF.Builder.GetInsertBlock() == TopBlock ||
1447 !isa<llvm::Instruction>(Val: LVal.getPointer(CGF)) ||
1448 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1449 TopBlock ||
1450 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1451 CGF.Builder.GetInsertBlock()) {
1452 ThreadID = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
1453 // If value loaded in entry block, cache it and use it everywhere in
1454 // function.
1455 if (CGF.Builder.GetInsertBlock() == TopBlock)
1456 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;
1457 return ThreadID;
1458 }
1459 }
1460 }
1461
1462 // This is not an outlined function region - need to call __kmpc_int32
1463 // kmpc_global_thread_num(ident_t *loc).
1464 // Generate thread id value and cache this value for use across the
1465 // function.
1466 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1467 if (!Elem.ServiceInsertPt)
1468 setLocThreadIdInsertPt(CGF);
1469 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1470 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);
1471 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
1472 llvm::CallInst *Call = CGF.Builder.CreateCall(
1473 Callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
1474 FnID: OMPRTL___kmpc_global_thread_num),
1475 Args: emitUpdateLocation(CGF, Loc));
1476 Call->setCallingConv(CGF.getRuntimeCC());
1477 Elem.ThreadID = Call;
1478 return Call;
1479}
1480
1481void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1482 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1483 if (OpenMPLocThreadIDMap.count(Val: CGF.CurFn)) {
1484 clearLocThreadIdInsertPt(CGF);
1485 OpenMPLocThreadIDMap.erase(Val: CGF.CurFn);
1486 }
1487 if (auto I = FunctionUDRMap.find(Val: CGF.CurFn); I != FunctionUDRMap.end()) {
1488 for (const auto *D : I->second)
1489 UDRMap.erase(Val: D);
1490 FunctionUDRMap.erase(I);
1491 }
1492 if (auto I = FunctionUDMMap.find(Val: CGF.CurFn); I != FunctionUDMMap.end()) {
1493 for (const auto *D : I->second)
1494 UDMMap.erase(Val: D);
1495 FunctionUDMMap.erase(I);
1496 }
1497 LastprivateConditionalToTypes.erase(Val: CGF.CurFn);
1498 FunctionToUntiedTaskStackMap.erase(Val: CGF.CurFn);
1499}
1500
1501llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1502 return OMPBuilder.IdentPtr;
1503}
1504
1505static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1506convertDeviceClause(const VarDecl *VD) {
1507 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1508 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1509 if (!DevTy)
1510 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1511
1512 switch ((int)*DevTy) { // Avoid -Wcovered-switch-default
1513 case OMPDeclareTargetDeclAttr::DT_Host:
1514 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1515 break;
1516 case OMPDeclareTargetDeclAttr::DT_NoHost:
1517 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1518 break;
1519 case OMPDeclareTargetDeclAttr::DT_Any:
1520 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1521 break;
1522 default:
1523 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1524 break;
1525 }
1526}
1527
1528static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1529convertCaptureClause(const VarDecl *VD) {
1530 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1531 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1532 if (!MapType)
1533 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1534 switch ((int)*MapType) { // Avoid -Wcovered-switch-default
1535 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1536 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1537 break;
1538 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1539 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1540 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1541 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1542 break;
1543 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local:
1544 // MT_Local variables don't need offload entry (device-local).
1545 llvm_unreachable("MT_Local should not reach convertCaptureClause");
1546 break;
1547 default:
1548 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1549 break;
1550 }
1551}
1552
1553static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
1554 CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder,
1555 SourceLocation BeginLoc, llvm::StringRef ParentName = "") {
1556
1557 auto FileInfoCallBack = [&]() {
1558 SourceManager &SM = CGM.getContext().getSourceManager();
1559 PresumedLoc PLoc = SM.getPresumedLoc(Loc: BeginLoc);
1560
1561 if (!CGM.getFileSystem()->exists(Path: PLoc.getFilename()))
1562 PLoc = SM.getPresumedLoc(Loc: BeginLoc, /*UseLineDirectives=*/false);
1563
1564 return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine());
1565 };
1566
1567 return OMPBuilder.getTargetEntryUniqueInfo(CallBack: FileInfoCallBack,
1568 VFS&: *CGM.getFileSystem(), ParentName);
1569}
1570
1571ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1572 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
1573
1574 auto LinkageForVariable = [&VD, this]() {
1575 return CGM.getLLVMLinkageVarDefinition(VD);
1576 };
1577
1578 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1579
1580 llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(
1581 T: CGM.getContext().getPointerType(T: VD->getType()));
1582 llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar(
1583 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
1584 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
1585 IsExternallyVisible: VD->isExternallyVisible(),
1586 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
1587 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
1588 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
1589 TargetTriple: CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, GlobalInitializer: AddrOfGlobal,
1590 VariableLinkage: LinkageForVariable);
1591
1592 if (!addr)
1593 return ConstantAddress::invalid();
1594 return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(D: VD));
1595}
1596
1597llvm::Constant *
1598CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1599 assert(!CGM.getLangOpts().OpenMPUseTLS ||
1600 !CGM.getContext().getTargetInfo().isTLSSupported());
1601 // Lookup the entry, lazily creating it if necessary.
1602 std::string Suffix = getName(Parts: {"cache", ""});
1603 return OMPBuilder.getOrCreateInternalVariable(
1604 Ty: CGM.Int8PtrPtrTy, Name: Twine(CGM.getMangledName(GD: VD)).concat(Suffix).str());
1605}
1606
1607Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1608 const VarDecl *VD,
1609 Address VDAddr,
1610 SourceLocation Loc) {
1611 if (CGM.getLangOpts().OpenMPUseTLS &&
1612 CGM.getContext().getTargetInfo().isTLSSupported())
1613 return VDAddr;
1614
1615 llvm::Type *VarTy = VDAddr.getElementType();
1616 llvm::Value *Args[] = {
1617 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1618 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy),
1619 CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy)),
1620 getOrCreateThreadPrivateCache(VD)};
1621 return Address(
1622 CGF.EmitRuntimeCall(
1623 callee: OMPBuilder.getOrCreateRuntimeFunction(
1624 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1625 args: Args),
1626 CGF.Int8Ty, VDAddr.getAlignment());
1627}
1628
1629void CGOpenMPRuntime::emitThreadPrivateVarInit(
1630 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1631 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1632 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1633 // library.
1634 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1635 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1636 M&: CGM.getModule(), FnID: OMPRTL___kmpc_global_thread_num),
1637 args: OMPLoc);
1638 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1639 // to register constructor/destructor for variable.
1640 llvm::Value *Args[] = {
1641 OMPLoc,
1642 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy),
1643 Ctor, CopyCtor, Dtor};
1644 CGF.EmitRuntimeCall(
1645 callee: OMPBuilder.getOrCreateRuntimeFunction(
1646 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_register),
1647 args: Args);
1648}
1649
1650llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1651 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1652 bool PerformInit, CodeGenFunction *CGF) {
1653 if (CGM.getLangOpts().OpenMPUseTLS &&
1654 CGM.getContext().getTargetInfo().isTLSSupported())
1655 return nullptr;
1656
1657 VD = VD->getDefinition(C&: CGM.getContext());
1658 if (VD && ThreadPrivateWithDefinition.insert(key: CGM.getMangledName(GD: VD)).second) {
1659 QualType ASTTy = VD->getType();
1660
1661 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1662 const Expr *Init = VD->getAnyInitializer();
1663 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1664 // Generate function that re-emits the declaration's initializer into the
1665 // threadprivate copy of the variable VD
1666 CodeGenFunction CtorCGF(CGM);
1667 auto *Dst = ImplicitParamDecl::Create(
1668 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1669 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1670
1671 FunctionArgList Args{Dst};
1672 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1673 resultType: CGM.getContext().VoidPtrTy, args: Args);
1674 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1675 std::string Name = getName(Parts: {"__kmpc_global_ctor_", ""});
1676 llvm::Function *Fn =
1677 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1678 CtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidPtrTy, Fn, FnInfo: FI,
1679 Args, Loc, StartLoc: Loc);
1680 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1681 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1682 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1683 Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(T: ASTTy),
1684 VDAddr.getAlignment());
1685 CtorCGF.EmitAnyExprToMem(E: Init, Location: Arg, Quals: Init->getType().getQualifiers(),
1686 /*IsInitializer=*/true);
1687 ArgVal = CtorCGF.EmitLoadOfScalar(
1688 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1689 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1690 CtorCGF.Builder.CreateStore(Val: ArgVal, Addr: CtorCGF.ReturnValue);
1691 CtorCGF.FinishFunction();
1692 Ctor = Fn;
1693 }
1694 if (VD->getType().isDestructedType() != QualType::DK_none) {
1695 // Generate function that emits destructor call for the threadprivate copy
1696 // of the variable VD
1697 CodeGenFunction DtorCGF(CGM);
1698 auto *Dst = ImplicitParamDecl::Create(
1699 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1700 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1701
1702 FunctionArgList Args{Dst};
1703 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1704 resultType: CGM.getContext().VoidTy, args: Args);
1705 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1706 std::string Name = getName(Parts: {"__kmpc_global_dtor_", ""});
1707 llvm::Function *Fn =
1708 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1709 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: DtorCGF);
1710 DtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn, FnInfo: FI, Args,
1711 Loc, StartLoc: Loc);
1712 // Create a scope with an artificial location for the body of this function.
1713 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: DtorCGF);
1714 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1715 Addr: DtorCGF.GetAddrOfLocalVar(VD: Dst),
1716 /*Volatile=*/false, Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1717 DtorCGF.emitDestroy(
1718 addr: Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), type: ASTTy,
1719 destroyer: DtorCGF.getDestroyer(destructionKind: ASTTy.isDestructedType()),
1720 useEHCleanupForArray: DtorCGF.needsEHCleanup(kind: ASTTy.isDestructedType()));
1721 DtorCGF.FinishFunction();
1722 Dtor = Fn;
1723 }
1724 // Do not emit init function if it is not required.
1725 if (!Ctor && !Dtor)
1726 return nullptr;
1727
1728 // Copying constructor for the threadprivate variable.
1729 // Must be NULL - reserved by runtime, but currently it requires that this
1730 // parameter is always NULL. Otherwise it fires assertion.
1731 CopyCtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1732 if (Ctor == nullptr) {
1733 Ctor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1734 }
1735 if (Dtor == nullptr) {
1736 Dtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1737 }
1738 if (!CGF) {
1739 auto *InitFunctionTy =
1740 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg*/ false);
1741 std::string Name = getName(Parts: {"__omp_threadprivate_init_", ""});
1742 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1743 ty: InitFunctionTy, name: Name, FI: CGM.getTypes().arrangeNullaryFunction());
1744 CodeGenFunction InitCGF(CGM);
1745 FunctionArgList ArgList;
1746 InitCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: InitFunction,
1747 FnInfo: CGM.getTypes().arrangeNullaryFunction(), Args: ArgList,
1748 Loc, StartLoc: Loc);
1749 emitThreadPrivateVarInit(CGF&: InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1750 InitCGF.FinishFunction();
1751 return InitFunction;
1752 }
1753 emitThreadPrivateVarInit(CGF&: *CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1754 }
1755 return nullptr;
1756}
1757
1758void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD,
1759 llvm::GlobalValue *GV) {
1760 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1761 OMPDeclareTargetDeclAttr::getActiveAttr(VD: FD);
1762
1763 // We only need to handle active 'indirect' declare target functions.
1764 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1765 return;
1766
1767 // Get a mangled name to store the new device global in.
1768 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1769 CGM, OMPBuilder, BeginLoc: FD->getCanonicalDecl()->getBeginLoc(), ParentName: FD->getName());
1770 SmallString<128> Name;
1771 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1772
1773 // We need to generate a new global to hold the address of the indirectly
1774 // called device function. Doing this allows us to keep the visibility and
1775 // linkage of the associated function unchanged while allowing the runtime to
1776 // access its value.
1777 llvm::GlobalValue *Addr = GV;
1778 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1779 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1780 C&: CGM.getLLVMContext(),
1781 AddressSpace: CGM.getModule().getDataLayout().getProgramAddressSpace());
1782 Addr = new llvm::GlobalVariable(
1783 CGM.getModule(), FnPtrTy,
1784 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1785 nullptr, llvm::GlobalValue::NotThreadLocal,
1786 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1787 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1788 }
1789
1790 // Register the indirect Vtable:
1791 // This is similar to OMPTargetGlobalVarEntryIndirect, except that the
1792 // size field refers to the size of memory pointed to, not the size of
1793 // the pointer symbol itself (which is implicitly the size of a pointer).
1794 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1795 VarName: Name, Addr, VarSize: CGM.GetTargetTypeStoreSize(Ty: CGM.VoidPtrTy).getQuantity(),
1796 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1797 Linkage: llvm::GlobalValue::WeakODRLinkage);
1798}
1799
1800void CGOpenMPRuntime::registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1801 const VarDecl *VD) {
1802 // TODO: add logic to avoid duplicate vtable registrations per
1803 // translation unit; though for external linkage, this should no
1804 // longer be an issue - or at least we can avoid the issue by
1805 // checking for an existing offloading entry. But, perhaps the
1806 // better approach is to defer emission of the vtables and offload
1807 // entries until later (by tracking a list of items that need to be
1808 // emitted).
1809
1810 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1811
1812 // Generate a new externally visible global to point to the
1813 // internally visible vtable. Doing this allows us to keep the
1814 // visibility and linkage of the associated vtable unchanged while
1815 // allowing the runtime to access its value. The externally
1816 // visible global var needs to be emitted with a unique mangled
1817 // name that won't conflict with similarly named (internal)
1818 // vtables in other translation units.
1819
1820 // Register vtable with source location of dynamic object in map
1821 // clause.
1822 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1823 CGM, OMPBuilder, BeginLoc: VD->getCanonicalDecl()->getBeginLoc(),
1824 ParentName: VTable->getName());
1825
1826 llvm::GlobalVariable *Addr = VTable;
1827 SmallString<128> AddrName;
1828 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name&: AddrName, EntryInfo);
1829 AddrName.append(RHS: "addr");
1830
1831 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1832 Addr = new llvm::GlobalVariable(
1833 CGM.getModule(), VTable->getType(),
1834 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, VTable,
1835 AddrName,
1836 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1837 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1838 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1839 }
1840 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1841 VarName: AddrName, Addr: VTable,
1842 VarSize: CGM.getDataLayout().getTypeAllocSize(Ty: VTable->getInitializer()->getType()),
1843 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable,
1844 Linkage: llvm::GlobalValue::WeakODRLinkage);
1845}
1846
1847void CGOpenMPRuntime::emitAndRegisterVTable(CodeGenModule &CGM,
1848 CXXRecordDecl *CXXRecord,
1849 const VarDecl *VD) {
1850 // Register C++ VTable to OpenMP Offload Entry if it's a new
1851 // CXXRecordDecl.
1852 if (CXXRecord && CXXRecord->isDynamicClass() &&
1853 !CGM.getOpenMPRuntime().VTableDeclMap.contains(Val: CXXRecord)) {
1854 auto Res = CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(Key: CXXRecord, Args&: VD);
1855 if (Res.second) {
1856 CGM.EmitVTable(Class: CXXRecord);
1857 CodeGenVTables VTables = CGM.getVTables();
1858 llvm::GlobalVariable *VTablesAddr = VTables.GetAddrOfVTable(RD: CXXRecord);
1859 assert(VTablesAddr && "Expected non-null VTable address");
1860 // Must set VTables to weak since we're emitting them in multiple TUs now
1861 if (VTablesAddr->hasExternalLinkage())
1862 VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1863 CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTable: VTablesAddr, VD);
1864 // Emit VTable for all the fields containing dynamic CXXRecord
1865 for (const FieldDecl *Field : CXXRecord->fields()) {
1866 if (CXXRecordDecl *RecordDecl = Field->getType()->getAsCXXRecordDecl())
1867 emitAndRegisterVTable(CGM, CXXRecord: RecordDecl, VD);
1868 }
1869 // Emit VTable for all dynamic parent class
1870 for (CXXBaseSpecifier &Base : CXXRecord->bases()) {
1871 if (CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl())
1872 emitAndRegisterVTable(CGM, CXXRecord: BaseDecl, VD);
1873 }
1874 }
1875 }
1876}
1877
1878void CGOpenMPRuntime::registerVTable(const OMPExecutableDirective &D) {
1879 // Register VTable by scanning through the map clause of OpenMP target region.
1880 // Get CXXRecordDecl and VarDecl from Expr.
1881 auto GetVTableDecl = [](const Expr *E) {
1882 QualType VDTy = E->getType();
1883 CXXRecordDecl *CXXRecord = nullptr;
1884 if (const auto *RefType = VDTy->getAs<LValueReferenceType>())
1885 VDTy = RefType->getPointeeType();
1886 if (VDTy->isPointerType())
1887 CXXRecord = VDTy->getPointeeType()->getAsCXXRecordDecl();
1888 else
1889 CXXRecord = VDTy->getAsCXXRecordDecl();
1890
1891 const VarDecl *VD = nullptr;
1892 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1893 VD = cast<VarDecl>(Val: DRE->getDecl());
1894 } else if (auto *MRE = dyn_cast<MemberExpr>(Val: E)) {
1895 if (auto *BaseDRE = dyn_cast<DeclRefExpr>(Val: MRE->getBase())) {
1896 if (auto *BaseVD = dyn_cast<VarDecl>(Val: BaseDRE->getDecl()))
1897 VD = BaseVD;
1898 }
1899 }
1900 return std::pair<CXXRecordDecl *, const VarDecl *>(CXXRecord, VD);
1901 };
1902 // Collect VTable from OpenMP map clause.
1903 for (const auto *C : D.getClausesOfKind<OMPMapClause>()) {
1904 for (const auto *E : C->varlist()) {
1905 auto DeclPair = GetVTableDecl(E);
1906 // Ensure VD is not null
1907 if (DeclPair.second)
1908 emitAndRegisterVTable(CGM, CXXRecord: DeclPair.first, VD: DeclPair.second);
1909 }
1910 }
1911}
1912
1913Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1914 QualType VarType,
1915 StringRef Name) {
1916 std::string Suffix = getName(Parts: {"artificial", ""});
1917 llvm::Type *VarLVType = CGF.ConvertTypeForMem(T: VarType);
1918 llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable(
1919 Ty: VarLVType, Name: Twine(Name).concat(Suffix).str());
1920 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1921 CGM.getTarget().isTLSSupported()) {
1922 GAddr->setThreadLocal(/*Val=*/true);
1923 return Address(GAddr, GAddr->getValueType(),
1924 CGM.getContext().getTypeAlignInChars(T: VarType));
1925 }
1926 std::string CacheSuffix = getName(Parts: {"cache", ""});
1927 llvm::Value *Args[] = {
1928 emitUpdateLocation(CGF, Loc: SourceLocation()),
1929 getThreadID(CGF, Loc: SourceLocation()),
1930 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: GAddr, DestTy: CGM.VoidPtrTy),
1931 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: VarType), DestTy: CGM.SizeTy,
1932 /*isSigned=*/false),
1933 OMPBuilder.getOrCreateInternalVariable(
1934 Ty: CGM.VoidPtrPtrTy,
1935 Name: Twine(Name).concat(Suffix).concat(Suffix: CacheSuffix).str())};
1936 return Address(
1937 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1938 V: CGF.EmitRuntimeCall(
1939 callee: OMPBuilder.getOrCreateRuntimeFunction(
1940 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1941 args: Args),
1942 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
1943 VarLVType, CGM.getContext().getTypeAlignInChars(T: VarType));
1944}
1945
1946void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
1947 const RegionCodeGenTy &ThenGen,
1948 const RegionCodeGenTy &ElseGen) {
1949 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1950
1951 // If the condition constant folds and can be elided, try to avoid emitting
1952 // the condition and the dead arm of the if/else.
1953 bool CondConstant;
1954 if (CGF.ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant)) {
1955 if (CondConstant)
1956 ThenGen(CGF);
1957 else
1958 ElseGen(CGF);
1959 return;
1960 }
1961
1962 // Otherwise, the condition did not fold, or we couldn't elide it. Just
1963 // emit the conditional branch.
1964 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
1965 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock(name: "omp_if.else");
1966 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "omp_if.end");
1967 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, /*TrueCount=*/0);
1968
1969 // Emit the 'then' code.
1970 CGF.EmitBlock(BB: ThenBlock);
1971 ThenGen(CGF);
1972 CGF.EmitBranch(Block: ContBlock);
1973 // Emit the 'else' code if present.
1974 // There is no need to emit line number for unconditional branch.
1975 (void)ApplyDebugLocation::CreateEmpty(CGF);
1976 CGF.EmitBlock(BB: ElseBlock);
1977 ElseGen(CGF);
1978 // There is no need to emit line number for unconditional branch.
1979 (void)ApplyDebugLocation::CreateEmpty(CGF);
1980 CGF.EmitBranch(Block: ContBlock);
1981 // Emit the continuation block for code after the if.
1982 CGF.EmitBlock(BB: ContBlock, /*IsFinished=*/true);
1983}
1984
1985void CGOpenMPRuntime::emitParallelCall(
1986 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
1987 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
1988 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
1989 OpenMPSeverityClauseKind Severity, const Expr *Message) {
1990 if (!CGF.HaveInsertPoint())
1991 return;
1992 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1993 auto &M = CGM.getModule();
1994 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
1995 this](CodeGenFunction &CGF, PrePostActionTy &) {
1996 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
1997 llvm::Value *Args[] = {
1998 RTLoc,
1999 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
2000 OutlinedFn};
2001 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2002 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
2003 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2004
2005 llvm::FunctionCallee RTLFn =
2006 OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_fork_call);
2007 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
2008 };
2009 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2010 this](CodeGenFunction &CGF, PrePostActionTy &) {
2011 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2012 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2013 // Build calls:
2014 // __kmpc_serialized_parallel(&Loc, GTid);
2015 llvm::Value *Args[] = {RTLoc, ThreadID};
2016 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2017 M, FnID: OMPRTL___kmpc_serialized_parallel),
2018 args: Args);
2019
2020 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2021 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2022 RawAddress ZeroAddrBound =
2023 CGF.CreateDefaultAlignTempAlloca(Ty: CGF.Int32Ty,
2024 /*Name=*/".bound.zero.addr");
2025 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(/*C*/ 0), Addr: ZeroAddrBound);
2026 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2027 // ThreadId for serialized parallels is 0.
2028 OutlinedFnArgs.push_back(Elt: ThreadIDAddr.emitRawPointer(CGF));
2029 OutlinedFnArgs.push_back(Elt: ZeroAddrBound.getPointer());
2030 OutlinedFnArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2031
2032 // Ensure we do not inline the function. This is trivially true for the ones
2033 // passed to __kmpc_fork_call but the ones called in serialized regions
2034 // could be inlined. This is not a perfect but it is closer to the invariant
2035 // we want, namely, every data environment starts with a new function.
2036 // TODO: We should pass the if condition to the runtime function and do the
2037 // handling there. Much cleaner code.
2038 OutlinedFn->removeFnAttr(Kind: llvm::Attribute::AlwaysInline);
2039 OutlinedFn->addFnAttr(Kind: llvm::Attribute::NoInline);
2040 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, Args: OutlinedFnArgs);
2041
2042 // __kmpc_end_serialized_parallel(&Loc, GTid);
2043 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2044 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2045 M, FnID: OMPRTL___kmpc_end_serialized_parallel),
2046 args: EndArgs);
2047 };
2048 if (IfCond) {
2049 emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
2050 } else {
2051 RegionCodeGenTy ThenRCG(ThenGen);
2052 ThenRCG(CGF);
2053 }
2054}
2055
2056// If we're inside an (outlined) parallel region, use the region info's
2057// thread-ID variable (it is passed in a first argument of the outlined function
2058// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2059// regular serial code region, get thread ID by calling kmp_int32
2060// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2061// return the address of that temp.
2062Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2063 SourceLocation Loc) {
2064 if (auto *OMPRegionInfo =
2065 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2066 if (OMPRegionInfo->getThreadIDVariable())
2067 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2068
2069 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2070 QualType Int32Ty =
2071 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2072 Address ThreadIDTemp =
2073 CGF.CreateMemTempWithoutCast(T: Int32Ty, /*Name*/ ".threadid_temp.");
2074 CGF.EmitStoreOfScalar(value: ThreadID,
2075 lvalue: CGF.MakeAddrLValue(Addr: ThreadIDTemp, T: Int32Ty));
2076
2077 return ThreadIDTemp;
2078}
2079
2080llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2081 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2082 std::string Name = getName(Parts: {Prefix, "var"});
2083 llvm::GlobalVariable *GV =
2084 OMPBuilder.getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
2085 CGM.setDSOLocal(GV);
2086 return GV;
2087}
2088
2089namespace {
2090/// Common pre(post)-action for different OpenMP constructs.
2091class CommonActionTy final : public PrePostActionTy {
2092 llvm::FunctionCallee EnterCallee;
2093 ArrayRef<llvm::Value *> EnterArgs;
2094 llvm::FunctionCallee ExitCallee;
2095 ArrayRef<llvm::Value *> ExitArgs;
2096 bool Conditional;
2097 llvm::BasicBlock *ContBlock = nullptr;
2098
2099public:
2100 CommonActionTy(llvm::FunctionCallee EnterCallee,
2101 ArrayRef<llvm::Value *> EnterArgs,
2102 llvm::FunctionCallee ExitCallee,
2103 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2104 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2105 ExitArgs(ExitArgs), Conditional(Conditional) {}
2106 void Enter(CodeGenFunction &CGF) override {
2107 llvm::Value *EnterRes = CGF.EmitRuntimeCall(callee: EnterCallee, args: EnterArgs);
2108 if (Conditional) {
2109 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(Arg: EnterRes);
2110 auto *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
2111 ContBlock = CGF.createBasicBlock(name: "omp_if.end");
2112 // Generate the branch (If-stmt)
2113 CGF.Builder.CreateCondBr(Cond: CallBool, True: ThenBlock, False: ContBlock);
2114 CGF.EmitBlock(BB: ThenBlock);
2115 }
2116 }
2117 void Done(CodeGenFunction &CGF) {
2118 // Emit the rest of blocks/branches
2119 CGF.EmitBranch(Block: ContBlock);
2120 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
2121 }
2122 void Exit(CodeGenFunction &CGF) override {
2123 CGF.EmitRuntimeCall(callee: ExitCallee, args: ExitArgs);
2124 }
2125};
2126} // anonymous namespace
2127
2128void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2129 StringRef CriticalName,
2130 const RegionCodeGenTy &CriticalOpGen,
2131 SourceLocation Loc, const Expr *Hint) {
2132 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2133 // CriticalOpGen();
2134 // __kmpc_end_critical(ident_t *, gtid, Lock);
2135 // Prepare arguments and build a call to __kmpc_critical
2136 if (!CGF.HaveInsertPoint())
2137 return;
2138 llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction(
2139 M&: CGM.getModule(),
2140 FnID: Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);
2141 llvm::Value *LockVar = getCriticalRegionLock(CriticalName);
2142 unsigned LockVarArgIdx = 2;
2143 if (cast<llvm::GlobalVariable>(Val: LockVar)->getAddressSpace() !=
2144 RuntimeFcn.getFunctionType()
2145 ->getParamType(i: LockVarArgIdx)
2146 ->getPointerAddressSpace())
2147 LockVar = CGF.Builder.CreateAddrSpaceCast(
2148 V: LockVar, DestTy: RuntimeFcn.getFunctionType()->getParamType(i: LockVarArgIdx));
2149 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2150 LockVar};
2151 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args),
2152 std::end(arr&: Args));
2153 if (Hint) {
2154 EnterArgs.push_back(Elt: CGF.Builder.CreateIntCast(
2155 V: CGF.EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, /*isSigned=*/false));
2156 }
2157 CommonActionTy Action(RuntimeFcn, EnterArgs,
2158 OMPBuilder.getOrCreateRuntimeFunction(
2159 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_critical),
2160 Args);
2161 CriticalOpGen.setAction(Action);
2162 emitInlinedDirective(CGF, InnermostKind: OMPD_critical, CodeGen: CriticalOpGen);
2163}
2164
2165void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2166 const RegionCodeGenTy &MasterOpGen,
2167 SourceLocation Loc) {
2168 if (!CGF.HaveInsertPoint())
2169 return;
2170 // if(__kmpc_master(ident_t *, gtid)) {
2171 // MasterOpGen();
2172 // __kmpc_end_master(ident_t *, gtid);
2173 // }
2174 // Prepare arguments and build a call to __kmpc_master
2175 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2176 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2177 M&: CGM.getModule(), FnID: OMPRTL___kmpc_master),
2178 Args,
2179 OMPBuilder.getOrCreateRuntimeFunction(
2180 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_master),
2181 Args,
2182 /*Conditional=*/true);
2183 MasterOpGen.setAction(Action);
2184 emitInlinedDirective(CGF, InnermostKind: OMPD_master, CodeGen: MasterOpGen);
2185 Action.Done(CGF);
2186}
2187
2188void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,
2189 const RegionCodeGenTy &MaskedOpGen,
2190 SourceLocation Loc, const Expr *Filter) {
2191 if (!CGF.HaveInsertPoint())
2192 return;
2193 // if(__kmpc_masked(ident_t *, gtid, filter)) {
2194 // MaskedOpGen();
2195 // __kmpc_end_masked(iden_t *, gtid);
2196 // }
2197 // Prepare arguments and build a call to __kmpc_masked
2198 llvm::Value *FilterVal = Filter
2199 ? CGF.EmitScalarExpr(E: Filter, IgnoreResultAssign: CGF.Int32Ty)
2200 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
2201 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2202 FilterVal};
2203 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2204 getThreadID(CGF, Loc)};
2205 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2206 M&: CGM.getModule(), FnID: OMPRTL___kmpc_masked),
2207 Args,
2208 OMPBuilder.getOrCreateRuntimeFunction(
2209 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_masked),
2210 ArgsEnd,
2211 /*Conditional=*/true);
2212 MaskedOpGen.setAction(Action);
2213 emitInlinedDirective(CGF, InnermostKind: OMPD_masked, CodeGen: MaskedOpGen);
2214 Action.Done(CGF);
2215}
2216
2217void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2218 SourceLocation Loc) {
2219 if (!CGF.HaveInsertPoint())
2220 return;
2221 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2222 OMPBuilder.createTaskyield(Loc: CGF.Builder);
2223 } else {
2224 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2225 llvm::Value *Args[] = {
2226 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2227 llvm::ConstantInt::get(Ty: CGM.IntTy, /*V=*/0, /*isSigned=*/IsSigned: true)};
2228 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2229 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_taskyield),
2230 args: Args);
2231 }
2232
2233 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2234 Region->emitUntiedSwitch(CGF);
2235}
2236
2237void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2238 const RegionCodeGenTy &TaskgroupOpGen,
2239 SourceLocation Loc) {
2240 if (!CGF.HaveInsertPoint())
2241 return;
2242 // __kmpc_taskgroup(ident_t *, gtid);
2243 // TaskgroupOpGen();
2244 // __kmpc_end_taskgroup(ident_t *, gtid);
2245 // Prepare arguments and build a call to __kmpc_taskgroup
2246 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2247 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2248 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskgroup),
2249 Args,
2250 OMPBuilder.getOrCreateRuntimeFunction(
2251 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_taskgroup),
2252 Args);
2253 TaskgroupOpGen.setAction(Action);
2254 emitInlinedDirective(CGF, InnermostKind: OMPD_taskgroup, CodeGen: TaskgroupOpGen);
2255}
2256
2257/// Given an array of pointers to variables, project the address of a
2258/// given variable.
2259static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2260 unsigned Index, const VarDecl *Var) {
2261 // Pull out the pointer to the variable.
2262 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Addr: Array, Index);
2263 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: PtrAddr);
2264
2265 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: Var->getType());
2266 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(D: Var));
2267}
2268
2269static llvm::Value *emitCopyprivateCopyFunction(
2270 CodeGenModule &CGM, llvm::Type *ArgsElemType,
2271 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2272 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2273 SourceLocation Loc) {
2274 ASTContext &C = CGM.getContext();
2275 // void copy_func(void *LHSArg, void *RHSArg);
2276
2277 auto *LHSArg =
2278 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2279 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2280 auto *RHSArg =
2281 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2282 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2283 FunctionArgList Args{LHSArg, RHSArg};
2284 const auto &CGFI =
2285 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
2286 std::string Name =
2287 CGM.getOpenMPRuntime().getName(Parts: {"omp", "copyprivate", "copy_func"});
2288 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
2289 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
2290 M: &CGM.getModule());
2291 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
2292 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
2293 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
2294 Fn->setDoesNotRecurse();
2295 CodeGenFunction CGF(CGM);
2296 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
2297 // Dest = (void*[n])(LHSArg);
2298 // Src = (void*[n])(RHSArg);
2299 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2300 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
2301 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2302 ArgsElemType, CGF.getPointerAlign());
2303 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2304 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
2305 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2306 ArgsElemType, CGF.getPointerAlign());
2307 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2308 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2309 // ...
2310 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2311 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2312 const auto *DestVar =
2313 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: DestExprs[I])->getDecl());
2314 Address DestAddr = emitAddrOfVarFromArray(CGF, Array: LHS, Index: I, Var: DestVar);
2315
2316 const auto *SrcVar =
2317 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: SrcExprs[I])->getDecl());
2318 Address SrcAddr = emitAddrOfVarFromArray(CGF, Array: RHS, Index: I, Var: SrcVar);
2319
2320 const auto *VD = cast<DeclRefExpr>(Val: CopyprivateVars[I])->getDecl();
2321 QualType Type = VD->getType();
2322 CGF.EmitOMPCopy(OriginalType: Type, DestAddr, SrcAddr, DestVD: DestVar, SrcVD: SrcVar, Copy: AssignmentOps[I]);
2323 }
2324 CGF.FinishFunction();
2325 return Fn;
2326}
2327
2328void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2329 const RegionCodeGenTy &SingleOpGen,
2330 SourceLocation Loc,
2331 ArrayRef<const Expr *> CopyprivateVars,
2332 ArrayRef<const Expr *> SrcExprs,
2333 ArrayRef<const Expr *> DstExprs,
2334 ArrayRef<const Expr *> AssignmentOps) {
2335 if (!CGF.HaveInsertPoint())
2336 return;
2337 assert(CopyprivateVars.size() == SrcExprs.size() &&
2338 CopyprivateVars.size() == DstExprs.size() &&
2339 CopyprivateVars.size() == AssignmentOps.size());
2340 ASTContext &C = CGM.getContext();
2341 // int32 did_it = 0;
2342 // if(__kmpc_single(ident_t *, gtid)) {
2343 // SingleOpGen();
2344 // __kmpc_end_single(ident_t *, gtid);
2345 // did_it = 1;
2346 // }
2347 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2348 // <copy_func>, did_it);
2349
2350 Address DidIt = Address::invalid();
2351 if (!CopyprivateVars.empty()) {
2352 // int32 did_it = 0;
2353 QualType KmpInt32Ty =
2354 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2355 DidIt = CGF.CreateMemTempWithoutCast(T: KmpInt32Ty, Name: ".omp.copyprivate.did_it");
2356 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 0), Addr: DidIt);
2357 }
2358 // Prepare arguments and build a call to __kmpc_single
2359 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2360 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2361 M&: CGM.getModule(), FnID: OMPRTL___kmpc_single),
2362 Args,
2363 OMPBuilder.getOrCreateRuntimeFunction(
2364 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_single),
2365 Args,
2366 /*Conditional=*/true);
2367 SingleOpGen.setAction(Action);
2368 emitInlinedDirective(CGF, InnermostKind: OMPD_single, CodeGen: SingleOpGen);
2369 if (DidIt.isValid()) {
2370 // did_it = 1;
2371 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 1), Addr: DidIt);
2372 }
2373 Action.Done(CGF);
2374 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2375 // <copy_func>, did_it);
2376 if (DidIt.isValid()) {
2377 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2378 QualType CopyprivateArrayTy = C.getConstantArrayType(
2379 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
2380 /*IndexTypeQuals=*/0);
2381 // Create a list of all private variables for copyprivate.
2382 Address CopyprivateList = CGF.CreateMemTempWithoutCast(
2383 T: CopyprivateArrayTy, Name: ".omp.copyprivate.cpr_list");
2384 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2385 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: CopyprivateList, Index: I);
2386 CGF.Builder.CreateStore(
2387 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2388 V: CGF.EmitLValue(E: CopyprivateVars[I]).getPointer(CGF),
2389 DestTy: CGF.VoidPtrTy),
2390 Addr: Elem);
2391 }
2392 // Build function that copies private values from single region to all other
2393 // threads in the corresponding parallel region.
2394 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2395 CGM, ArgsElemType: CGF.ConvertTypeForMem(T: CopyprivateArrayTy), CopyprivateVars,
2396 DestExprs: SrcExprs, SrcExprs: DstExprs, AssignmentOps, Loc);
2397 llvm::Value *BufSize = CGF.getTypeSize(Ty: CopyprivateArrayTy);
2398 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2399 Addr: CopyprivateList, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
2400 llvm::Value *DidItVal = CGF.Builder.CreateLoad(Addr: DidIt);
2401 llvm::Value *Args[] = {
2402 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2403 getThreadID(CGF, Loc), // i32 <gtid>
2404 BufSize, // size_t <buf_size>
2405 CL.emitRawPointer(CGF), // void *<copyprivate list>
2406 CpyFn, // void (*) (void *, void *) <copy_func>
2407 DidItVal // i32 did_it
2408 };
2409 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2410 M&: CGM.getModule(), FnID: OMPRTL___kmpc_copyprivate),
2411 args: Args);
2412 }
2413}
2414
2415void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2416 const RegionCodeGenTy &OrderedOpGen,
2417 SourceLocation Loc, bool IsThreads) {
2418 if (!CGF.HaveInsertPoint())
2419 return;
2420 // __kmpc_ordered(ident_t *, gtid);
2421 // OrderedOpGen();
2422 // __kmpc_end_ordered(ident_t *, gtid);
2423 // Prepare arguments and build a call to __kmpc_ordered
2424 if (IsThreads) {
2425 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2426 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2427 M&: CGM.getModule(), FnID: OMPRTL___kmpc_ordered),
2428 Args,
2429 OMPBuilder.getOrCreateRuntimeFunction(
2430 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_ordered),
2431 Args);
2432 OrderedOpGen.setAction(Action);
2433 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2434 return;
2435 }
2436 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2437}
2438
2439unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2440 unsigned Flags;
2441 if (Kind == OMPD_for)
2442 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2443 else if (Kind == OMPD_sections)
2444 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2445 else if (Kind == OMPD_single)
2446 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2447 else if (Kind == OMPD_barrier)
2448 Flags = OMP_IDENT_BARRIER_EXPL;
2449 else
2450 Flags = OMP_IDENT_BARRIER_IMPL;
2451 return Flags;
2452}
2453
2454void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2455 CodeGenFunction &CGF, const OMPLoopDirective &S,
2456 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2457 // Check if the loop directive is actually a doacross loop directive. In this
2458 // case choose static, 1 schedule.
2459 if (llvm::any_of(
2460 Range: S.getClausesOfKind<OMPOrderedClause>(),
2461 P: [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2462 ScheduleKind = OMPC_SCHEDULE_static;
2463 // Chunk size is 1 in this case.
2464 llvm::APInt ChunkSize(32, 1);
2465 ChunkExpr = IntegerLiteral::Create(
2466 C: CGF.getContext(), V: ChunkSize,
2467 type: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
2468 l: SourceLocation());
2469 }
2470}
2471
2472void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2473 OpenMPDirectiveKind Kind, bool EmitChecks,
2474 bool ForceSimpleCall) {
2475 // Check if we should use the OMPBuilder
2476 auto *OMPRegionInfo =
2477 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo);
2478 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2479 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2480 cantFail(ValOrErr: OMPBuilder.createBarrier(Loc: CGF.Builder, Kind, ForceSimpleCall,
2481 CheckCancelFlag: EmitChecks));
2482 CGF.Builder.restoreIP(IP: AfterIP);
2483 return;
2484 }
2485
2486 if (!CGF.HaveInsertPoint())
2487 return;
2488 // Build call __kmpc_cancel_barrier(loc, thread_id);
2489 // Build call __kmpc_barrier(loc, thread_id);
2490 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2491 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2492 // thread_id);
2493 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2494 getThreadID(CGF, Loc)};
2495 if (OMPRegionInfo) {
2496 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2497 llvm::Value *Result = CGF.EmitRuntimeCall(
2498 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
2499 FnID: OMPRTL___kmpc_cancel_barrier),
2500 args: Args);
2501 if (EmitChecks) {
2502 // if (__kmpc_cancel_barrier()) {
2503 // exit from construct;
2504 // }
2505 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
2506 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
2507 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
2508 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
2509 CGF.EmitBlock(BB: ExitBB);
2510 // exit from construct;
2511 CodeGenFunction::JumpDest CancelDestination =
2512 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
2513 CGF.EmitBranchThroughCleanup(Dest: CancelDestination);
2514 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
2515 }
2516 return;
2517 }
2518 }
2519 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2520 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
2521 args: Args);
2522}
2523
2524void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,
2525 Expr *ME, bool IsFatal) {
2526 llvm::Value *MVL = ME ? CGF.EmitScalarExpr(E: ME)
2527 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2528 // Build call void __kmpc_error(ident_t *loc, int severity, const char
2529 // *message)
2530 llvm::Value *Args[] = {
2531 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/EmitLoc: true),
2532 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: IsFatal ? 2 : 1),
2533 CGF.Builder.CreatePointerCast(V: MVL, DestTy: CGM.Int8PtrTy)};
2534 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2535 M&: CGM.getModule(), FnID: OMPRTL___kmpc_error),
2536 args: Args);
2537}
2538
2539/// Map the OpenMP loop schedule to the runtime enumeration.
2540static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2541 bool Chunked, bool Ordered) {
2542 switch (ScheduleKind) {
2543 case OMPC_SCHEDULE_static:
2544 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2545 : (Ordered ? OMP_ord_static : OMP_sch_static);
2546 case OMPC_SCHEDULE_dynamic:
2547 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2548 case OMPC_SCHEDULE_guided:
2549 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2550 case OMPC_SCHEDULE_runtime:
2551 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2552 case OMPC_SCHEDULE_auto:
2553 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2554 case OMPC_SCHEDULE_unknown:
2555 assert(!Chunked && "chunk was specified but schedule kind not known");
2556 return Ordered ? OMP_ord_static : OMP_sch_static;
2557 }
2558 llvm_unreachable("Unexpected runtime schedule");
2559}
2560
2561/// Map the OpenMP distribute schedule to the runtime enumeration.
2562static OpenMPSchedType
2563getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2564 // only static is allowed for dist_schedule
2565 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2566}
2567
2568bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2569 bool Chunked) const {
2570 OpenMPSchedType Schedule =
2571 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2572 return Schedule == OMP_sch_static;
2573}
2574
2575bool CGOpenMPRuntime::isStaticNonchunked(
2576 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2577 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2578 return Schedule == OMP_dist_sch_static;
2579}
2580
2581bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2582 bool Chunked) const {
2583 OpenMPSchedType Schedule =
2584 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2585 return Schedule == OMP_sch_static_chunked;
2586}
2587
2588bool CGOpenMPRuntime::isStaticChunked(
2589 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2590 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2591 return Schedule == OMP_dist_sch_static_chunked;
2592}
2593
2594bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2595 OpenMPSchedType Schedule =
2596 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2597 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
2598 return Schedule != OMP_sch_static;
2599}
2600
2601static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2602 OpenMPScheduleClauseModifier M1,
2603 OpenMPScheduleClauseModifier M2) {
2604 int Modifier = 0;
2605 switch (M1) {
2606 case OMPC_SCHEDULE_MODIFIER_monotonic:
2607 Modifier = OMP_sch_modifier_monotonic;
2608 break;
2609 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2610 Modifier = OMP_sch_modifier_nonmonotonic;
2611 break;
2612 case OMPC_SCHEDULE_MODIFIER_simd:
2613 if (Schedule == OMP_sch_static_chunked)
2614 Schedule = OMP_sch_static_balanced_chunked;
2615 break;
2616 case OMPC_SCHEDULE_MODIFIER_last:
2617 case OMPC_SCHEDULE_MODIFIER_unknown:
2618 break;
2619 }
2620 switch (M2) {
2621 case OMPC_SCHEDULE_MODIFIER_monotonic:
2622 Modifier = OMP_sch_modifier_monotonic;
2623 break;
2624 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2625 Modifier = OMP_sch_modifier_nonmonotonic;
2626 break;
2627 case OMPC_SCHEDULE_MODIFIER_simd:
2628 if (Schedule == OMP_sch_static_chunked)
2629 Schedule = OMP_sch_static_balanced_chunked;
2630 break;
2631 case OMPC_SCHEDULE_MODIFIER_last:
2632 case OMPC_SCHEDULE_MODIFIER_unknown:
2633 break;
2634 }
2635 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2636 // If the static schedule kind is specified or if the ordered clause is
2637 // specified, and if the nonmonotonic modifier is not specified, the effect is
2638 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2639 // modifier is specified, the effect is as if the nonmonotonic modifier is
2640 // specified.
2641 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2642 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2643 Schedule == OMP_sch_static_balanced_chunked ||
2644 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2645 Schedule == OMP_dist_sch_static_chunked ||
2646 Schedule == OMP_dist_sch_static ||
2647 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone))
2648 Modifier = OMP_sch_modifier_nonmonotonic;
2649 }
2650 return Schedule | Modifier;
2651}
2652
2653void CGOpenMPRuntime::emitForDispatchInit(
2654 CodeGenFunction &CGF, SourceLocation Loc,
2655 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2656 bool Ordered, const DispatchRTInput &DispatchValues) {
2657 if (!CGF.HaveInsertPoint())
2658 return;
2659 OpenMPSchedType Schedule = getRuntimeSchedule(
2660 ScheduleKind: ScheduleKind.Schedule, Chunked: DispatchValues.Chunk != nullptr, Ordered);
2661 assert(Ordered ||
2662 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2663 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2664 Schedule != OMP_sch_static_balanced_chunked));
2665 // Call __kmpc_dispatch_init(
2666 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2667 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2668 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2669
2670 // If the Chunk was not specified in the clause - use default value 1.
2671 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2672 : CGF.Builder.getIntN(N: IVSize, C: 1);
2673 llvm::Value *Args[] = {
2674 emitUpdateLocation(CGF, Loc),
2675 getThreadID(CGF, Loc),
2676 CGF.Builder.getInt32(C: addMonoNonMonoModifier(
2677 CGM, Schedule, M1: ScheduleKind.M1, M2: ScheduleKind.M2)), // Schedule type
2678 DispatchValues.LB, // Lower
2679 DispatchValues.UB, // Upper
2680 CGF.Builder.getIntN(N: IVSize, C: 1), // Stride
2681 Chunk // Chunk
2682 };
2683 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2684 args: Args);
2685}
2686
2687void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
2688 SourceLocation Loc) {
2689 if (!CGF.HaveInsertPoint())
2690 return;
2691 // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid);
2692 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2693 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchDeinitFunction(), args: Args);
2694}
2695
2696static void emitForStaticInitCall(
2697 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2698 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2699 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2700 const CGOpenMPRuntime::StaticRTInput &Values) {
2701 if (!CGF.HaveInsertPoint())
2702 return;
2703
2704 assert(!Values.Ordered);
2705 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2706 Schedule == OMP_sch_static_balanced_chunked ||
2707 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2708 Schedule == OMP_dist_sch_static ||
2709 Schedule == OMP_dist_sch_static_chunked ||
2710 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone);
2711
2712 // Call __kmpc_for_static_init(
2713 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2714 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2715 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2716 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2717 llvm::Value *Chunk = Values.Chunk;
2718 if (Chunk == nullptr) {
2719 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2720 Schedule == OMP_dist_sch_static) &&
2721 "expected static non-chunked schedule");
2722 // If the Chunk was not specified in the clause - use default value 1.
2723 Chunk = CGF.Builder.getIntN(N: Values.IVSize, C: 1);
2724 } else {
2725 assert((Schedule == OMP_sch_static_chunked ||
2726 Schedule == OMP_sch_static_balanced_chunked ||
2727 Schedule == OMP_ord_static_chunked ||
2728 Schedule == OMP_dist_sch_static_chunked ||
2729 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) &&
2730 "expected static chunked schedule");
2731 }
2732 llvm::Value *Args[] = {
2733 UpdateLocation,
2734 ThreadId,
2735 CGF.Builder.getInt32(C: addMonoNonMonoModifier(CGM&: CGF.CGM, Schedule, M1,
2736 M2)), // Schedule type
2737 Values.IL.emitRawPointer(CGF), // &isLastIter
2738 Values.LB.emitRawPointer(CGF), // &LB
2739 Values.UB.emitRawPointer(CGF), // &UB
2740 Values.ST.emitRawPointer(CGF), // &Stride
2741 CGF.Builder.getIntN(N: Values.IVSize, C: 1), // Incr
2742 Chunk // Chunk
2743 };
2744 CGF.EmitRuntimeCall(callee: ForStaticInitFunction, args: Args);
2745}
2746
2747void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2748 SourceLocation Loc,
2749 OpenMPDirectiveKind DKind,
2750 const OpenMPScheduleTy &ScheduleKind,
2751 const StaticRTInput &Values) {
2752 OpenMPSchedType ScheduleNum =
2753 ScheduleKind.UseFusedDistChunkSchedule
2754 ? OMP_dist_sch_static_chunked_sch_static_chunkone
2755 : getRuntimeSchedule(ScheduleKind: ScheduleKind.Schedule, Chunked: Values.Chunk != nullptr,
2756 Ordered: Values.Ordered);
2757 assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&
2758 "Expected loop-based or sections-based directive.");
2759 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2760 Flags: isOpenMPLoopDirective(DKind)
2761 ? OMP_IDENT_WORK_LOOP
2762 : OMP_IDENT_WORK_SECTIONS);
2763 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2764 llvm::FunctionCallee StaticInitFunction =
2765 OMPBuilder.createForStaticInitFunction(IVSize: Values.IVSize, IVSigned: Values.IVSigned,
2766 IsGPUDistribute: false);
2767 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2768 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2769 Schedule: ScheduleNum, M1: ScheduleKind.M1, M2: ScheduleKind.M2, Values);
2770}
2771
2772void CGOpenMPRuntime::emitDistributeStaticInit(
2773 CodeGenFunction &CGF, SourceLocation Loc,
2774 OpenMPDistScheduleClauseKind SchedKind,
2775 const CGOpenMPRuntime::StaticRTInput &Values) {
2776 OpenMPSchedType ScheduleNum =
2777 getRuntimeSchedule(ScheduleKind: SchedKind, Chunked: Values.Chunk != nullptr);
2778 llvm::Value *UpdatedLocation =
2779 emitUpdateLocation(CGF, Loc, Flags: OMP_IDENT_WORK_DISTRIBUTE);
2780 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2781 llvm::FunctionCallee StaticInitFunction;
2782 bool isGPUDistribute =
2783 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU();
2784 StaticInitFunction = OMPBuilder.createForStaticInitFunction(
2785 IVSize: Values.IVSize, IVSigned: Values.IVSigned, IsGPUDistribute: isGPUDistribute);
2786
2787 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2788 Schedule: ScheduleNum, M1: OMPC_SCHEDULE_MODIFIER_unknown,
2789 M2: OMPC_SCHEDULE_MODIFIER_unknown, Values);
2790}
2791
2792void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2793 SourceLocation Loc,
2794 OpenMPDirectiveKind DKind) {
2795 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2796 DKind == OMPD_sections) &&
2797 "Expected distribute, for, or sections directive kind");
2798 if (!CGF.HaveInsertPoint())
2799 return;
2800 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2801 llvm::Value *Args[] = {
2802 emitUpdateLocation(CGF, Loc,
2803 Flags: isOpenMPDistributeDirective(DKind) ||
2804 (DKind == OMPD_target_teams_loop)
2805 ? OMP_IDENT_WORK_DISTRIBUTE
2806 : isOpenMPLoopDirective(DKind)
2807 ? OMP_IDENT_WORK_LOOP
2808 : OMP_IDENT_WORK_SECTIONS),
2809 getThreadID(CGF, Loc)};
2810 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2811 if (isOpenMPDistributeDirective(DKind) &&
2812 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU())
2813 CGF.EmitRuntimeCall(
2814 callee: OMPBuilder.getOrCreateRuntimeFunction(
2815 M&: CGM.getModule(), FnID: OMPRTL___kmpc_distribute_static_fini),
2816 args: Args);
2817 else
2818 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2819 M&: CGM.getModule(), FnID: OMPRTL___kmpc_for_static_fini),
2820 args: Args);
2821}
2822
2823void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2824 SourceLocation Loc,
2825 unsigned IVSize,
2826 bool IVSigned) {
2827 if (!CGF.HaveInsertPoint())
2828 return;
2829 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2830 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2831 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2832 args: Args);
2833}
2834
2835llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2836 SourceLocation Loc, unsigned IVSize,
2837 bool IVSigned, Address IL,
2838 Address LB, Address UB,
2839 Address ST) {
2840 // Call __kmpc_dispatch_next(
2841 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2842 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2843 // kmp_int[32|64] *p_stride);
2844 llvm::Value *Args[] = {
2845 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2846 IL.emitRawPointer(CGF), // &isLastIter
2847 LB.emitRawPointer(CGF), // &Lower
2848 UB.emitRawPointer(CGF), // &Upper
2849 ST.emitRawPointer(CGF) // &Stride
2850 };
2851 llvm::Value *Call = CGF.EmitRuntimeCall(
2852 callee: OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), args: Args);
2853 return CGF.EmitScalarConversion(
2854 Src: Call, SrcTy: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/1),
2855 DstTy: CGF.getContext().BoolTy, Loc);
2856}
2857
2858llvm::Value *CGOpenMPRuntime::emitMessageClause(CodeGenFunction &CGF,
2859 const Expr *Message,
2860 SourceLocation Loc) {
2861 if (!Message)
2862 return llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2863 return CGF.EmitScalarExpr(E: Message);
2864}
2865
2866llvm::Value *
2867CGOpenMPRuntime::emitSeverityClause(OpenMPSeverityClauseKind Severity,
2868 SourceLocation Loc) {
2869 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is
2870 // as if sev-level is fatal."
2871 return llvm::ConstantInt::get(Ty: CGM.Int32Ty,
2872 V: Severity == OMPC_SEVERITY_warning ? 1 : 2);
2873}
2874
2875void CGOpenMPRuntime::emitNumThreadsClause(
2876 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
2877 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
2878 SourceLocation SeverityLoc, const Expr *Message,
2879 SourceLocation MessageLoc) {
2880 if (!CGF.HaveInsertPoint())
2881 return;
2882 llvm::SmallVector<llvm::Value *, 4> Args(
2883 {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2884 CGF.Builder.CreateIntCast(V: NumThreads, DestTy: CGF.Int32Ty, /*isSigned*/ true)});
2885 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2886 // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity,
2887 // messsage) if strict modifier is used.
2888 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2889 if (Modifier == OMPC_NUMTHREADS_strict) {
2890 FnID = OMPRTL___kmpc_push_num_threads_strict;
2891 Args.push_back(Elt: emitSeverityClause(Severity, Loc: SeverityLoc));
2892 Args.push_back(Elt: emitMessageClause(CGF, Message, Loc: MessageLoc));
2893 }
2894 CGF.EmitRuntimeCall(
2895 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args);
2896}
2897
2898void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2899 ProcBindKind ProcBind,
2900 SourceLocation Loc) {
2901 if (!CGF.HaveInsertPoint())
2902 return;
2903 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");
2904 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2905 llvm::Value *Args[] = {
2906 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2907 llvm::ConstantInt::get(Ty: CGM.IntTy, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
2908 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2909 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_proc_bind),
2910 args: Args);
2911}
2912
2913void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2914 SourceLocation Loc, llvm::AtomicOrdering AO) {
2915 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2916 OMPBuilder.createFlush(Loc: CGF.Builder);
2917 } else {
2918 if (!CGF.HaveInsertPoint())
2919 return;
2920 // Build call void __kmpc_flush(ident_t *loc)
2921 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2922 M&: CGM.getModule(), FnID: OMPRTL___kmpc_flush),
2923 args: emitUpdateLocation(CGF, Loc));
2924 }
2925}
2926
2927namespace {
2928/// Indexes of fields for type kmp_task_t.
2929enum KmpTaskTFields {
2930 /// List of shared variables.
2931 KmpTaskTShareds,
2932 /// Task routine.
2933 KmpTaskTRoutine,
2934 /// Partition id for the untied tasks.
2935 KmpTaskTPartId,
2936 /// Function with call of destructors for private variables.
2937 Data1,
2938 /// Task priority.
2939 Data2,
2940 /// (Taskloops only) Lower bound.
2941 KmpTaskTLowerBound,
2942 /// (Taskloops only) Upper bound.
2943 KmpTaskTUpperBound,
2944 /// (Taskloops only) Stride.
2945 KmpTaskTStride,
2946 /// (Taskloops only) Is last iteration flag.
2947 KmpTaskTLastIter,
2948 /// (Taskloops only) Reduction data.
2949 KmpTaskTReductions,
2950};
2951} // anonymous namespace
2952
2953void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
2954 // If we are in simd mode or there are no entries, we don't need to do
2955 // anything.
2956 if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty())
2957 return;
2958
2959 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2960 [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2961 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
2962 SourceLocation Loc;
2963 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2964 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
2965 E = CGM.getContext().getSourceManager().fileinfo_end();
2966 I != E; ++I) {
2967 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2968 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2969 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
2970 SourceFile: I->getFirst(), Line: EntryInfo.Line, Col: 1);
2971 break;
2972 }
2973 }
2974 }
2975 switch (Kind) {
2976 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2977 CGM.getDiags().Report(Loc,
2978 DiagID: diag::err_target_region_offloading_entry_incorrect)
2979 << EntryInfo.ParentName;
2980 } break;
2981 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2982 CGM.getDiags().Report(
2983 Loc, DiagID: diag::err_target_var_offloading_entry_incorrect_with_parent)
2984 << EntryInfo.ParentName;
2985 } break;
2986 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2987 CGM.getDiags().Report(DiagID: diag::err_target_var_offloading_entry_incorrect);
2988 } break;
2989 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: {
2990 unsigned DiagID = CGM.getDiags().getCustomDiagID(
2991 L: DiagnosticsEngine::Error, FormatString: "Offloading entry for indirect declare "
2992 "target variable is incorrect: the "
2993 "address is invalid.");
2994 CGM.getDiags().Report(DiagID);
2995 } break;
2996 }
2997 };
2998
2999 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
3000}
3001
3002void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3003 if (!KmpRoutineEntryPtrTy) {
3004 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3005 ASTContext &C = CGM.getContext();
3006 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3007 FunctionProtoType::ExtProtoInfo EPI;
3008 KmpRoutineEntryPtrQTy = C.getPointerType(
3009 T: C.getFunctionType(ResultTy: KmpInt32Ty, Args: KmpRoutineEntryTyArgs, EPI));
3010 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(T: KmpRoutineEntryPtrQTy);
3011 }
3012}
3013
3014namespace {
3015struct PrivateHelpersTy {
3016 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3017 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3018 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3019 PrivateElemInit(PrivateElemInit) {}
3020 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3021 const Expr *OriginalRef = nullptr;
3022 const VarDecl *Original = nullptr;
3023 const VarDecl *PrivateCopy = nullptr;
3024 const VarDecl *PrivateElemInit = nullptr;
3025 bool isLocalPrivate() const {
3026 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3027 }
3028};
3029typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3030} // anonymous namespace
3031
3032static bool isAllocatableDecl(const VarDecl *VD) {
3033 const VarDecl *CVD = VD->getCanonicalDecl();
3034 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3035 return false;
3036 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3037 // Use the default allocation.
3038 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3039 !AA->getAllocator());
3040}
3041
3042static RecordDecl *
3043createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3044 if (!Privates.empty()) {
3045 ASTContext &C = CGM.getContext();
3046 // Build struct .kmp_privates_t. {
3047 // /* private vars */
3048 // };
3049 RecordDecl *RD = C.buildImplicitRecord(Name: ".kmp_privates.t");
3050 RD->startDefinition();
3051 for (const auto &Pair : Privates) {
3052 const VarDecl *VD = Pair.second.Original;
3053 QualType Type = VD->getType().getNonReferenceType();
3054 // If the private variable is a local variable with lvalue ref type,
3055 // allocate the pointer instead of the pointee type.
3056 if (Pair.second.isLocalPrivate()) {
3057 if (VD->getType()->isLValueReferenceType())
3058 Type = C.getPointerType(T: Type);
3059 if (isAllocatableDecl(VD))
3060 Type = C.getPointerType(T: Type);
3061 }
3062 FieldDecl *FD = addFieldToRecordDecl(C, DC: RD, FieldTy: Type);
3063 if (VD->hasAttrs()) {
3064 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3065 E(VD->getAttrs().end());
3066 I != E; ++I)
3067 FD->addAttr(A: *I);
3068 }
3069 }
3070 RD->completeDefinition();
3071 return RD;
3072 }
3073 return nullptr;
3074}
3075
3076static RecordDecl *
3077createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3078 QualType KmpInt32Ty,
3079 QualType KmpRoutineEntryPointerQTy) {
3080 ASTContext &C = CGM.getContext();
3081 // Build struct kmp_task_t {
3082 // void * shareds;
3083 // kmp_routine_entry_t routine;
3084 // kmp_int32 part_id;
3085 // kmp_cmplrdata_t data1;
3086 // kmp_cmplrdata_t data2;
3087 // For taskloops additional fields:
3088 // kmp_uint64 lb;
3089 // kmp_uint64 ub;
3090 // kmp_int64 st;
3091 // kmp_int32 liter;
3092 // void * reductions;
3093 // };
3094 RecordDecl *UD = C.buildImplicitRecord(Name: "kmp_cmplrdata_t", TK: TagTypeKind::Union);
3095 UD->startDefinition();
3096 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpInt32Ty);
3097 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpRoutineEntryPointerQTy);
3098 UD->completeDefinition();
3099 CanQualType KmpCmplrdataTy = C.getCanonicalTagType(TD: UD);
3100 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t");
3101 RD->startDefinition();
3102 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3103 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpRoutineEntryPointerQTy);
3104 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3105 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3106 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3107 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3108 QualType KmpUInt64Ty =
3109 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3110 QualType KmpInt64Ty =
3111 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3112 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3113 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3114 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt64Ty);
3115 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3116 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3117 }
3118 RD->completeDefinition();
3119 return RD;
3120}
3121
3122static RecordDecl *
3123createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3124 ArrayRef<PrivateDataTy> Privates) {
3125 ASTContext &C = CGM.getContext();
3126 // Build struct kmp_task_t_with_privates {
3127 // kmp_task_t task_data;
3128 // .kmp_privates_t. privates;
3129 // };
3130 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t_with_privates");
3131 RD->startDefinition();
3132 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpTaskTQTy);
3133 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3134 addFieldToRecordDecl(C, DC: RD, FieldTy: C.getCanonicalTagType(TD: PrivateRD));
3135 RD->completeDefinition();
3136 return RD;
3137}
3138
3139/// Emit a proxy function which accepts kmp_task_t as the second
3140/// argument.
3141/// \code
3142/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3143/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3144/// For taskloops:
3145/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3146/// tt->reductions, tt->shareds);
3147/// return 0;
3148/// }
3149/// \endcode
3150static llvm::Function *
3151emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3152 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3153 QualType KmpTaskTWithPrivatesPtrQTy,
3154 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3155 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3156 llvm::Value *TaskPrivatesMap) {
3157 ASTContext &C = CGM.getContext();
3158 auto *GtidArg =
3159 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3160 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3161 auto *TaskTypeArg = ImplicitParamDecl::Create(
3162 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3163 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3164 FunctionArgList Args{GtidArg, TaskTypeArg};
3165 const auto &TaskEntryFnInfo =
3166 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3167 llvm::FunctionType *TaskEntryTy =
3168 CGM.getTypes().GetFunctionType(Info: TaskEntryFnInfo);
3169 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_entry", ""});
3170 auto *TaskEntry = llvm::Function::Create(
3171 Ty: TaskEntryTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3172 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskEntry, FI: TaskEntryFnInfo);
3173 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3174 TaskEntry->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3175 TaskEntry->setDoesNotRecurse();
3176 CodeGenFunction CGF(CGM);
3177 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: TaskEntry, FnInfo: TaskEntryFnInfo, Args,
3178 Loc, StartLoc: Loc);
3179
3180 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3181 // tt,
3182 // For taskloops:
3183 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3184 // tt->task_data.shareds);
3185 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3186 Addr: CGF.GetAddrOfLocalVar(VD: GtidArg), /*Volatile=*/false, Ty: KmpInt32Ty, Loc);
3187 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3188 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3189 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3190 const auto *KmpTaskTWithPrivatesQTyRD =
3191 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3192 LValue Base =
3193 CGF.EmitLValueForField(Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3194 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3195 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
3196 LValue PartIdLVal = CGF.EmitLValueForField(Base, Field: *PartIdFI);
3197 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3198
3199 auto SharedsFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds);
3200 LValue SharedsLVal = CGF.EmitLValueForField(Base, Field: *SharedsFI);
3201 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3202 V: CGF.EmitLoadOfScalar(lvalue: SharedsLVal, Loc),
3203 DestTy: CGF.ConvertTypeForMem(T: SharedsPtrTy));
3204
3205 auto PrivatesFI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin(), n: 1);
3206 llvm::Value *PrivatesParam;
3207 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3208 LValue PrivatesLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PrivatesFI);
3209 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3210 V: PrivatesLVal.getPointer(CGF), DestTy: CGF.VoidPtrTy);
3211 } else {
3212 PrivatesParam = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
3213 }
3214
3215 llvm::Value *CommonArgs[] = {
3216 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3217 CGF.Builder
3218 .CreatePointerBitCastOrAddrSpaceCast(Addr: TDBase.getAddress(),
3219 Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty)
3220 .emitRawPointer(CGF)};
3221 SmallVector<llvm::Value *, 16> CallArgs(std::begin(arr&: CommonArgs),
3222 std::end(arr&: CommonArgs));
3223 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3224 auto LBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound);
3225 LValue LBLVal = CGF.EmitLValueForField(Base, Field: *LBFI);
3226 llvm::Value *LBParam = CGF.EmitLoadOfScalar(lvalue: LBLVal, Loc);
3227 auto UBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound);
3228 LValue UBLVal = CGF.EmitLValueForField(Base, Field: *UBFI);
3229 llvm::Value *UBParam = CGF.EmitLoadOfScalar(lvalue: UBLVal, Loc);
3230 auto StFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride);
3231 LValue StLVal = CGF.EmitLValueForField(Base, Field: *StFI);
3232 llvm::Value *StParam = CGF.EmitLoadOfScalar(lvalue: StLVal, Loc);
3233 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3234 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3235 llvm::Value *LIParam = CGF.EmitLoadOfScalar(lvalue: LILVal, Loc);
3236 auto RFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions);
3237 LValue RLVal = CGF.EmitLValueForField(Base, Field: *RFI);
3238 llvm::Value *RParam = CGF.EmitLoadOfScalar(lvalue: RLVal, Loc);
3239 CallArgs.push_back(Elt: LBParam);
3240 CallArgs.push_back(Elt: UBParam);
3241 CallArgs.push_back(Elt: StParam);
3242 CallArgs.push_back(Elt: LIParam);
3243 CallArgs.push_back(Elt: RParam);
3244 }
3245 CallArgs.push_back(Elt: SharedsParam);
3246
3247 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskFunction,
3248 Args: CallArgs);
3249 CGF.EmitStoreThroughLValue(Src: RValue::get(V: CGF.Builder.getInt32(/*C=*/0)),
3250 Dst: CGF.MakeAddrLValue(Addr: CGF.ReturnValue, T: KmpInt32Ty));
3251 CGF.FinishFunction();
3252 return TaskEntry;
3253}
3254
3255static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3256 SourceLocation Loc,
3257 QualType KmpInt32Ty,
3258 QualType KmpTaskTWithPrivatesPtrQTy,
3259 QualType KmpTaskTWithPrivatesQTy) {
3260 ASTContext &C = CGM.getContext();
3261 auto *GtidArg =
3262 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3263 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3264 auto *TaskTypeArg = ImplicitParamDecl::Create(
3265 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3266 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3267 FunctionArgList Args{GtidArg, TaskTypeArg};
3268 const auto &DestructorFnInfo =
3269 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3270 llvm::FunctionType *DestructorFnTy =
3271 CGM.getTypes().GetFunctionType(Info: DestructorFnInfo);
3272 std::string Name =
3273 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_destructor", ""});
3274 auto *DestructorFn =
3275 llvm::Function::Create(Ty: DestructorFnTy, Linkage: llvm::GlobalValue::InternalLinkage,
3276 N: Name, M: &CGM.getModule());
3277 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: DestructorFn,
3278 FI: DestructorFnInfo);
3279 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3280 DestructorFn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3281 DestructorFn->setDoesNotRecurse();
3282 CodeGenFunction CGF(CGM);
3283 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: DestructorFn, FnInfo: DestructorFnInfo,
3284 Args, Loc, StartLoc: Loc);
3285
3286 LValue Base = CGF.EmitLoadOfPointerLValue(
3287 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3288 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3289 const auto *KmpTaskTWithPrivatesQTyRD =
3290 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3291 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3292 Base = CGF.EmitLValueForField(Base, Field: *FI);
3293 for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3294 if (QualType::DestructionKind DtorKind =
3295 Field->getType().isDestructedType()) {
3296 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3297 CGF.pushDestroy(dtorKind: DtorKind, addr: FieldLValue.getAddress(), type: Field->getType());
3298 }
3299 }
3300 CGF.FinishFunction();
3301 return DestructorFn;
3302}
3303
3304/// Emit a privates mapping function for correct handling of private and
3305/// firstprivate variables.
3306/// \code
3307/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3308/// **noalias priv1,..., <tyn> **noalias privn) {
3309/// *priv1 = &.privates.priv1;
3310/// ...;
3311/// *privn = &.privates.privn;
3312/// }
3313/// \endcode
3314static llvm::Value *
3315emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3316 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3317 ArrayRef<PrivateDataTy> Privates) {
3318 ASTContext &C = CGM.getContext();
3319 FunctionArgList Args;
3320 auto *TaskPrivatesArg = ImplicitParamDecl::Create(
3321 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3322 T: C.getPointerType(T: PrivatesQTy).withConst().withRestrict(),
3323 ParamKind: ImplicitParamKind::Other);
3324 Args.push_back(Elt: TaskPrivatesArg);
3325 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3326 unsigned Counter = 1;
3327 for (const Expr *E : Data.PrivateVars) {
3328 Args.push_back(Elt: ImplicitParamDecl::Create(
3329 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3330 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3331 .withConst()
3332 .withRestrict(),
3333 ParamKind: ImplicitParamKind::Other));
3334 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3335 PrivateVarsPos[VD] = Counter;
3336 ++Counter;
3337 }
3338 for (const Expr *E : Data.FirstprivateVars) {
3339 Args.push_back(Elt: ImplicitParamDecl::Create(
3340 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3341 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3342 .withConst()
3343 .withRestrict(),
3344 ParamKind: ImplicitParamKind::Other));
3345 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3346 PrivateVarsPos[VD] = Counter;
3347 ++Counter;
3348 }
3349 for (const Expr *E : Data.LastprivateVars) {
3350 Args.push_back(Elt: ImplicitParamDecl::Create(
3351 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3352 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3353 .withConst()
3354 .withRestrict(),
3355 ParamKind: ImplicitParamKind::Other));
3356 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3357 PrivateVarsPos[VD] = Counter;
3358 ++Counter;
3359 }
3360 for (const VarDecl *VD : Data.PrivateLocals) {
3361 QualType Ty = VD->getType().getNonReferenceType();
3362 if (VD->getType()->isLValueReferenceType())
3363 Ty = C.getPointerType(T: Ty);
3364 if (isAllocatableDecl(VD))
3365 Ty = C.getPointerType(T: Ty);
3366 Args.push_back(Elt: ImplicitParamDecl::Create(
3367 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3368 T: C.getPointerType(T: C.getPointerType(T: Ty)).withConst().withRestrict(),
3369 ParamKind: ImplicitParamKind::Other));
3370 PrivateVarsPos[VD] = Counter;
3371 ++Counter;
3372 }
3373 const auto &TaskPrivatesMapFnInfo =
3374 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3375 llvm::FunctionType *TaskPrivatesMapTy =
3376 CGM.getTypes().GetFunctionType(Info: TaskPrivatesMapFnInfo);
3377 std::string Name =
3378 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_privates_map", ""});
3379 auto *TaskPrivatesMap = llvm::Function::Create(
3380 Ty: TaskPrivatesMapTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
3381 M: &CGM.getModule());
3382 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskPrivatesMap,
3383 FI: TaskPrivatesMapFnInfo);
3384 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3385 TaskPrivatesMap->addFnAttr(Kind: "sample-profile-suffix-elision-policy",
3386 Val: "selected");
3387 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
3388 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::NoInline);
3389 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
3390 TaskPrivatesMap->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
3391 }
3392 CodeGenFunction CGF(CGM);
3393 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskPrivatesMap,
3394 FnInfo: TaskPrivatesMapFnInfo, Args, Loc, StartLoc: Loc);
3395
3396 // *privi = &.privates.privi;
3397 LValue Base = CGF.EmitLoadOfPointerLValue(
3398 Ptr: CGF.GetAddrOfLocalVar(VD: TaskPrivatesArg),
3399 PtrTy: TaskPrivatesArg->getType()->castAs<PointerType>());
3400 const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl();
3401 Counter = 0;
3402 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3403 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3404 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3405 LValue RefLVal =
3406 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD), T: VD->getType());
3407 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3408 Ptr: RefLVal.getAddress(), PtrTy: RefLVal.getType()->castAs<PointerType>());
3409 CGF.EmitStoreOfScalar(value: FieldLVal.getPointer(CGF), lvalue: RefLoadLVal);
3410 ++Counter;
3411 }
3412 CGF.FinishFunction();
3413 return TaskPrivatesMap;
3414}
3415
3416/// Emit initialization for private variables in task-based directives.
3417static void emitPrivatesInit(CodeGenFunction &CGF,
3418 const OMPExecutableDirective &D,
3419 Address KmpTaskSharedsPtr, LValue TDBase,
3420 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3421 QualType SharedsTy, QualType SharedsPtrTy,
3422 const OMPTaskDataTy &Data,
3423 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3424 ASTContext &C = CGF.getContext();
3425 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3426 LValue PrivatesBase = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
3427 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())
3428 ? OMPD_taskloop
3429 : OMPD_task;
3430 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: Kind);
3431 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3432 LValue SrcBase;
3433 bool IsTargetTask =
3434 isOpenMPTargetDataManagementDirective(DKind: D.getDirectiveKind()) ||
3435 isOpenMPTargetExecutionDirective(DKind: D.getDirectiveKind());
3436 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3437 // PointersArray, SizesArray, and MappersArray. The original variables for
3438 // these arrays are not captured and we get their addresses explicitly.
3439 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3440 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3441 SrcBase = CGF.MakeAddrLValue(
3442 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3443 Addr: KmpTaskSharedsPtr, Ty: CGF.ConvertTypeForMem(T: SharedsPtrTy),
3444 ElementTy: CGF.ConvertTypeForMem(T: SharedsTy)),
3445 T: SharedsTy);
3446 }
3447 FI = FI->getType()->castAsRecordDecl()->field_begin();
3448 for (const PrivateDataTy &Pair : Privates) {
3449 // Do not initialize private locals.
3450 if (Pair.second.isLocalPrivate()) {
3451 ++FI;
3452 continue;
3453 }
3454 const VarDecl *VD = Pair.second.PrivateCopy;
3455 const Expr *Init = VD->getAnyInitializer();
3456 if (Init && (!ForDup || (isa<CXXConstructExpr>(Val: Init) &&
3457 !CGF.isTrivialInitializer(Init)))) {
3458 LValue PrivateLValue = CGF.EmitLValueForField(Base: PrivatesBase, Field: *FI);
3459 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3460 const VarDecl *OriginalVD = Pair.second.Original;
3461 // Check if the variable is the target-based BasePointersArray,
3462 // PointersArray, SizesArray, or MappersArray.
3463 LValue SharedRefLValue;
3464 QualType Type = PrivateLValue.getType();
3465 const FieldDecl *SharedField = CapturesInfo.lookup(VD: OriginalVD);
3466 if (IsTargetTask && !SharedField) {
3467 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3468 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
3469 cast<CapturedDecl>(OriginalVD->getDeclContext())
3470 ->getNumParams() == 0 &&
3471 isa<TranslationUnitDecl>(
3472 cast<CapturedDecl>(OriginalVD->getDeclContext())
3473 ->getDeclContext()) &&
3474 "Expected artificial target data variable.");
3475 SharedRefLValue =
3476 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: OriginalVD), T: Type);
3477 } else if (ForDup) {
3478 SharedRefLValue = CGF.EmitLValueForField(Base: SrcBase, Field: SharedField);
3479 SharedRefLValue = CGF.MakeAddrLValue(
3480 Addr: SharedRefLValue.getAddress().withAlignment(
3481 NewAlignment: C.getDeclAlign(D: OriginalVD)),
3482 T: SharedRefLValue.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3483 TBAAInfo: SharedRefLValue.getTBAAInfo());
3484 } else if (CGF.LambdaCaptureFields.count(
3485 Val: Pair.second.Original->getCanonicalDecl()) > 0 ||
3486 isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl)) {
3487 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3488 } else {
3489 // Processing for implicitly captured variables.
3490 InlinedOpenMPRegionRAII Region(
3491 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3492 /*HasCancel=*/false, /*NoInheritance=*/true);
3493 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3494 }
3495 if (Type->isArrayType()) {
3496 // Initialize firstprivate array.
3497 if (!isa<CXXConstructExpr>(Val: Init) || CGF.isTrivialInitializer(Init)) {
3498 // Perform simple memcpy.
3499 CGF.EmitAggregateAssign(Dest: PrivateLValue, Src: SharedRefLValue, EltTy: Type);
3500 } else {
3501 // Initialize firstprivate array using element-by-element
3502 // initialization.
3503 CGF.EmitOMPAggregateAssign(
3504 DestAddr: PrivateLValue.getAddress(), SrcAddr: SharedRefLValue.getAddress(), OriginalType: Type,
3505 CopyGen: [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3506 Address SrcElement) {
3507 // Clean up any temporaries needed by the initialization.
3508 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3509 InitScope.addPrivate(LocalVD: Elem, Addr: SrcElement);
3510 (void)InitScope.Privatize();
3511 // Emit initialization for single element.
3512 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3513 CGF, &CapturesInfo);
3514 CGF.EmitAnyExprToMem(E: Init, Location: DestElement,
3515 Quals: Init->getType().getQualifiers(),
3516 /*IsInitializer=*/false);
3517 });
3518 }
3519 } else {
3520 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3521 InitScope.addPrivate(LocalVD: Elem, Addr: SharedRefLValue.getAddress());
3522 (void)InitScope.Privatize();
3523 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3524 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue,
3525 /*capturedByInit=*/false);
3526 }
3527 } else {
3528 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue, /*capturedByInit=*/false);
3529 }
3530 }
3531 ++FI;
3532 }
3533}
3534
3535/// Check if duplication function is required for taskloops.
3536static bool checkInitIsRequired(CodeGenFunction &CGF,
3537 ArrayRef<PrivateDataTy> Privates) {
3538 bool InitRequired = false;
3539 for (const PrivateDataTy &Pair : Privates) {
3540 if (Pair.second.isLocalPrivate())
3541 continue;
3542 const VarDecl *VD = Pair.second.PrivateCopy;
3543 const Expr *Init = VD->getAnyInitializer();
3544 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Val: Init) &&
3545 !CGF.isTrivialInitializer(Init));
3546 if (InitRequired)
3547 break;
3548 }
3549 return InitRequired;
3550}
3551
3552
3553/// Emit task_dup function (for initialization of
3554/// private/firstprivate/lastprivate vars and last_iter flag)
3555/// \code
3556/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3557/// lastpriv) {
3558/// // setup lastprivate flag
3559/// task_dst->last = lastpriv;
3560/// // could be constructor calls here...
3561/// }
3562/// \endcode
3563static llvm::Value *
3564emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3565 const OMPExecutableDirective &D,
3566 QualType KmpTaskTWithPrivatesPtrQTy,
3567 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3568 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3569 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3570 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3571 ASTContext &C = CGM.getContext();
3572 auto *DstArg = ImplicitParamDecl::Create(
3573 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3574 ParamKind: ImplicitParamKind::Other);
3575 auto *SrcArg = ImplicitParamDecl::Create(
3576 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3577 ParamKind: ImplicitParamKind::Other);
3578 auto *LastprivArg =
3579 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: C.IntTy,
3580 ParamKind: ImplicitParamKind::Other);
3581 FunctionArgList Args{DstArg, SrcArg, LastprivArg};
3582 const auto &TaskDupFnInfo =
3583 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3584 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(Info: TaskDupFnInfo);
3585 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_dup", ""});
3586 auto *TaskDup = llvm::Function::Create(
3587 Ty: TaskDupTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3588 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskDup, FI: TaskDupFnInfo);
3589 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3590 TaskDup->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3591 TaskDup->setDoesNotRecurse();
3592 CodeGenFunction CGF(CGM);
3593 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskDup, FnInfo: TaskDupFnInfo, Args, Loc,
3594 StartLoc: Loc);
3595
3596 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3597 Ptr: CGF.GetAddrOfLocalVar(VD: DstArg),
3598 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3599 // task_dst->liter = lastpriv;
3600 if (WithLastIter) {
3601 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3602 LValue Base = CGF.EmitLValueForField(
3603 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3604 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3605 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3606 Addr: CGF.GetAddrOfLocalVar(VD: LastprivArg), /*Volatile=*/false, Ty: C.IntTy, Loc);
3607 CGF.EmitStoreOfScalar(value: Lastpriv, lvalue: LILVal);
3608 }
3609
3610 // Emit initial values for private copies (if any).
3611 assert(!Privates.empty());
3612 Address KmpTaskSharedsPtr = Address::invalid();
3613 if (!Data.FirstprivateVars.empty()) {
3614 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3615 Ptr: CGF.GetAddrOfLocalVar(VD: SrcArg),
3616 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3617 LValue Base = CGF.EmitLValueForField(
3618 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3619 KmpTaskSharedsPtr = Address(
3620 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValueForField(
3621 Base, Field: *std::next(x: KmpTaskTQTyRD->field_begin(),
3622 n: KmpTaskTShareds)),
3623 Loc),
3624 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
3625 }
3626 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3627 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3628 CGF.FinishFunction();
3629 return TaskDup;
3630}
3631
3632/// Checks if destructor function is required to be generated.
3633/// \return true if cleanups are required, false otherwise.
3634static bool
3635checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3636 ArrayRef<PrivateDataTy> Privates) {
3637 for (const PrivateDataTy &P : Privates) {
3638 if (P.second.isLocalPrivate())
3639 continue;
3640 QualType Ty = P.second.Original->getType().getNonReferenceType();
3641 if (Ty.isDestructedType())
3642 return true;
3643 }
3644 return false;
3645}
3646
3647namespace {
3648/// Loop generator for OpenMP iterator expression.
3649class OMPIteratorGeneratorScope final
3650 : public CodeGenFunction::OMPPrivateScope {
3651 CodeGenFunction &CGF;
3652 const OMPIteratorExpr *E = nullptr;
3653 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3654 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3655 OMPIteratorGeneratorScope() = delete;
3656 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
3657
3658public:
3659 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3660 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3661 if (!E)
3662 return;
3663 SmallVector<llvm::Value *, 4> Uppers;
3664 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3665 Uppers.push_back(Elt: CGF.EmitScalarExpr(E: E->getHelper(I).Upper));
3666 const auto *VD = cast<VarDecl>(Val: E->getIteratorDecl(I));
3667 addPrivate(LocalVD: VD, Addr: CGF.CreateMemTemp(T: VD->getType(), Name: VD->getName()));
3668 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3669 addPrivate(
3670 LocalVD: HelperData.CounterVD,
3671 Addr: CGF.CreateMemTemp(T: HelperData.CounterVD->getType(), Name: "counter.addr"));
3672 }
3673 Privatize();
3674
3675 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3676 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3677 LValue CLVal =
3678 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: HelperData.CounterVD),
3679 T: HelperData.CounterVD->getType());
3680 // Counter = 0;
3681 CGF.EmitStoreOfScalar(
3682 value: llvm::ConstantInt::get(Ty: CLVal.getAddress().getElementType(), V: 0),
3683 lvalue: CLVal);
3684 CodeGenFunction::JumpDest &ContDest =
3685 ContDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.cont"));
3686 CodeGenFunction::JumpDest &ExitDest =
3687 ExitDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.exit"));
3688 // N = <number-of_iterations>;
3689 llvm::Value *N = Uppers[I];
3690 // cont:
3691 // if (Counter < N) goto body; else goto exit;
3692 CGF.EmitBlock(BB: ContDest.getBlock());
3693 auto *CVal =
3694 CGF.EmitLoadOfScalar(lvalue: CLVal, Loc: HelperData.CounterVD->getLocation());
3695 llvm::Value *Cmp =
3696 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3697 ? CGF.Builder.CreateICmpSLT(LHS: CVal, RHS: N)
3698 : CGF.Builder.CreateICmpULT(LHS: CVal, RHS: N);
3699 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "iter.body");
3700 CGF.Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitDest.getBlock());
3701 // body:
3702 CGF.EmitBlock(BB: BodyBB);
3703 // Iteri = Begini + Counter * Stepi;
3704 CGF.EmitIgnoredExpr(E: HelperData.Update);
3705 }
3706 }
3707 ~OMPIteratorGeneratorScope() {
3708 if (!E)
3709 return;
3710 for (unsigned I = E->numOfIterators(); I > 0; --I) {
3711 // Counter = Counter + 1;
3712 const OMPIteratorHelperData &HelperData = E->getHelper(I: I - 1);
3713 CGF.EmitIgnoredExpr(E: HelperData.CounterUpdate);
3714 // goto cont;
3715 CGF.EmitBranchThroughCleanup(Dest: ContDests[I - 1]);
3716 // exit:
3717 CGF.EmitBlock(BB: ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3718 }
3719 }
3720};
3721} // namespace
3722
3723static std::pair<llvm::Value *, llvm::Value *>
3724getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
3725 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(Val: E);
3726 llvm::Value *Addr;
3727 if (OASE) {
3728 const Expr *Base = OASE->getBase();
3729 Addr = CGF.EmitScalarExpr(E: Base);
3730 } else {
3731 Addr = CGF.EmitLValue(E).getPointer(CGF);
3732 }
3733 llvm::Value *SizeVal;
3734 QualType Ty = E->getType();
3735 if (OASE) {
3736 SizeVal = CGF.getTypeSize(Ty: OASE->getBase()->getType()->getPointeeType());
3737 for (const Expr *SE : OASE->getDimensions()) {
3738 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
3739 Sz = CGF.EmitScalarConversion(
3740 Src: Sz, SrcTy: SE->getType(), DstTy: CGF.getContext().getSizeType(), Loc: SE->getExprLoc());
3741 SizeVal = CGF.Builder.CreateNUWMul(LHS: SizeVal, RHS: Sz);
3742 }
3743 } else if (const auto *ASE =
3744 dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenImpCasts())) {
3745 LValue UpAddrLVal = CGF.EmitArraySectionExpr(E: ASE, /*IsLowerBound=*/false);
3746 Address UpAddrAddress = UpAddrLVal.getAddress();
3747 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3748 Ty: UpAddrAddress.getElementType(), Ptr: UpAddrAddress.emitRawPointer(CGF),
3749 /*Idx0=*/1);
3750 SizeVal = CGF.Builder.CreatePtrDiff(LHS: UpAddr, RHS: Addr, Name: "", /*IsNUW=*/true);
3751 } else {
3752 SizeVal = CGF.getTypeSize(Ty);
3753 }
3754 return std::make_pair(x&: Addr, y&: SizeVal);
3755}
3756
3757/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
3758static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
3759 QualType FlagsTy = C.getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/false);
3760 if (KmpTaskAffinityInfoTy.isNull()) {
3761 RecordDecl *KmpAffinityInfoRD =
3762 C.buildImplicitRecord(Name: "kmp_task_affinity_info_t");
3763 KmpAffinityInfoRD->startDefinition();
3764 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getIntPtrType());
3765 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getSizeType());
3766 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: FlagsTy);
3767 KmpAffinityInfoRD->completeDefinition();
3768 KmpTaskAffinityInfoTy = C.getCanonicalTagType(TD: KmpAffinityInfoRD);
3769 }
3770}
3771
3772CGOpenMPRuntime::TaskResultTy
3773CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3774 const OMPExecutableDirective &D,
3775 llvm::Function *TaskFunction, QualType SharedsTy,
3776 Address Shareds, const OMPTaskDataTy &Data) {
3777 ASTContext &C = CGM.getContext();
3778 llvm::SmallVector<PrivateDataTy, 4> Privates;
3779 // Aggregate privates and sort them by the alignment.
3780 const auto *I = Data.PrivateCopies.begin();
3781 for (const Expr *E : Data.PrivateVars) {
3782 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3783 Privates.emplace_back(
3784 Args: C.getDeclAlign(D: VD),
3785 Args: PrivateHelpersTy(E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3786 /*PrivateElemInit=*/nullptr));
3787 ++I;
3788 }
3789 I = Data.FirstprivateCopies.begin();
3790 const auto *IElemInitRef = Data.FirstprivateInits.begin();
3791 for (const Expr *E : Data.FirstprivateVars) {
3792 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3793 Privates.emplace_back(
3794 Args: C.getDeclAlign(D: VD),
3795 Args: PrivateHelpersTy(
3796 E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3797 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IElemInitRef)->getDecl())));
3798 ++I;
3799 ++IElemInitRef;
3800 }
3801 I = Data.LastprivateCopies.begin();
3802 for (const Expr *E : Data.LastprivateVars) {
3803 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3804 Privates.emplace_back(
3805 Args: C.getDeclAlign(D: VD),
3806 Args: PrivateHelpersTy(E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3807 /*PrivateElemInit=*/nullptr));
3808 ++I;
3809 }
3810 for (const VarDecl *VD : Data.PrivateLocals) {
3811 if (isAllocatableDecl(VD))
3812 Privates.emplace_back(Args: CGM.getPointerAlign(), Args: PrivateHelpersTy(VD));
3813 else
3814 Privates.emplace_back(Args: C.getDeclAlign(D: VD), Args: PrivateHelpersTy(VD));
3815 }
3816 llvm::stable_sort(Range&: Privates,
3817 C: [](const PrivateDataTy &L, const PrivateDataTy &R) {
3818 return L.first > R.first;
3819 });
3820 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3821 // Build type kmp_routine_entry_t (if not built yet).
3822 emitKmpRoutineEntryT(KmpInt32Ty);
3823 // Build type kmp_task_t (if not built yet).
3824 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())) {
3825 if (SavedKmpTaskloopTQTy.isNull()) {
3826 SavedKmpTaskloopTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
3827 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
3828 }
3829 KmpTaskTQTy = SavedKmpTaskloopTQTy;
3830 } else {
3831 assert((D.getDirectiveKind() == OMPD_task ||
3832 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
3833 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
3834 "Expected taskloop, task or target directive");
3835 if (SavedKmpTaskTQTy.isNull()) {
3836 SavedKmpTaskTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
3837 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
3838 }
3839 KmpTaskTQTy = SavedKmpTaskTQTy;
3840 }
3841 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3842 // Build particular struct kmp_task_t for the given task.
3843 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3844 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
3845 CanQualType KmpTaskTWithPrivatesQTy =
3846 C.getCanonicalTagType(TD: KmpTaskTWithPrivatesQTyRD);
3847 QualType KmpTaskTWithPrivatesPtrQTy =
3848 C.getPointerType(T: KmpTaskTWithPrivatesQTy);
3849 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(AddrSpace: 0);
3850 llvm::Value *KmpTaskTWithPrivatesTySize =
3851 CGF.getTypeSize(Ty: KmpTaskTWithPrivatesQTy);
3852 QualType SharedsPtrTy = C.getPointerType(T: SharedsTy);
3853
3854 // Emit initial values for private copies (if any).
3855 llvm::Value *TaskPrivatesMap = nullptr;
3856 llvm::Type *TaskPrivatesMapTy =
3857 std::next(x: TaskFunction->arg_begin(), n: 3)->getType();
3858 if (!Privates.empty()) {
3859 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3860 TaskPrivatesMap =
3861 emitTaskPrivateMappingFunction(CGM, Loc, Data, PrivatesQTy: FI->getType(), Privates);
3862 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3863 V: TaskPrivatesMap, DestTy: TaskPrivatesMapTy);
3864 } else {
3865 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3866 T: cast<llvm::PointerType>(Val: TaskPrivatesMapTy));
3867 }
3868 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
3869 // kmp_task_t *tt);
3870 llvm::Function *TaskEntry = emitProxyTaskFunction(
3871 CGM, Loc, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3872 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3873 TaskPrivatesMap);
3874
3875 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
3876 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
3877 // kmp_routine_entry_t *task_entry);
3878 // Task flags. Format is taken from
3879 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
3880 // description of kmp_tasking_flags struct.
3881 enum {
3882 TiedFlag = 0x1,
3883 FinalFlag = 0x2,
3884 DestructorsFlag = 0x8,
3885 PriorityFlag = 0x20,
3886 DetachableFlag = 0x40,
3887 FreeAgentFlag = 0x80,
3888 TransparentFlag = 0x100,
3889 };
3890 unsigned Flags = Data.Tied ? TiedFlag : 0;
3891 bool NeedsCleanup = false;
3892 if (!Privates.empty()) {
3893 NeedsCleanup =
3894 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
3895 if (NeedsCleanup)
3896 Flags = Flags | DestructorsFlag;
3897 }
3898 if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {
3899 OpenMPThreadsetKind Kind = Clause->getThreadsetKind();
3900 if (Kind == OMPC_THREADSET_omp_pool)
3901 Flags = Flags | FreeAgentFlag;
3902 }
3903 if (D.getSingleClause<OMPTransparentClause>())
3904 Flags |= TransparentFlag;
3905
3906 if (Data.Priority.getInt())
3907 Flags = Flags | PriorityFlag;
3908 if (D.hasClausesOfKind<OMPDetachClause>())
3909 Flags = Flags | DetachableFlag;
3910 llvm::Value *TaskFlags =
3911 Data.Final.getPointer()
3912 ? CGF.Builder.CreateSelect(C: Data.Final.getPointer(),
3913 True: CGF.Builder.getInt32(C: FinalFlag),
3914 False: CGF.Builder.getInt32(/*C=*/0))
3915 : CGF.Builder.getInt32(C: Data.Final.getInt() ? FinalFlag : 0);
3916 TaskFlags = CGF.Builder.CreateOr(LHS: TaskFlags, RHS: CGF.Builder.getInt32(C: Flags));
3917 llvm::Value *SharedsSize = CGM.getSize(numChars: C.getTypeSizeInChars(T: SharedsTy));
3918 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
3919 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3920 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3921 V: TaskEntry, DestTy: KmpRoutineEntryPtrTy)};
3922 llvm::Value *NewTask;
3923 if (D.hasClausesOfKind<OMPNowaitClause>()) {
3924 // Check if we have any device clause associated with the directive.
3925 const Expr *Device = nullptr;
3926 if (auto *C = D.getSingleClause<OMPDeviceClause>())
3927 Device = C->getDevice();
3928 // Emit device ID if any otherwise use default value.
3929 llvm::Value *DeviceID;
3930 if (Device)
3931 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
3932 DestTy: CGF.Int64Ty, /*isSigned=*/true);
3933 else
3934 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
3935 AllocArgs.push_back(Elt: DeviceID);
3936 NewTask = CGF.EmitRuntimeCall(
3937 callee: OMPBuilder.getOrCreateRuntimeFunction(
3938 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_target_task_alloc),
3939 args: AllocArgs);
3940 } else {
3941 NewTask =
3942 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
3943 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_alloc),
3944 args: AllocArgs);
3945 }
3946 // Emit detach clause initialization.
3947 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
3948 // task_descriptor);
3949 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
3950 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
3951 LValue EvtLVal = CGF.EmitLValue(E: Evt);
3952
3953 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
3954 // int gtid, kmp_task_t *task);
3955 llvm::Value *Loc = emitUpdateLocation(CGF, Loc: DC->getBeginLoc());
3956 llvm::Value *Tid = getThreadID(CGF, Loc: DC->getBeginLoc());
3957 Tid = CGF.Builder.CreateIntCast(V: Tid, DestTy: CGF.IntTy, /*isSigned=*/false);
3958 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
3959 callee: OMPBuilder.getOrCreateRuntimeFunction(
3960 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_allow_completion_event),
3961 args: {Loc, Tid, NewTask});
3962 EvtVal = CGF.EmitScalarConversion(Src: EvtVal, SrcTy: C.VoidPtrTy, DstTy: Evt->getType(),
3963 Loc: Evt->getExprLoc());
3964 CGF.EmitStoreOfScalar(value: EvtVal, lvalue: EvtLVal);
3965 }
3966 // Process affinity clauses.
3967 if (D.hasClausesOfKind<OMPAffinityClause>()) {
3968 // Process list of affinity data.
3969 ASTContext &C = CGM.getContext();
3970 Address AffinitiesArray = Address::invalid();
3971 // Calculate number of elements to form the array of affinity data.
3972 llvm::Value *NumOfElements = nullptr;
3973 unsigned NumAffinities = 0;
3974 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
3975 if (const Expr *Modifier = C->getModifier()) {
3976 const auto *IE = cast<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts());
3977 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3978 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
3979 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
3980 NumOfElements =
3981 NumOfElements ? CGF.Builder.CreateNUWMul(LHS: NumOfElements, RHS: Sz) : Sz;
3982 }
3983 } else {
3984 NumAffinities += C->varlist_size();
3985 }
3986 }
3987 getKmpAffinityType(C&: CGM.getContext(), KmpTaskAffinityInfoTy);
3988 // Fields ids in kmp_task_affinity_info record.
3989 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3990
3991 QualType KmpTaskAffinityInfoArrayTy;
3992 if (NumOfElements) {
3993 NumOfElements = CGF.Builder.CreateNUWAdd(
3994 LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumAffinities), RHS: NumOfElements);
3995 auto *OVE = new (C) OpaqueValueExpr(
3996 Loc,
3997 C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.getSizeType()), /*Signed=*/0),
3998 VK_PRValue);
3999 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4000 RValue::get(V: NumOfElements));
4001 KmpTaskAffinityInfoArrayTy = C.getVariableArrayType(
4002 EltTy: KmpTaskAffinityInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4003 /*IndexTypeQuals=*/0);
4004 // Properly emit variable-sized array.
4005 auto *PD = ImplicitParamDecl::Create(C, T: KmpTaskAffinityInfoArrayTy,
4006 ParamKind: ImplicitParamKind::Other);
4007 CGF.EmitVarDecl(D: *PD);
4008 AffinitiesArray = CGF.GetAddrOfLocalVar(VD: PD);
4009 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4010 /*isSigned=*/false);
4011 } else {
4012 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4013 EltTy: KmpTaskAffinityInfoTy,
4014 ArySize: llvm::APInt(C.getTypeSize(T: C.getSizeType()), NumAffinities), SizeExpr: nullptr,
4015 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4016 AffinitiesArray = CGF.CreateMemTempWithoutCast(T: KmpTaskAffinityInfoArrayTy,
4017 Name: ".affs.arr.addr");
4018 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(Addr: AffinitiesArray, Index: 0);
4019 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumAffinities,
4020 /*isSigned=*/IsSigned: false);
4021 }
4022
4023 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4024 // Fill array by elements without iterators.
4025 unsigned Pos = 0;
4026 bool HasIterator = false;
4027 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4028 if (C->getModifier()) {
4029 HasIterator = true;
4030 continue;
4031 }
4032 for (const Expr *E : C->varlist()) {
4033 llvm::Value *Addr;
4034 llvm::Value *Size;
4035 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4036 LValue Base =
4037 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateConstGEP(Addr: AffinitiesArray, Index: Pos),
4038 T: KmpTaskAffinityInfoTy);
4039 // affs[i].base_addr = &<Affinities[i].second>;
4040 LValue BaseAddrLVal = CGF.EmitLValueForField(
4041 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4042 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4043 lvalue: BaseAddrLVal);
4044 // affs[i].len = sizeof(<Affinities[i].second>);
4045 LValue LenLVal = CGF.EmitLValueForField(
4046 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4047 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4048 ++Pos;
4049 }
4050 }
4051 LValue PosLVal;
4052 if (HasIterator) {
4053 PosLVal = CGF.MakeAddrLValue(
4054 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "affs.counter.addr"),
4055 T: C.getSizeType());
4056 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4057 }
4058 // Process elements with iterators.
4059 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4060 const Expr *Modifier = C->getModifier();
4061 if (!Modifier)
4062 continue;
4063 OMPIteratorGeneratorScope IteratorScope(
4064 CGF, cast_or_null<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts()));
4065 for (const Expr *E : C->varlist()) {
4066 llvm::Value *Addr;
4067 llvm::Value *Size;
4068 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4069 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4070 LValue Base =
4071 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateGEP(CGF, Addr: AffinitiesArray, Index: Idx),
4072 T: KmpTaskAffinityInfoTy);
4073 // affs[i].base_addr = &<Affinities[i].second>;
4074 LValue BaseAddrLVal = CGF.EmitLValueForField(
4075 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4076 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4077 lvalue: BaseAddrLVal);
4078 // affs[i].len = sizeof(<Affinities[i].second>);
4079 LValue LenLVal = CGF.EmitLValueForField(
4080 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4081 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4082 Idx = CGF.Builder.CreateNUWAdd(
4083 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4084 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4085 }
4086 }
4087 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4088 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4089 // naffins, kmp_task_affinity_info_t *affin_list);
4090 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4091 llvm::Value *GTid = getThreadID(CGF, Loc);
4092 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4093 V: AffinitiesArray.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy);
4094 // FIXME: Emit the function and ignore its result for now unless the
4095 // runtime function is properly implemented.
4096 (void)CGF.EmitRuntimeCall(
4097 callee: OMPBuilder.getOrCreateRuntimeFunction(
4098 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_reg_task_with_affinity),
4099 args: {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4100 }
4101 llvm::Value *NewTaskNewTaskTTy =
4102 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4103 V: NewTask, DestTy: KmpTaskTWithPrivatesPtrTy);
4104 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(V: NewTaskNewTaskTTy,
4105 T: KmpTaskTWithPrivatesQTy);
4106 LValue TDBase =
4107 CGF.EmitLValueForField(Base, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
4108 // Fill the data in the resulting kmp_task_t record.
4109 // Copy shareds if there are any.
4110 Address KmpTaskSharedsPtr = Address::invalid();
4111 if (!SharedsTy->castAsRecordDecl()->field_empty()) {
4112 KmpTaskSharedsPtr = Address(
4113 CGF.EmitLoadOfScalar(
4114 lvalue: CGF.EmitLValueForField(
4115 Base: TDBase,
4116 Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds)),
4117 Loc),
4118 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
4119 LValue Dest = CGF.MakeAddrLValue(Addr: KmpTaskSharedsPtr, T: SharedsTy);
4120 LValue Src = CGF.MakeAddrLValue(Addr: Shareds, T: SharedsTy);
4121 CGF.EmitAggregateCopy(Dest, Src, EltTy: SharedsTy, MayOverlap: AggValueSlot::DoesNotOverlap);
4122 }
4123 // Emit initial values for private copies (if any).
4124 TaskResultTy Result;
4125 if (!Privates.empty()) {
4126 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase: Base, KmpTaskTWithPrivatesQTyRD,
4127 SharedsTy, SharedsPtrTy, Data, Privates,
4128 /*ForDup=*/false);
4129 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) &&
4130 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4131 Result.TaskDupFn = emitTaskDupFunction(
4132 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4133 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4134 /*WithLastIter=*/!Data.LastprivateVars.empty());
4135 }
4136 }
4137 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4138 enum { Priority = 0, Destructors = 1 };
4139 // Provide pointer to function with destructors for privates.
4140 auto FI = std::next(x: KmpTaskTQTyRD->field_begin(), n: Data1);
4141 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
4142 assert(KmpCmplrdataUD->isUnion());
4143 if (NeedsCleanup) {
4144 llvm::Value *DestructorFn = emitDestructorsFunction(
4145 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4146 KmpTaskTWithPrivatesQTy);
4147 LValue Data1LV = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
4148 LValue DestructorsLV = CGF.EmitLValueForField(
4149 Base: Data1LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Destructors));
4150 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4151 V: DestructorFn, DestTy: KmpRoutineEntryPtrTy),
4152 lvalue: DestructorsLV);
4153 }
4154 // Set priority.
4155 if (Data.Priority.getInt()) {
4156 LValue Data2LV = CGF.EmitLValueForField(
4157 Base: TDBase, Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: Data2));
4158 LValue PriorityLV = CGF.EmitLValueForField(
4159 Base: Data2LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Priority));
4160 CGF.EmitStoreOfScalar(value: Data.Priority.getPointer(), lvalue: PriorityLV);
4161 }
4162 Result.NewTask = NewTask;
4163 Result.TaskEntry = TaskEntry;
4164 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4165 Result.TDBase = TDBase;
4166 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4167 return Result;
4168}
4169
4170/// Translates internal dependency kind into the runtime kind.
4171static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4172 RTLDependenceKindTy DepKind;
4173 switch (K) {
4174 case OMPC_DEPEND_in:
4175 DepKind = RTLDependenceKindTy::DepIn;
4176 break;
4177 // Out and InOut dependencies must use the same code.
4178 case OMPC_DEPEND_out:
4179 case OMPC_DEPEND_inout:
4180 DepKind = RTLDependenceKindTy::DepInOut;
4181 break;
4182 case OMPC_DEPEND_mutexinoutset:
4183 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4184 break;
4185 case OMPC_DEPEND_inoutset:
4186 DepKind = RTLDependenceKindTy::DepInOutSet;
4187 break;
4188 case OMPC_DEPEND_outallmemory:
4189 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4190 break;
4191 case OMPC_DEPEND_source:
4192 case OMPC_DEPEND_sink:
4193 case OMPC_DEPEND_depobj:
4194 case OMPC_DEPEND_inoutallmemory:
4195 case OMPC_DEPEND_unknown:
4196 llvm_unreachable("Unknown task dependence type");
4197 }
4198 return DepKind;
4199}
4200
4201/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4202static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4203 QualType &FlagsTy) {
4204 FlagsTy = C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.BoolTy), /*Signed=*/false);
4205 if (KmpDependInfoTy.isNull()) {
4206 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord(Name: "kmp_depend_info");
4207 KmpDependInfoRD->startDefinition();
4208 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getIntPtrType());
4209 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getSizeType());
4210 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: FlagsTy);
4211 KmpDependInfoRD->completeDefinition();
4212 KmpDependInfoTy = C.getCanonicalTagType(TD: KmpDependInfoRD);
4213 }
4214}
4215
4216std::pair<llvm::Value *, LValue>
4217CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4218 SourceLocation Loc) {
4219 ASTContext &C = CGM.getContext();
4220 QualType FlagsTy;
4221 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4222 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4223 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4224 LValue Base = CGF.EmitLoadOfPointerLValue(
4225 Ptr: DepobjLVal.getAddress().withElementType(
4226 ElemTy: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy)),
4227 PtrTy: KmpDependInfoPtrTy->castAs<PointerType>());
4228 Address DepObjAddr = CGF.Builder.CreateGEP(
4229 CGF, Addr: Base.getAddress(),
4230 Index: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4231 LValue NumDepsBase = CGF.MakeAddrLValue(
4232 Addr: DepObjAddr, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(), TBAAInfo: Base.getTBAAInfo());
4233 // NumDeps = deps[i].base_addr;
4234 LValue BaseAddrLVal = CGF.EmitLValueForField(
4235 Base: NumDepsBase,
4236 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4237 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4238 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(lvalue: BaseAddrLVal, Loc);
4239 return std::make_pair(x&: NumDeps, y&: Base);
4240}
4241
4242static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4243 llvm::PointerUnion<unsigned *, LValue *> Pos,
4244 const OMPTaskDataTy::DependData &Data,
4245 Address DependenciesArray) {
4246 CodeGenModule &CGM = CGF.CGM;
4247 ASTContext &C = CGM.getContext();
4248 QualType FlagsTy;
4249 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4250 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4251 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4252
4253 OMPIteratorGeneratorScope IteratorScope(
4254 CGF, cast_or_null<OMPIteratorExpr>(
4255 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4256 : nullptr));
4257 for (const Expr *E : Data.DepExprs) {
4258 llvm::Value *Addr;
4259 llvm::Value *Size;
4260
4261 // The expression will be a nullptr in the 'omp_all_memory' case.
4262 if (E) {
4263 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4264 Addr = CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy);
4265 } else {
4266 Addr = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4267 Size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
4268 }
4269 LValue Base;
4270 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4271 Base = CGF.MakeAddrLValue(
4272 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: *P), T: KmpDependInfoTy);
4273 } else {
4274 assert(E && "Expected a non-null expression");
4275 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4276 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4277 Base = CGF.MakeAddrLValue(
4278 Addr: CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Idx), T: KmpDependInfoTy);
4279 }
4280 // deps[i].base_addr = &<Dependencies[i].second>;
4281 LValue BaseAddrLVal = CGF.EmitLValueForField(
4282 Base,
4283 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4284 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4285 CGF.EmitStoreOfScalar(value: Addr, lvalue: BaseAddrLVal);
4286 // deps[i].len = sizeof(<Dependencies[i].second>);
4287 LValue LenLVal = CGF.EmitLValueForField(
4288 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4289 n: static_cast<unsigned int>(RTLDependInfoFields::Len)));
4290 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4291 // deps[i].flags = <Dependencies[i].first>;
4292 RTLDependenceKindTy DepKind = translateDependencyKind(K: Data.DepKind);
4293 LValue FlagsLVal = CGF.EmitLValueForField(
4294 Base,
4295 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4296 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4297 CGF.EmitStoreOfScalar(
4298 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4299 lvalue: FlagsLVal);
4300 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4301 ++(*P);
4302 } else {
4303 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4304 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4305 Idx = CGF.Builder.CreateNUWAdd(LHS: Idx,
4306 RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4307 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4308 }
4309 }
4310}
4311
4312SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes(
4313 CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4314 const OMPTaskDataTy::DependData &Data) {
4315 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4316 "Expected depobj dependency kind.");
4317 SmallVector<llvm::Value *, 4> Sizes;
4318 SmallVector<LValue, 4> SizeLVals;
4319 ASTContext &C = CGF.getContext();
4320 {
4321 OMPIteratorGeneratorScope IteratorScope(
4322 CGF, cast_or_null<OMPIteratorExpr>(
4323 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4324 : nullptr));
4325 for (const Expr *E : Data.DepExprs) {
4326 llvm::Value *NumDeps;
4327 LValue Base;
4328 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4329 std::tie(args&: NumDeps, args&: Base) =
4330 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4331 LValue NumLVal = CGF.MakeAddrLValue(
4332 Addr: CGF.CreateMemTempWithoutCast(T: C.getUIntPtrType(), Name: "depobj.size.addr"),
4333 T: C.getUIntPtrType());
4334 CGF.Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0),
4335 Addr: NumLVal.getAddress());
4336 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(lvalue: NumLVal, Loc: E->getExprLoc());
4337 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: PrevVal, RHS: NumDeps);
4338 CGF.EmitStoreOfScalar(value: Add, lvalue: NumLVal);
4339 SizeLVals.push_back(Elt: NumLVal);
4340 }
4341 }
4342 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4343 llvm::Value *Size =
4344 CGF.EmitLoadOfScalar(lvalue: SizeLVals[I], Loc: Data.DepExprs[I]->getExprLoc());
4345 Sizes.push_back(Elt: Size);
4346 }
4347 return Sizes;
4348}
4349
4350void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
4351 QualType &KmpDependInfoTy,
4352 LValue PosLVal,
4353 const OMPTaskDataTy::DependData &Data,
4354 Address DependenciesArray) {
4355 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4356 "Expected depobj dependency kind.");
4357 llvm::Value *ElSize = CGF.getTypeSize(Ty: KmpDependInfoTy);
4358 {
4359 OMPIteratorGeneratorScope IteratorScope(
4360 CGF, cast_or_null<OMPIteratorExpr>(
4361 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4362 : nullptr));
4363 for (const Expr *E : Data.DepExprs) {
4364 llvm::Value *NumDeps;
4365 LValue Base;
4366 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4367 std::tie(args&: NumDeps, args&: Base) =
4368 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4369
4370 // memcopy dependency data.
4371 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4372 LHS: ElSize,
4373 RHS: CGF.Builder.CreateIntCast(V: NumDeps, DestTy: CGF.SizeTy, /*isSigned=*/false));
4374 llvm::Value *Pos = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4375 Address DepAddr = CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Pos);
4376 CGF.Builder.CreateMemCpy(Dest: DepAddr, Src: Base.getAddress(), Size);
4377
4378 // Increase pos.
4379 // pos += size;
4380 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: Pos, RHS: NumDeps);
4381 CGF.EmitStoreOfScalar(value: Add, lvalue: PosLVal);
4382 }
4383 }
4384}
4385
4386std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4387 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4388 SourceLocation Loc) {
4389 if (llvm::all_of(Range&: Dependencies, P: [](const OMPTaskDataTy::DependData &D) {
4390 return D.DepExprs.empty();
4391 }))
4392 return std::make_pair(x: nullptr, y: Address::invalid());
4393 // Process list of dependencies.
4394 ASTContext &C = CGM.getContext();
4395 Address DependenciesArray = Address::invalid();
4396 llvm::Value *NumOfElements = nullptr;
4397 unsigned NumDependencies = std::accumulate(
4398 first: Dependencies.begin(), last: Dependencies.end(), init: 0,
4399 binary_op: [](unsigned V, const OMPTaskDataTy::DependData &D) {
4400 return D.DepKind == OMPC_DEPEND_depobj
4401 ? V
4402 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4403 });
4404 QualType FlagsTy;
4405 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4406 bool HasDepobjDeps = false;
4407 bool HasRegularWithIterators = false;
4408 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4409 llvm::Value *NumOfRegularWithIterators =
4410 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4411 // Calculate number of depobj dependencies and regular deps with the
4412 // iterators.
4413 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4414 if (D.DepKind == OMPC_DEPEND_depobj) {
4415 SmallVector<llvm::Value *, 4> Sizes =
4416 emitDepobjElementsSizes(CGF, KmpDependInfoTy, Data: D);
4417 for (llvm::Value *Size : Sizes) {
4418 NumOfDepobjElements =
4419 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: Size);
4420 }
4421 HasDepobjDeps = true;
4422 continue;
4423 }
4424 // Include number of iterations, if any.
4425
4426 if (const auto *IE = cast_or_null<OMPIteratorExpr>(Val: D.IteratorExpr)) {
4427 llvm::Value *ClauseIteratorSpace =
4428 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 1);
4429 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4430 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4431 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4432 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(LHS: Sz, RHS: ClauseIteratorSpace);
4433 }
4434 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4435 LHS: ClauseIteratorSpace,
4436 RHS: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: D.DepExprs.size()));
4437 NumOfRegularWithIterators =
4438 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumClauseDeps);
4439 HasRegularWithIterators = true;
4440 continue;
4441 }
4442 }
4443
4444 QualType KmpDependInfoArrayTy;
4445 if (HasDepobjDeps || HasRegularWithIterators) {
4446 NumOfElements = llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: NumDependencies,
4447 /*isSigned=*/IsSigned: false);
4448 if (HasDepobjDeps) {
4449 NumOfElements =
4450 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: NumOfElements);
4451 }
4452 if (HasRegularWithIterators) {
4453 NumOfElements =
4454 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumOfElements);
4455 }
4456 auto *OVE = new (C) OpaqueValueExpr(
4457 Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4458 VK_PRValue);
4459 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4460 RValue::get(V: NumOfElements));
4461 KmpDependInfoArrayTy =
4462 C.getVariableArrayType(EltTy: KmpDependInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4463 /*IndexTypeQuals=*/0);
4464 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4465 // Properly emit variable-sized array.
4466 auto *PD = ImplicitParamDecl::Create(C, T: KmpDependInfoArrayTy,
4467 ParamKind: ImplicitParamKind::Other);
4468 CGF.EmitVarDecl(D: *PD);
4469 DependenciesArray = CGF.GetAddrOfLocalVar(VD: PD);
4470 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4471 /*isSigned=*/false);
4472 } else {
4473 KmpDependInfoArrayTy = C.getConstantArrayType(
4474 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies), SizeExpr: nullptr,
4475 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4476 DependenciesArray =
4477 CGF.CreateMemTempWithoutCast(T: KmpDependInfoArrayTy, Name: ".dep.arr.addr");
4478 DependenciesArray = CGF.Builder.CreateConstArrayGEP(Addr: DependenciesArray, Index: 0);
4479 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumDependencies,
4480 /*isSigned=*/IsSigned: false);
4481 }
4482 unsigned Pos = 0;
4483 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4484 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4485 continue;
4486 emitDependData(CGF, KmpDependInfoTy, Pos: &Pos, Data: Dep, DependenciesArray);
4487 }
4488 // Copy regular dependencies with iterators.
4489 LValue PosLVal = CGF.MakeAddrLValue(
4490 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "dep.counter.addr"),
4491 T: C.getSizeType());
4492 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4493 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4494 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4495 continue;
4496 emitDependData(CGF, KmpDependInfoTy, Pos: &PosLVal, Data: Dep, DependenciesArray);
4497 }
4498 // Copy final depobj arrays without iterators.
4499 if (HasDepobjDeps) {
4500 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4501 if (Dep.DepKind != OMPC_DEPEND_depobj)
4502 continue;
4503 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Data: Dep, DependenciesArray);
4504 }
4505 }
4506 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4507 Addr: DependenciesArray, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
4508 return std::make_pair(x&: NumOfElements, y&: DependenciesArray);
4509}
4510
4511Address CGOpenMPRuntime::emitDepobjDependClause(
4512 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4513 SourceLocation Loc) {
4514 if (Dependencies.DepExprs.empty())
4515 return Address::invalid();
4516 // Process list of dependencies.
4517 ASTContext &C = CGM.getContext();
4518 Address DependenciesArray = Address::invalid();
4519 unsigned NumDependencies = Dependencies.DepExprs.size();
4520 QualType FlagsTy;
4521 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4522 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4523
4524 llvm::Value *Size;
4525 // Define type kmp_depend_info[<Dependencies.size()>];
4526 // For depobj reserve one extra element to store the number of elements.
4527 // It is required to handle depobj(x) update(in) construct.
4528 // kmp_depend_info[<Dependencies.size()>] deps;
4529 llvm::Value *NumDepsVal;
4530 CharUnits Align = C.getTypeAlignInChars(T: KmpDependInfoTy);
4531 if (const auto *IE =
4532 cast_or_null<OMPIteratorExpr>(Val: Dependencies.IteratorExpr)) {
4533 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
4534 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4535 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4536 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
4537 NumDepsVal = CGF.Builder.CreateNUWMul(LHS: NumDepsVal, RHS: Sz);
4538 }
4539 Size = CGF.Builder.CreateNUWAdd(LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1),
4540 RHS: NumDepsVal);
4541 CharUnits SizeInBytes =
4542 C.getTypeSizeInChars(T: KmpDependInfoTy).alignTo(Align);
4543 llvm::Value *RecSize = CGM.getSize(numChars: SizeInBytes);
4544 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: RecSize);
4545 NumDepsVal =
4546 CGF.Builder.CreateIntCast(V: NumDepsVal, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4547 } else {
4548 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4549 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4550 SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4551 CharUnits Sz = C.getTypeSizeInChars(T: KmpDependInfoArrayTy);
4552 Size = CGM.getSize(numChars: Sz.alignTo(Align));
4553 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: NumDependencies);
4554 }
4555 // Need to allocate on the dynamic memory.
4556 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4557 // Use default allocator.
4558 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4559 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4560
4561 llvm::Value *Addr =
4562 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4563 M&: CGM.getModule(), FnID: OMPRTL___kmpc_alloc),
4564 args: Args, name: ".dep.arr.addr");
4565 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(T: KmpDependInfoTy);
4566 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4567 V: Addr, DestTy: CGF.Builder.getPtrTy(AddrSpace: 0));
4568 DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
4569 // Write number of elements in the first element of array for depobj.
4570 LValue Base = CGF.MakeAddrLValue(Addr: DependenciesArray, T: KmpDependInfoTy);
4571 // deps[i].base_addr = NumDependencies;
4572 LValue BaseAddrLVal = CGF.EmitLValueForField(
4573 Base,
4574 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4575 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4576 CGF.EmitStoreOfScalar(value: NumDepsVal, lvalue: BaseAddrLVal);
4577 llvm::PointerUnion<unsigned *, LValue *> Pos;
4578 unsigned Idx = 1;
4579 LValue PosLVal;
4580 if (Dependencies.IteratorExpr) {
4581 PosLVal = CGF.MakeAddrLValue(
4582 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "iterator.counter.addr"),
4583 T: C.getSizeType());
4584 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Idx), lvalue: PosLVal,
4585 /*IsInit=*/isInit: true);
4586 Pos = &PosLVal;
4587 } else {
4588 Pos = &Idx;
4589 }
4590 emitDependData(CGF, KmpDependInfoTy, Pos, Data: Dependencies, DependenciesArray);
4591 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4592 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: 1), Ty: CGF.VoidPtrTy,
4593 ElementTy: CGF.Int8Ty);
4594 return DependenciesArray;
4595}
4596
4597void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4598 SourceLocation Loc) {
4599 ASTContext &C = CGM.getContext();
4600 QualType FlagsTy;
4601 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4602 LValue Base = CGF.EmitLoadOfPointerLValue(Ptr: DepobjLVal.getAddress(),
4603 PtrTy: C.VoidPtrTy.castAs<PointerType>());
4604 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4605 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4606 Addr: Base.getAddress(), Ty: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy),
4607 ElementTy: CGF.ConvertTypeForMem(T: KmpDependInfoTy));
4608 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4609 Ty: Addr.getElementType(), Ptr: Addr.emitRawPointer(CGF),
4610 IdxList: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4611 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: DepObjAddr,
4612 DestTy: CGF.VoidPtrTy);
4613 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4614 // Use default allocator.
4615 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4616 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4617
4618 // _kmpc_free(gtid, addr, nullptr);
4619 (void)CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4620 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free),
4621 args: Args);
4622}
4623
4624void CGOpenMPRuntime::emitUpdateDependObjectsClause(
4625 CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind,
4626 SourceLocation Loc) {
4627 ASTContext &C = CGM.getContext();
4628 QualType FlagsTy;
4629 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4630 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4631 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4632 llvm::Value *NumDeps;
4633 LValue Base;
4634 std::tie(args&: NumDeps, args&: Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4635
4636 Address Begin = Base.getAddress();
4637 // Cast from pointer to array type to pointer to single element.
4638 llvm::Value *End = CGF.Builder.CreateGEP(Ty: Begin.getElementType(),
4639 Ptr: Begin.emitRawPointer(CGF), IdxList: NumDeps);
4640 // The basic structure here is a while-do loop.
4641 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.body");
4642 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.done");
4643 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4644 CGF.EmitBlock(BB: BodyBB);
4645 llvm::PHINode *ElementPHI =
4646 CGF.Builder.CreatePHI(Ty: Begin.getType(), NumReservedValues: 2, Name: "omp.elementPast");
4647 ElementPHI->addIncoming(V: Begin.emitRawPointer(CGF), BB: EntryBB);
4648 Begin = Begin.withPointer(NewPointer: ElementPHI, IsKnownNonNull: KnownNonNull);
4649 Base = CGF.MakeAddrLValue(Addr: Begin, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(),
4650 TBAAInfo: Base.getTBAAInfo());
4651 // deps[i].flags = NewDepKind;
4652 RTLDependenceKindTy DepKind = translateDependencyKind(K: NewDepKind);
4653 LValue FlagsLVal = CGF.EmitLValueForField(
4654 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4655 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4656 CGF.EmitStoreOfScalar(
4657 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4658 lvalue: FlagsLVal);
4659
4660 // Shift the address forward by one element.
4661 llvm::Value *ElementNext =
4662 CGF.Builder.CreateConstGEP(Addr: Begin, /*Index=*/1, Name: "omp.elementNext")
4663 .emitRawPointer(CGF);
4664 ElementPHI->addIncoming(V: ElementNext, BB: CGF.Builder.GetInsertBlock());
4665 llvm::Value *IsEmpty =
4666 CGF.Builder.CreateICmpEQ(LHS: ElementNext, RHS: End, Name: "omp.isempty");
4667 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4668 // Done.
4669 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4670}
4671
4672void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4673 const OMPExecutableDirective &D,
4674 llvm::Function *TaskFunction,
4675 QualType SharedsTy, Address Shareds,
4676 const Expr *IfCond,
4677 const OMPTaskDataTy &Data) {
4678 if (!CGF.HaveInsertPoint())
4679 return;
4680
4681 TaskResultTy Result =
4682 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4683 llvm::Value *NewTask = Result.NewTask;
4684 llvm::Function *TaskEntry = Result.TaskEntry;
4685 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4686 LValue TDBase = Result.TDBase;
4687 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
4688 // Process list of dependences.
4689 Address DependenciesArray = Address::invalid();
4690 llvm::Value *NumOfElements;
4691 std::tie(args&: NumOfElements, args&: DependenciesArray) =
4692 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
4693
4694 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4695 // libcall.
4696 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
4697 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
4698 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
4699 // list is not empty
4700 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4701 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4702 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4703 llvm::Value *DepTaskArgs[7];
4704 if (!Data.Dependences.empty()) {
4705 DepTaskArgs[0] = UpLoc;
4706 DepTaskArgs[1] = ThreadID;
4707 DepTaskArgs[2] = NewTask;
4708 DepTaskArgs[3] = NumOfElements;
4709 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4710 DepTaskArgs[5] = CGF.Builder.getInt32(C: 0);
4711 DepTaskArgs[6] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4712 }
4713 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4714 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4715 if (!Data.Tied) {
4716 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
4717 LValue PartIdLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PartIdFI);
4718 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: 0), lvalue: PartIdLVal);
4719 }
4720 if (!Data.Dependences.empty()) {
4721 CGF.EmitRuntimeCall(
4722 callee: OMPBuilder.getOrCreateRuntimeFunction(
4723 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_with_deps),
4724 args: DepTaskArgs);
4725 } else {
4726 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4727 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
4728 args: TaskArgs);
4729 }
4730 // Check if parent region is untied and build return for untied task;
4731 if (auto *Region =
4732 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
4733 Region->emitUntiedSwitch(CGF);
4734 };
4735
4736 llvm::Value *DepWaitTaskArgs[7];
4737 if (!Data.Dependences.empty()) {
4738 DepWaitTaskArgs[0] = UpLoc;
4739 DepWaitTaskArgs[1] = ThreadID;
4740 DepWaitTaskArgs[2] = NumOfElements;
4741 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4742 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
4743 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4744 DepWaitTaskArgs[6] =
4745 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
4746 }
4747 auto &M = CGM.getModule();
4748 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4749 TaskEntry, &Data, &DepWaitTaskArgs,
4750 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4751 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4752 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
4753 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
4754 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
4755 // is specified.
4756 if (!Data.Dependences.empty())
4757 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4758 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
4759 args: DepWaitTaskArgs);
4760 // Call proxy_task_entry(gtid, new_task);
4761 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4762 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4763 Action.Enter(CGF);
4764 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4765 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskEntry,
4766 Args: OutlinedFnArgs);
4767 };
4768
4769 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
4770 // kmp_task_t *new_task);
4771 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
4772 // kmp_task_t *new_task);
4773 RegionCodeGenTy RCG(CodeGen);
4774 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
4775 M, FnID: OMPRTL___kmpc_omp_task_begin_if0),
4776 TaskArgs,
4777 OMPBuilder.getOrCreateRuntimeFunction(
4778 M, FnID: OMPRTL___kmpc_omp_task_complete_if0),
4779 TaskArgs);
4780 RCG.setAction(Action);
4781 RCG(CGF);
4782 };
4783
4784 if (IfCond) {
4785 emitIfClause(CGF, Cond: IfCond, ThenGen: ThenCodeGen, ElseGen: ElseCodeGen);
4786 } else {
4787 RegionCodeGenTy ThenRCG(ThenCodeGen);
4788 ThenRCG(CGF);
4789 }
4790}
4791
4792void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4793 const OMPLoopDirective &D,
4794 llvm::Function *TaskFunction,
4795 QualType SharedsTy, Address Shareds,
4796 const Expr *IfCond,
4797 const OMPTaskDataTy &Data) {
4798 if (!CGF.HaveInsertPoint())
4799 return;
4800 TaskResultTy Result =
4801 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4802 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4803 // libcall.
4804 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
4805 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
4806 // sched, kmp_uint64 grainsize, void *task_dup);
4807 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4808 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4809 llvm::Value *IfVal;
4810 if (IfCond) {
4811 IfVal = CGF.Builder.CreateIntCast(V: CGF.EvaluateExprAsBool(E: IfCond), DestTy: CGF.IntTy,
4812 /*isSigned=*/true);
4813 } else {
4814 IfVal = llvm::ConstantInt::getSigned(Ty: CGF.IntTy, /*V=*/1);
4815 }
4816
4817 LValue LBLVal = CGF.EmitLValueForField(
4818 Base: Result.TDBase,
4819 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound));
4820 const auto *LBVar =
4821 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getLowerBoundVariable())->getDecl());
4822 CGF.EmitAnyExprToMem(E: LBVar->getInit(), Location: LBLVal.getAddress(), Quals: LBLVal.getQuals(),
4823 /*IsInitializer=*/true);
4824 LValue UBLVal = CGF.EmitLValueForField(
4825 Base: Result.TDBase,
4826 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound));
4827 const auto *UBVar =
4828 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getUpperBoundVariable())->getDecl());
4829 CGF.EmitAnyExprToMem(E: UBVar->getInit(), Location: UBLVal.getAddress(), Quals: UBLVal.getQuals(),
4830 /*IsInitializer=*/true);
4831 LValue StLVal = CGF.EmitLValueForField(
4832 Base: Result.TDBase,
4833 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride));
4834 const auto *StVar =
4835 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getStrideVariable())->getDecl());
4836 CGF.EmitAnyExprToMem(E: StVar->getInit(), Location: StLVal.getAddress(), Quals: StLVal.getQuals(),
4837 /*IsInitializer=*/true);
4838 // Store reductions address.
4839 LValue RedLVal = CGF.EmitLValueForField(
4840 Base: Result.TDBase,
4841 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions));
4842 if (Data.Reductions) {
4843 CGF.EmitStoreOfScalar(value: Data.Reductions, lvalue: RedLVal);
4844 } else {
4845 CGF.EmitNullInitialization(DestPtr: RedLVal.getAddress(),
4846 Ty: CGF.getContext().VoidPtrTy);
4847 }
4848 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4849 llvm::SmallVector<llvm::Value *, 12> TaskArgs{
4850 UpLoc,
4851 ThreadID,
4852 Result.NewTask,
4853 IfVal,
4854 LBLVal.getPointer(CGF),
4855 UBLVal.getPointer(CGF),
4856 CGF.EmitLoadOfScalar(lvalue: StLVal, Loc),
4857 llvm::ConstantInt::getSigned(
4858 Ty: CGF.IntTy, V: 1), // Always 1 because taskgroup emitted by the compiler
4859 llvm::ConstantInt::getSigned(
4860 Ty: CGF.IntTy, V: Data.Schedule.getPointer()
4861 ? Data.Schedule.getInt() ? NumTasks : Grainsize
4862 : NoSchedule),
4863 Data.Schedule.getPointer()
4864 ? CGF.Builder.CreateIntCast(V: Data.Schedule.getPointer(), DestTy: CGF.Int64Ty,
4865 /*isSigned=*/false)
4866 : llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/0)};
4867 if (Data.HasModifier)
4868 TaskArgs.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 1));
4869
4870 TaskArgs.push_back(Elt: Result.TaskDupFn
4871 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4872 V: Result.TaskDupFn, DestTy: CGF.VoidPtrTy)
4873 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy));
4874 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4875 M&: CGM.getModule(), FnID: Data.HasModifier
4876 ? OMPRTL___kmpc_taskloop_5
4877 : OMPRTL___kmpc_taskloop),
4878 args: TaskArgs);
4879}
4880
4881/// Emit reduction operation for each element of array (required for
4882/// array sections) LHS op = RHS.
4883/// \param Type Type of array.
4884/// \param LHSVar Variable on the left side of the reduction operation
4885/// (references element of array in original variable).
4886/// \param RHSVar Variable on the right side of the reduction operation
4887/// (references element of array in original variable).
4888/// \param RedOpGen Generator of reduction operation with use of LHSVar and
4889/// RHSVar.
4890static void EmitOMPAggregateReduction(
4891 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
4892 const VarDecl *RHSVar,
4893 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
4894 const Expr *, const Expr *)> &RedOpGen,
4895 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
4896 const Expr *UpExpr = nullptr) {
4897 // Perform element-by-element initialization.
4898 QualType ElementTy;
4899 Address LHSAddr = CGF.GetAddrOfLocalVar(VD: LHSVar);
4900 Address RHSAddr = CGF.GetAddrOfLocalVar(VD: RHSVar);
4901
4902 // Drill down to the base element type on both arrays.
4903 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
4904 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: LHSAddr);
4905
4906 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
4907 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
4908 // Cast from pointer to array type to pointer to single element.
4909 llvm::Value *LHSEnd =
4910 CGF.Builder.CreateGEP(Ty: LHSAddr.getElementType(), Ptr: LHSBegin, IdxList: NumElements);
4911 // The basic structure here is a while-do loop.
4912 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arraycpy.body");
4913 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arraycpy.done");
4914 llvm::Value *IsEmpty =
4915 CGF.Builder.CreateICmpEQ(LHS: LHSBegin, RHS: LHSEnd, Name: "omp.arraycpy.isempty");
4916 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4917
4918 // Enter the loop body, making that address the current address.
4919 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4920 CGF.EmitBlock(BB: BodyBB);
4921
4922 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
4923
4924 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
4925 Ty: RHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
4926 RHSElementPHI->addIncoming(V: RHSBegin, BB: EntryBB);
4927 Address RHSElementCurrent(
4928 RHSElementPHI, RHSAddr.getElementType(),
4929 RHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
4930
4931 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
4932 Ty: LHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
4933 LHSElementPHI->addIncoming(V: LHSBegin, BB: EntryBB);
4934 Address LHSElementCurrent(
4935 LHSElementPHI, LHSAddr.getElementType(),
4936 LHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
4937
4938 // Emit copy.
4939 CodeGenFunction::OMPPrivateScope Scope(CGF);
4940 Scope.addPrivate(LocalVD: LHSVar, Addr: LHSElementCurrent);
4941 Scope.addPrivate(LocalVD: RHSVar, Addr: RHSElementCurrent);
4942 Scope.Privatize();
4943 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4944 Scope.ForceCleanup();
4945
4946 // Shift the address forward by one element.
4947 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
4948 Ty: LHSAddr.getElementType(), Ptr: LHSElementPHI, /*Idx0=*/1,
4949 Name: "omp.arraycpy.dest.element");
4950 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
4951 Ty: RHSAddr.getElementType(), Ptr: RHSElementPHI, /*Idx0=*/1,
4952 Name: "omp.arraycpy.src.element");
4953 // Check whether we've reached the end.
4954 llvm::Value *Done =
4955 CGF.Builder.CreateICmpEQ(LHS: LHSElementNext, RHS: LHSEnd, Name: "omp.arraycpy.done");
4956 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
4957 LHSElementPHI->addIncoming(V: LHSElementNext, BB: CGF.Builder.GetInsertBlock());
4958 RHSElementPHI->addIncoming(V: RHSElementNext, BB: CGF.Builder.GetInsertBlock());
4959
4960 // Done.
4961 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4962}
4963
4964/// Emit reduction combiner. If the combiner is a simple expression emit it as
4965/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
4966/// UDR combiner function.
4967static void emitReductionCombiner(CodeGenFunction &CGF,
4968 const Expr *ReductionOp) {
4969 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
4970 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
4971 if (const auto *DRE =
4972 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
4973 if (const auto *DRD =
4974 dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl())) {
4975 std::pair<llvm::Function *, llvm::Function *> Reduction =
4976 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
4977 RValue Func = RValue::get(V: Reduction.first);
4978 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
4979 CGF.EmitIgnoredExpr(E: ReductionOp);
4980 return;
4981 }
4982 CGF.EmitIgnoredExpr(E: ReductionOp);
4983}
4984
4985llvm::Function *CGOpenMPRuntime::emitReductionFunction(
4986 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
4987 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
4988 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
4989 ASTContext &C = CGM.getContext();
4990
4991 // void reduction_func(void *LHSArg, void *RHSArg);
4992 auto *LHSArg =
4993 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
4994 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4995 auto *RHSArg =
4996 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
4997 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
4998 FunctionArgList Args{LHSArg, RHSArg};
4999 const auto &CGFI =
5000 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5001 std::string Name = getReductionFuncName(Name: ReducerName);
5002 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
5003 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
5004 M: &CGM.getModule());
5005 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
5006 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5007 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5008 Fn->setDoesNotRecurse();
5009 CodeGenFunction CGF(CGM);
5010 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
5011
5012 // Dst = (void*[n])(LHSArg);
5013 // Src = (void*[n])(RHSArg);
5014 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5015 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
5016 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5017 ArgsElemType, CGF.getPointerAlign());
5018 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5019 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
5020 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5021 ArgsElemType, CGF.getPointerAlign());
5022
5023 // ...
5024 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5025 // ...
5026 CodeGenFunction::OMPPrivateScope Scope(CGF);
5027 const auto *IPriv = Privates.begin();
5028 unsigned Idx = 0;
5029 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5030 const auto *RHSVar =
5031 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSExprs[I])->getDecl());
5032 Scope.addPrivate(LocalVD: RHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: RHS, Index: Idx, Var: RHSVar));
5033 const auto *LHSVar =
5034 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSExprs[I])->getDecl());
5035 Scope.addPrivate(LocalVD: LHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: LHS, Index: Idx, Var: LHSVar));
5036 QualType PrivTy = (*IPriv)->getType();
5037 if (PrivTy->isVariablyModifiedType()) {
5038 // Get array size and emit VLA type.
5039 ++Idx;
5040 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: LHS, Index: Idx);
5041 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Elem);
5042 const VariableArrayType *VLA =
5043 CGF.getContext().getAsVariableArrayType(T: PrivTy);
5044 const auto *OVE = cast<OpaqueValueExpr>(Val: VLA->getSizeExpr());
5045 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5046 CGF, OVE, RValue::get(V: CGF.Builder.CreatePtrToInt(V: Ptr, DestTy: CGF.SizeTy)));
5047 CGF.EmitVariablyModifiedType(Ty: PrivTy);
5048 }
5049 }
5050 Scope.Privatize();
5051 IPriv = Privates.begin();
5052 const auto *ILHS = LHSExprs.begin();
5053 const auto *IRHS = RHSExprs.begin();
5054 for (const Expr *E : ReductionOps) {
5055 if ((*IPriv)->getType()->isArrayType()) {
5056 // Emit reduction for array section.
5057 const auto *LHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5058 const auto *RHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5059 EmitOMPAggregateReduction(
5060 CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5061 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5062 emitReductionCombiner(CGF, ReductionOp: E);
5063 });
5064 } else {
5065 // Emit reduction for array subscript or single variable.
5066 emitReductionCombiner(CGF, ReductionOp: E);
5067 }
5068 ++IPriv;
5069 ++ILHS;
5070 ++IRHS;
5071 }
5072 Scope.ForceCleanup();
5073 CGF.FinishFunction();
5074 return Fn;
5075}
5076
5077void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5078 const Expr *ReductionOp,
5079 const Expr *PrivateRef,
5080 const DeclRefExpr *LHS,
5081 const DeclRefExpr *RHS) {
5082 if (PrivateRef->getType()->isArrayType()) {
5083 // Emit reduction for array section.
5084 const auto *LHSVar = cast<VarDecl>(Val: LHS->getDecl());
5085 const auto *RHSVar = cast<VarDecl>(Val: RHS->getDecl());
5086 EmitOMPAggregateReduction(
5087 CGF, Type: PrivateRef->getType(), LHSVar, RHSVar,
5088 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5089 emitReductionCombiner(CGF, ReductionOp);
5090 });
5091 } else {
5092 // Emit reduction for array subscript or single variable.
5093 emitReductionCombiner(CGF, ReductionOp);
5094 }
5095}
5096
5097static std::string generateUniqueName(CodeGenModule &CGM,
5098 llvm::StringRef Prefix, const Expr *Ref);
5099
5100void CGOpenMPRuntime::emitPrivateReduction(
5101 CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,
5102 const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {
5103
5104 // Create a shared global variable (__shared_reduction_var) to accumulate the
5105 // final result.
5106 //
5107 // Call __kmpc_barrier to synchronize threads before initialization.
5108 //
5109 // The master thread (thread_id == 0) initializes __shared_reduction_var
5110 // with the identity value or initializer.
5111 //
5112 // Call __kmpc_barrier to synchronize before combining.
5113 // For each i:
5114 // - Thread enters critical section.
5115 // - Reads its private value from LHSExprs[i].
5116 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
5117 // Privates[i]).
5118 // - Exits critical section.
5119 //
5120 // Call __kmpc_barrier after combining.
5121 //
5122 // Each thread copies __shared_reduction_var[i] back to RHSExprs[i].
5123 //
5124 // Final __kmpc_barrier to synchronize after broadcasting
5125 QualType PrivateType = Privates->getType();
5126 llvm::Type *LLVMType = CGF.ConvertTypeForMem(T: PrivateType);
5127
5128 const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOp: ReductionOps);
5129 std::string ReductionVarNameStr;
5130 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates->IgnoreParenCasts()))
5131 ReductionVarNameStr =
5132 generateUniqueName(CGM, Prefix: DRE->getDecl()->getNameAsString(), Ref: Privates);
5133 else
5134 ReductionVarNameStr = "unnamed_priv_var";
5135
5136 // Create an internal shared variable
5137 std::string SharedName =
5138 CGM.getOpenMPRuntime().getName(Parts: {"internal_pivate_", ReductionVarNameStr});
5139 llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(
5140 Ty: LLVMType, Name: ".omp.reduction." + SharedName);
5141
5142 SharedVar->setAlignment(
5143 llvm::MaybeAlign(CGF.getContext().getTypeAlign(T: PrivateType) / 8));
5144
5145 Address SharedResult =
5146 CGF.MakeNaturalAlignRawAddrLValue(V: SharedVar, T: PrivateType).getAddress();
5147
5148 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5149 llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5150 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
5151
5152 llvm::BasicBlock *InitBB = CGF.createBasicBlock(name: "init");
5153 llvm::BasicBlock *InitEndBB = CGF.createBasicBlock(name: "init.end");
5154
5155 llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(
5156 LHS: ThreadId, RHS: llvm::ConstantInt::get(Ty: ThreadId->getType(), V: 0));
5157 CGF.Builder.CreateCondBr(Cond: IsWorker, True: InitBB, False: InitEndBB);
5158
5159 CGF.EmitBlock(BB: InitBB);
5160
5161 auto EmitSharedInit = [&]() {
5162 if (UDR) { // Check if it's a User-Defined Reduction
5163 if (const Expr *UDRInitExpr = UDR->getInitializer()) {
5164 std::pair<llvm::Function *, llvm::Function *> FnPair =
5165 getUserDefinedReduction(D: UDR);
5166 llvm::Function *InitializerFn = FnPair.second;
5167 if (InitializerFn) {
5168 if (const auto *CE =
5169 dyn_cast<CallExpr>(Val: UDRInitExpr->IgnoreParenImpCasts())) {
5170 const auto *OutDRE = cast<DeclRefExpr>(
5171 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5172 ->getSubExpr());
5173 const VarDecl *OutVD = cast<VarDecl>(Val: OutDRE->getDecl());
5174
5175 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5176 LocalScope.addPrivate(LocalVD: OutVD, Addr: SharedResult);
5177
5178 (void)LocalScope.Privatize();
5179 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(
5180 Val: CE->getCallee()->IgnoreParenImpCasts())) {
5181 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5182 CGF, OVE, RValue::get(V: InitializerFn));
5183 CGF.EmitIgnoredExpr(E: CE);
5184 } else {
5185 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5186 Quals: PrivateType.getQualifiers(),
5187 /*IsInitializer=*/true);
5188 }
5189 } else {
5190 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5191 Quals: PrivateType.getQualifiers(),
5192 /*IsInitializer=*/true);
5193 }
5194 } else {
5195 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5196 Quals: PrivateType.getQualifiers(),
5197 /*IsInitializer=*/true);
5198 }
5199 } else {
5200 // EmitNullInitialization handles default construction for C++ classes
5201 // and zeroing for scalars, which is a reasonable default.
5202 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5203 }
5204 return; // UDR initialization handled
5205 }
5206 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates)) {
5207 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
5208 if (const Expr *InitExpr = VD->getInit()) {
5209 CGF.EmitAnyExprToMem(E: InitExpr, Location: SharedResult,
5210 Quals: PrivateType.getQualifiers(), IsInitializer: true);
5211 return;
5212 }
5213 }
5214 }
5215 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5216 };
5217 EmitSharedInit();
5218 CGF.Builder.CreateBr(Dest: InitEndBB);
5219 CGF.EmitBlock(BB: InitEndBB);
5220
5221 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5222 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5223 args: BarrierArgs);
5224
5225 const Expr *ReductionOp = ReductionOps;
5226 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
5227 LValue SharedLV = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5228 LValue LHSLV = CGF.EmitLValue(E: Privates);
5229
5230 auto EmitCriticalReduction = [&](auto ReductionGen) {
5231 std::string CriticalName = getName(Parts: {"reduction_critical"});
5232 emitCriticalRegion(CGF, CriticalName, CriticalOpGen: ReductionGen, Loc);
5233 };
5234
5235 if (CurrentUDR) {
5236 // Handle user-defined reduction.
5237 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5238 Action.Enter(CGF);
5239 std::pair<llvm::Function *, llvm::Function *> FnPair =
5240 getUserDefinedReduction(D: CurrentUDR);
5241 if (FnPair.first) {
5242 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp)) {
5243 const auto *OutDRE = cast<DeclRefExpr>(
5244 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5245 ->getSubExpr());
5246 const auto *InDRE = cast<DeclRefExpr>(
5247 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 1)->IgnoreParenImpCasts())
5248 ->getSubExpr());
5249 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5250 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: OutDRE->getDecl()),
5251 Addr: SharedLV.getAddress());
5252 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: InDRE->getDecl()),
5253 Addr: LHSLV.getAddress());
5254 (void)LocalScope.Privatize();
5255 emitReductionCombiner(CGF, ReductionOp);
5256 }
5257 }
5258 };
5259 EmitCriticalReduction(ReductionGen);
5260 } else {
5261 // Handle built-in reduction operations.
5262#ifndef NDEBUG
5263 const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();
5264 if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5265 ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
5266
5267 const Expr *AssignRHS = nullptr;
5268 if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5269 if (BinOp->getOpcode() == BO_Assign)
5270 AssignRHS = BinOp->getRHS();
5271 } else if (const auto *OpCall =
5272 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5273 if (OpCall->getOperator() == OO_Equal)
5274 AssignRHS = OpCall->getArg(1);
5275 }
5276
5277 assert(AssignRHS &&
5278 "Private Variable Reduction : Invalid ReductionOp expression");
5279#endif
5280
5281 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5282 Action.Enter(CGF);
5283 const auto *OmpOutDRE =
5284 dyn_cast<DeclRefExpr>(Val: LHSExprs->IgnoreParenImpCasts());
5285 const auto *OmpInDRE =
5286 dyn_cast<DeclRefExpr>(Val: RHSExprs->IgnoreParenImpCasts());
5287 assert(
5288 OmpOutDRE && OmpInDRE &&
5289 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5290 const VarDecl *OmpOutVD = cast<VarDecl>(Val: OmpOutDRE->getDecl());
5291 const VarDecl *OmpInVD = cast<VarDecl>(Val: OmpInDRE->getDecl());
5292 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5293 LocalScope.addPrivate(LocalVD: OmpOutVD, Addr: SharedLV.getAddress());
5294 LocalScope.addPrivate(LocalVD: OmpInVD, Addr: LHSLV.getAddress());
5295 (void)LocalScope.Privatize();
5296 // Emit the actual reduction operation
5297 CGF.EmitIgnoredExpr(E: ReductionOp);
5298 };
5299 EmitCriticalReduction(ReductionGen);
5300 }
5301
5302 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5303 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5304 args: BarrierArgs);
5305
5306 // Broadcast final result
5307 bool IsAggregate = PrivateType->isAggregateType();
5308 LValue SharedLV1 = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5309 llvm::Value *FinalResultVal = nullptr;
5310 Address FinalResultAddr = Address::invalid();
5311
5312 if (IsAggregate)
5313 FinalResultAddr = SharedResult;
5314 else
5315 FinalResultVal = CGF.EmitLoadOfScalar(lvalue: SharedLV1, Loc);
5316
5317 LValue TargetLHSLV = CGF.EmitLValue(E: RHSExprs);
5318 if (IsAggregate) {
5319 CGF.EmitAggregateCopy(Dest: TargetLHSLV,
5320 Src: CGF.MakeAddrLValue(Addr: FinalResultAddr, T: PrivateType),
5321 EltTy: PrivateType, MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: false);
5322 } else {
5323 CGF.EmitStoreOfScalar(value: FinalResultVal, lvalue: TargetLHSLV);
5324 }
5325 // Final synchronization barrier
5326 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5327 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5328 args: BarrierArgs);
5329
5330 // Combiner with original list item
5331 auto OriginalListCombiner = [&](CodeGenFunction &CGF,
5332 PrePostActionTy &Action) {
5333 Action.Enter(CGF);
5334 emitSingleReductionCombiner(CGF, ReductionOp: ReductionOps, PrivateRef: Privates,
5335 LHS: cast<DeclRefExpr>(Val: LHSExprs),
5336 RHS: cast<DeclRefExpr>(Val: RHSExprs));
5337 };
5338 EmitCriticalReduction(OriginalListCombiner);
5339}
5340
5341void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5342 ArrayRef<const Expr *> OrgPrivates,
5343 ArrayRef<const Expr *> OrgLHSExprs,
5344 ArrayRef<const Expr *> OrgRHSExprs,
5345 ArrayRef<const Expr *> OrgReductionOps,
5346 ReductionOptionsTy Options) {
5347 if (!CGF.HaveInsertPoint())
5348 return;
5349
5350 bool WithNowait = Options.WithNowait;
5351 bool SimpleReduction = Options.SimpleReduction;
5352
5353 // Next code should be emitted for reduction:
5354 //
5355 // static kmp_critical_name lock = { 0 };
5356 //
5357 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5358 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5359 // ...
5360 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5361 // *(Type<n>-1*)rhs[<n>-1]);
5362 // }
5363 //
5364 // ...
5365 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5366 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5367 // RedList, reduce_func, &<lock>)) {
5368 // case 1:
5369 // ...
5370 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5371 // ...
5372 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5373 // break;
5374 // case 2:
5375 // ...
5376 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5377 // ...
5378 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5379 // break;
5380 // default:;
5381 // }
5382 //
5383 // if SimpleReduction is true, only the next code is generated:
5384 // ...
5385 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5386 // ...
5387
5388 ASTContext &C = CGM.getContext();
5389
5390 if (SimpleReduction) {
5391 CodeGenFunction::RunCleanupsScope Scope(CGF);
5392 const auto *IPriv = OrgPrivates.begin();
5393 const auto *ILHS = OrgLHSExprs.begin();
5394 const auto *IRHS = OrgRHSExprs.begin();
5395 for (const Expr *E : OrgReductionOps) {
5396 emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5397 RHS: cast<DeclRefExpr>(Val: *IRHS));
5398 ++IPriv;
5399 ++ILHS;
5400 ++IRHS;
5401 }
5402 return;
5403 }
5404
5405 // Filter out shared reduction variables based on IsPrivateVarReduction flag.
5406 // Only keep entries where the corresponding variable is not private.
5407 SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
5408 FilteredRHSExprs, FilteredReductionOps;
5409 for (unsigned I : llvm::seq<unsigned>(
5410 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5411 if (!Options.IsPrivateVarReduction[I]) {
5412 FilteredPrivates.emplace_back(Args: OrgPrivates[I]);
5413 FilteredLHSExprs.emplace_back(Args: OrgLHSExprs[I]);
5414 FilteredRHSExprs.emplace_back(Args: OrgRHSExprs[I]);
5415 FilteredReductionOps.emplace_back(Args: OrgReductionOps[I]);
5416 }
5417 }
5418 // Wrap filtered vectors in ArrayRef for downstream shared reduction
5419 // processing.
5420 ArrayRef<const Expr *> Privates = FilteredPrivates;
5421 ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
5422 ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
5423 ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
5424
5425 // 1. Build a list of reduction variables.
5426 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5427 auto Size = RHSExprs.size();
5428 for (const Expr *E : Privates) {
5429 if (E->getType()->isVariablyModifiedType())
5430 // Reserve place for array size.
5431 ++Size;
5432 }
5433 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5434 QualType ReductionArrayTy = C.getConstantArrayType(
5435 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5436 /*IndexTypeQuals=*/0);
5437 RawAddress ReductionList =
5438 CGF.CreateMemTemp(T: ReductionArrayTy, Name: ".omp.reduction.red_list");
5439 const auto *IPriv = Privates.begin();
5440 unsigned Idx = 0;
5441 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5442 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5443 CGF.Builder.CreateStore(
5444 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5445 V: CGF.EmitLValue(E: RHSExprs[I]).getPointer(CGF), DestTy: CGF.VoidPtrTy),
5446 Addr: Elem);
5447 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5448 // Store array size.
5449 ++Idx;
5450 Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5451 llvm::Value *Size = CGF.Builder.CreateIntCast(
5452 V: CGF.getVLASize(
5453 vla: CGF.getContext().getAsVariableArrayType(T: (*IPriv)->getType()))
5454 .NumElts,
5455 DestTy: CGF.SizeTy, /*isSigned=*/false);
5456 CGF.Builder.CreateStore(Val: CGF.Builder.CreateIntToPtr(V: Size, DestTy: CGF.VoidPtrTy),
5457 Addr: Elem);
5458 }
5459 }
5460
5461 // 2. Emit reduce_func().
5462 llvm::Function *ReductionFn = emitReductionFunction(
5463 ReducerName: CGF.CurFn->getName(), Loc, ArgsElemType: CGF.ConvertTypeForMem(T: ReductionArrayTy),
5464 Privates, LHSExprs, RHSExprs, ReductionOps);
5465
5466 // 3. Create static kmp_critical_name lock = { 0 };
5467 std::string Name = getName(Parts: {"reduction"});
5468 llvm::Value *Lock = getCriticalRegionLock(CriticalName: Name);
5469
5470 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5471 // RedList, reduce_func, &<lock>);
5472 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5473 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5474 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(Ty: ReductionArrayTy);
5475 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5476 V: ReductionList.getPointer(), DestTy: CGF.VoidPtrTy);
5477 llvm::Value *Args[] = {
5478 IdentTLoc, // ident_t *<loc>
5479 ThreadId, // i32 <gtid>
5480 CGF.Builder.getInt32(C: RHSExprs.size()), // i32 <n>
5481 ReductionArrayTySize, // size_type sizeof(RedList)
5482 RL, // void *RedList
5483 ReductionFn, // void (*) (void *, void *) <reduce_func>
5484 Lock // kmp_critical_name *&<lock>
5485 };
5486 llvm::Value *Res = CGF.EmitRuntimeCall(
5487 callee: OMPBuilder.getOrCreateRuntimeFunction(
5488 M&: CGM.getModule(),
5489 FnID: WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5490 args: Args);
5491
5492 // 5. Build switch(res)
5493 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(name: ".omp.reduction.default");
5494 llvm::SwitchInst *SwInst =
5495 CGF.Builder.CreateSwitch(V: Res, Dest: DefaultBB, /*NumCases=*/2);
5496
5497 // 6. Build case 1:
5498 // ...
5499 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5500 // ...
5501 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5502 // break;
5503 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(name: ".omp.reduction.case1");
5504 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 1), Dest: Case1BB);
5505 CGF.EmitBlock(BB: Case1BB);
5506
5507 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5508 llvm::Value *EndArgs[] = {
5509 IdentTLoc, // ident_t *<loc>
5510 ThreadId, // i32 <gtid>
5511 Lock // kmp_critical_name *&<lock>
5512 };
5513 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5514 CodeGenFunction &CGF, PrePostActionTy &Action) {
5515 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5516 const auto *IPriv = Privates.begin();
5517 const auto *ILHS = LHSExprs.begin();
5518 const auto *IRHS = RHSExprs.begin();
5519 for (const Expr *E : ReductionOps) {
5520 RT.emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5521 RHS: cast<DeclRefExpr>(Val: *IRHS));
5522 ++IPriv;
5523 ++ILHS;
5524 ++IRHS;
5525 }
5526 };
5527 RegionCodeGenTy RCG(CodeGen);
5528 CommonActionTy Action(
5529 nullptr, {},
5530 OMPBuilder.getOrCreateRuntimeFunction(
5531 M&: CGM.getModule(), FnID: WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5532 : OMPRTL___kmpc_end_reduce),
5533 EndArgs);
5534 RCG.setAction(Action);
5535 RCG(CGF);
5536
5537 CGF.EmitBranch(Block: DefaultBB);
5538
5539 // 7. Build case 2:
5540 // ...
5541 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5542 // ...
5543 // break;
5544 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(name: ".omp.reduction.case2");
5545 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 2), Dest: Case2BB);
5546 CGF.EmitBlock(BB: Case2BB);
5547
5548 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5549 CodeGenFunction &CGF, PrePostActionTy &Action) {
5550 const auto *ILHS = LHSExprs.begin();
5551 const auto *IRHS = RHSExprs.begin();
5552 const auto *IPriv = Privates.begin();
5553 for (const Expr *E : ReductionOps) {
5554 const Expr *XExpr = nullptr;
5555 const Expr *EExpr = nullptr;
5556 const Expr *UpExpr = nullptr;
5557 BinaryOperatorKind BO = BO_Comma;
5558 if (const auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
5559 if (BO->getOpcode() == BO_Assign) {
5560 XExpr = BO->getLHS();
5561 UpExpr = BO->getRHS();
5562 }
5563 }
5564 // Try to emit update expression as a simple atomic.
5565 const Expr *RHSExpr = UpExpr;
5566 if (RHSExpr) {
5567 // Analyze RHS part of the whole expression.
5568 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5569 Val: RHSExpr->IgnoreParenImpCasts())) {
5570 // If this is a conditional operator, analyze its condition for
5571 // min/max reduction operator.
5572 RHSExpr = ACO->getCond();
5573 }
5574 if (const auto *BORHS =
5575 dyn_cast<BinaryOperator>(Val: RHSExpr->IgnoreParenImpCasts())) {
5576 EExpr = BORHS->getRHS();
5577 BO = BORHS->getOpcode();
5578 }
5579 }
5580 if (XExpr) {
5581 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5582 auto &&AtomicRedGen = [BO, VD,
5583 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5584 const Expr *EExpr, const Expr *UpExpr) {
5585 LValue X = CGF.EmitLValue(E: XExpr);
5586 RValue E;
5587 if (EExpr)
5588 E = CGF.EmitAnyExpr(E: EExpr);
5589 CGF.EmitOMPAtomicSimpleUpdateExpr(
5590 X, E, BO, /*IsXLHSInRHSPart=*/true,
5591 AO: llvm::AtomicOrdering::Monotonic, Loc,
5592 CommonGen: [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5593 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5594 Address LHSTemp = CGF.CreateMemTemp(T: VD->getType());
5595 CGF.emitOMPSimpleStore(
5596 LVal: CGF.MakeAddrLValue(Addr: LHSTemp, T: VD->getType()), RVal: XRValue,
5597 RValTy: VD->getType().getNonReferenceType(), Loc);
5598 PrivateScope.addPrivate(LocalVD: VD, Addr: LHSTemp);
5599 (void)PrivateScope.Privatize();
5600 return CGF.EmitAnyExpr(E: UpExpr);
5601 });
5602 };
5603 if ((*IPriv)->getType()->isArrayType()) {
5604 // Emit atomic reduction for array section.
5605 const auto *RHSVar =
5606 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5607 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar: VD, RHSVar,
5608 RedOpGen: AtomicRedGen, XExpr, EExpr, UpExpr);
5609 } else {
5610 // Emit atomic reduction for array subscript or single variable.
5611 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5612 }
5613 } else {
5614 // Emit as a critical region.
5615 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5616 const Expr *, const Expr *) {
5617 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5618 std::string Name = RT.getName(Parts: {"atomic_reduction"});
5619 RT.emitCriticalRegion(
5620 CGF, CriticalName: Name,
5621 CriticalOpGen: [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5622 Action.Enter(CGF);
5623 emitReductionCombiner(CGF, ReductionOp: E);
5624 },
5625 Loc);
5626 };
5627 if ((*IPriv)->getType()->isArrayType()) {
5628 const auto *LHSVar =
5629 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5630 const auto *RHSVar =
5631 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5632 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5633 RedOpGen: CritRedGen);
5634 } else {
5635 CritRedGen(CGF, nullptr, nullptr, nullptr);
5636 }
5637 }
5638 ++ILHS;
5639 ++IRHS;
5640 ++IPriv;
5641 }
5642 };
5643 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5644 if (!WithNowait) {
5645 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5646 llvm::Value *EndArgs[] = {
5647 IdentTLoc, // ident_t *<loc>
5648 ThreadId, // i32 <gtid>
5649 Lock // kmp_critical_name *&<lock>
5650 };
5651 CommonActionTy Action(nullptr, {},
5652 OMPBuilder.getOrCreateRuntimeFunction(
5653 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_reduce),
5654 EndArgs);
5655 AtomicRCG.setAction(Action);
5656 AtomicRCG(CGF);
5657 } else {
5658 AtomicRCG(CGF);
5659 }
5660
5661 CGF.EmitBranch(Block: DefaultBB);
5662 CGF.EmitBlock(BB: DefaultBB, /*IsFinished=*/true);
5663 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5664 "PrivateVarReduction: Privates size mismatch");
5665 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5666 "PrivateVarReduction: ReductionOps size mismatch");
5667 for (unsigned I : llvm::seq<unsigned>(
5668 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5669 if (Options.IsPrivateVarReduction[I])
5670 emitPrivateReduction(CGF, Loc, Privates: OrgPrivates[I], LHSExprs: OrgLHSExprs[I],
5671 RHSExprs: OrgRHSExprs[I], ReductionOps: OrgReductionOps[I]);
5672 }
5673}
5674
5675/// Generates unique name for artificial threadprivate variables.
5676/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5677static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5678 const Expr *Ref) {
5679 SmallString<256> Buffer;
5680 llvm::raw_svector_ostream Out(Buffer);
5681 const clang::DeclRefExpr *DE;
5682 const VarDecl *D = ::getBaseDecl(Ref, DE);
5683 if (!D)
5684 D = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Ref)->getDecl());
5685 D = D->getCanonicalDecl();
5686 std::string Name = CGM.getOpenMPRuntime().getName(
5687 Parts: {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(GD: D)});
5688 Out << Prefix << Name << "_"
5689 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5690 return std::string(Out.str());
5691}
5692
5693/// Emits reduction initializer function:
5694/// \code
5695/// void @.red_init(void* %arg, void* %orig) {
5696/// %0 = bitcast void* %arg to <type>*
5697/// store <type> <init>, <type>* %0
5698/// ret void
5699/// }
5700/// \endcode
5701static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5702 SourceLocation Loc,
5703 ReductionCodeGen &RCG, unsigned N) {
5704 ASTContext &C = CGM.getContext();
5705 QualType VoidPtrTy = C.VoidPtrTy;
5706 VoidPtrTy.addRestrict();
5707 FunctionArgList Args;
5708 auto *Param =
5709 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5710 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5711 auto *ParamOrig =
5712 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5713 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5714 Args.emplace_back(Args&: Param);
5715 Args.emplace_back(Args&: ParamOrig);
5716 const auto &FnInfo =
5717 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5718 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5719 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_init", ""});
5720 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5721 N: Name, M: &CGM.getModule());
5722 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5723 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5724 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5725 Fn->setDoesNotRecurse();
5726 CodeGenFunction CGF(CGM);
5727 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5728 QualType PrivateType = RCG.getPrivateType(N);
5729 Address PrivateAddr = CGF.EmitLoadOfPointer(
5730 Ptr: CGF.GetAddrOfLocalVar(VD: Param).withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5731 PtrTy: C.getPointerType(T: PrivateType)->castAs<PointerType>());
5732 llvm::Value *Size = nullptr;
5733 // If the size of the reduction item is non-constant, load it from global
5734 // threadprivate variable.
5735 if (RCG.getSizes(N).second) {
5736 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5737 CGF, VarType: CGM.getContext().getSizeType(),
5738 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5739 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5740 Ty: CGM.getContext().getSizeType(), Loc);
5741 }
5742 RCG.emitAggregateType(CGF, N, Size);
5743 Address OrigAddr = Address::invalid();
5744 // If initializer uses initializer from declare reduction construct, emit a
5745 // pointer to the address of the original reduction item (reuired by reduction
5746 // initializer)
5747 if (RCG.usesReductionInitializer(N)) {
5748 Address SharedAddr = CGF.GetAddrOfLocalVar(VD: ParamOrig);
5749 OrigAddr = CGF.EmitLoadOfPointer(
5750 Ptr: SharedAddr,
5751 PtrTy: CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5752 }
5753 // Emit the initializer:
5754 // %0 = bitcast void* %arg to <type>*
5755 // store <type> <init>, <type>* %0
5756 RCG.emitInitialization(CGF, N, PrivateAddr, SharedAddr: OrigAddr,
5757 DefaultInit: [](CodeGenFunction &) { return false; });
5758 CGF.FinishFunction();
5759 return Fn;
5760}
5761
5762/// Emits reduction combiner function:
5763/// \code
5764/// void @.red_comb(void* %arg0, void* %arg1) {
5765/// %lhs = bitcast void* %arg0 to <type>*
5766/// %rhs = bitcast void* %arg1 to <type>*
5767/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5768/// store <type> %2, <type>* %lhs
5769/// ret void
5770/// }
5771/// \endcode
5772static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5773 SourceLocation Loc,
5774 ReductionCodeGen &RCG, unsigned N,
5775 const Expr *ReductionOp,
5776 const Expr *LHS, const Expr *RHS,
5777 const Expr *PrivateRef) {
5778 ASTContext &C = CGM.getContext();
5779 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHS)->getDecl());
5780 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHS)->getDecl());
5781 FunctionArgList Args;
5782 auto *ParamInOut =
5783 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5784 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5785 auto *ParamIn =
5786 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5787 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5788 Args.emplace_back(Args&: ParamInOut);
5789 Args.emplace_back(Args&: ParamIn);
5790 const auto &FnInfo =
5791 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5792 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5793 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_comb", ""});
5794 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5795 N: Name, M: &CGM.getModule());
5796 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5797 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5798 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5799 Fn->setDoesNotRecurse();
5800 CodeGenFunction CGF(CGM);
5801 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5802 llvm::Value *Size = nullptr;
5803 // If the size of the reduction item is non-constant, load it from global
5804 // threadprivate variable.
5805 if (RCG.getSizes(N).second) {
5806 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5807 CGF, VarType: CGM.getContext().getSizeType(),
5808 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5809 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5810 Ty: CGM.getContext().getSizeType(), Loc);
5811 }
5812 RCG.emitAggregateType(CGF, N, Size);
5813 // Remap lhs and rhs variables to the addresses of the function arguments.
5814 // %lhs = bitcast void* %arg0 to <type>*
5815 // %rhs = bitcast void* %arg1 to <type>*
5816 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5817 PrivateScope.addPrivate(
5818 LocalVD: LHSVD,
5819 // Pull out the pointer to the variable.
5820 Addr: CGF.EmitLoadOfPointer(
5821 Ptr: CGF.GetAddrOfLocalVar(VD: ParamInOut)
5822 .withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5823 PtrTy: C.getPointerType(T: LHSVD->getType())->castAs<PointerType>()));
5824 PrivateScope.addPrivate(
5825 LocalVD: RHSVD,
5826 // Pull out the pointer to the variable.
5827 Addr: CGF.EmitLoadOfPointer(
5828 Ptr: CGF.GetAddrOfLocalVar(VD: ParamIn).withElementType(
5829 ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5830 PtrTy: C.getPointerType(T: RHSVD->getType())->castAs<PointerType>()));
5831 PrivateScope.Privatize();
5832 // Emit the combiner body:
5833 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
5834 // store <type> %2, <type>* %lhs
5835 CGM.getOpenMPRuntime().emitSingleReductionCombiner(
5836 CGF, ReductionOp, PrivateRef, LHS: cast<DeclRefExpr>(Val: LHS),
5837 RHS: cast<DeclRefExpr>(Val: RHS));
5838 CGF.FinishFunction();
5839 return Fn;
5840}
5841
5842/// Emits reduction finalizer function:
5843/// \code
5844/// void @.red_fini(void* %arg) {
5845/// %0 = bitcast void* %arg to <type>*
5846/// <destroy>(<type>* %0)
5847/// ret void
5848/// }
5849/// \endcode
5850static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
5851 SourceLocation Loc,
5852 ReductionCodeGen &RCG, unsigned N) {
5853 if (!RCG.needCleanups(N))
5854 return nullptr;
5855 ASTContext &C = CGM.getContext();
5856 FunctionArgList Args;
5857 auto *Param =
5858 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5859 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5860 Args.emplace_back(Args&: Param);
5861 const auto &FnInfo =
5862 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5863 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5864 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_fini", ""});
5865 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5866 N: Name, M: &CGM.getModule());
5867 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5868 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5869 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5870 Fn->setDoesNotRecurse();
5871 CodeGenFunction CGF(CGM);
5872 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5873 Address PrivateAddr = CGF.EmitLoadOfPointer(
5874 Ptr: CGF.GetAddrOfLocalVar(VD: Param), PtrTy: C.VoidPtrTy.castAs<PointerType>());
5875 llvm::Value *Size = nullptr;
5876 // If the size of the reduction item is non-constant, load it from global
5877 // threadprivate variable.
5878 if (RCG.getSizes(N).second) {
5879 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5880 CGF, VarType: CGM.getContext().getSizeType(),
5881 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5882 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5883 Ty: CGM.getContext().getSizeType(), Loc);
5884 }
5885 RCG.emitAggregateType(CGF, N, Size);
5886 // Emit the finalizer body:
5887 // <destroy>(<type>* %0)
5888 RCG.emitCleanups(CGF, N, PrivateAddr);
5889 CGF.FinishFunction(EndLoc: Loc);
5890 return Fn;
5891}
5892
5893llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
5894 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
5895 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
5896 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
5897 return nullptr;
5898
5899 // Build typedef struct:
5900 // kmp_taskred_input {
5901 // void *reduce_shar; // shared reduction item
5902 // void *reduce_orig; // original reduction item used for initialization
5903 // size_t reduce_size; // size of data item
5904 // void *reduce_init; // data initialization routine
5905 // void *reduce_fini; // data finalization routine
5906 // void *reduce_comb; // data combiner routine
5907 // kmp_task_red_flags_t flags; // flags for additional info from compiler
5908 // } kmp_taskred_input_t;
5909 ASTContext &C = CGM.getContext();
5910 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_taskred_input_t");
5911 RD->startDefinition();
5912 const FieldDecl *SharedFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5913 const FieldDecl *OrigFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5914 const FieldDecl *SizeFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.getSizeType());
5915 const FieldDecl *InitFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5916 const FieldDecl *FiniFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5917 const FieldDecl *CombFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5918 const FieldDecl *FlagsFD = addFieldToRecordDecl(
5919 C, DC: RD, FieldTy: C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
5920 RD->completeDefinition();
5921 CanQualType RDType = C.getCanonicalTagType(TD: RD);
5922 unsigned Size = Data.ReductionVars.size();
5923 llvm::APInt ArraySize(/*numBits=*/64, Size);
5924 QualType ArrayRDType =
5925 C.getConstantArrayType(EltTy: RDType, ArySize: ArraySize, SizeExpr: nullptr,
5926 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
5927 // kmp_task_red_input_t .rd_input.[Size];
5928 RawAddress TaskRedInput = CGF.CreateMemTemp(T: ArrayRDType, Name: ".rd_input.");
5929 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
5930 Data.ReductionCopies, Data.ReductionOps);
5931 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5932 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
5933 llvm::Value *Idxs[] = {llvm::ConstantInt::get(Ty: CGM.SizeTy, /*V=*/0),
5934 llvm::ConstantInt::get(Ty: CGM.SizeTy, V: Cnt)};
5935 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
5936 ElemTy: TaskRedInput.getElementType(), Ptr: TaskRedInput.getPointer(), IdxList: Idxs,
5937 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
5938 Name: ".rd_input.gep.");
5939 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(V: GEP, T: RDType);
5940 // ElemLVal.reduce_shar = &Shareds[Cnt];
5941 LValue SharedLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SharedFD);
5942 RCG.emitSharedOrigLValue(CGF, N: Cnt);
5943 llvm::Value *Shared = RCG.getSharedLValue(N: Cnt).getPointer(CGF);
5944 CGF.EmitStoreOfScalar(value: Shared, lvalue: SharedLVal);
5945 // ElemLVal.reduce_orig = &Origs[Cnt];
5946 LValue OrigLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: OrigFD);
5947 llvm::Value *Orig = RCG.getOrigLValue(N: Cnt).getPointer(CGF);
5948 CGF.EmitStoreOfScalar(value: Orig, lvalue: OrigLVal);
5949 RCG.emitAggregateType(CGF, N: Cnt);
5950 llvm::Value *SizeValInChars;
5951 llvm::Value *SizeVal;
5952 std::tie(args&: SizeValInChars, args&: SizeVal) = RCG.getSizes(N: Cnt);
5953 // We use delayed creation/initialization for VLAs and array sections. It is
5954 // required because runtime does not provide the way to pass the sizes of
5955 // VLAs/array sections to initializer/combiner/finalizer functions. Instead
5956 // threadprivate global variables are used to store these values and use
5957 // them in the functions.
5958 bool DelayedCreation = !!SizeVal;
5959 SizeValInChars = CGF.Builder.CreateIntCast(V: SizeValInChars, DestTy: CGM.SizeTy,
5960 /*isSigned=*/false);
5961 LValue SizeLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SizeFD);
5962 CGF.EmitStoreOfScalar(value: SizeValInChars, lvalue: SizeLVal);
5963 // ElemLVal.reduce_init = init;
5964 LValue InitLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: InitFD);
5965 llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, N: Cnt);
5966 CGF.EmitStoreOfScalar(value: InitAddr, lvalue: InitLVal);
5967 // ElemLVal.reduce_fini = fini;
5968 LValue FiniLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FiniFD);
5969 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, N: Cnt);
5970 llvm::Value *FiniAddr =
5971 Fini ? Fini : llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
5972 CGF.EmitStoreOfScalar(value: FiniAddr, lvalue: FiniLVal);
5973 // ElemLVal.reduce_comb = comb;
5974 LValue CombLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: CombFD);
5975 llvm::Value *CombAddr = emitReduceCombFunction(
5976 CGM, Loc, RCG, N: Cnt, ReductionOp: Data.ReductionOps[Cnt], LHS: LHSExprs[Cnt],
5977 RHS: RHSExprs[Cnt], PrivateRef: Data.ReductionCopies[Cnt]);
5978 CGF.EmitStoreOfScalar(value: CombAddr, lvalue: CombLVal);
5979 // ElemLVal.flags = 0;
5980 LValue FlagsLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FlagsFD);
5981 if (DelayedCreation) {
5982 CGF.EmitStoreOfScalar(
5983 value: llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/1, /*isSigned=*/IsSigned: true),
5984 lvalue: FlagsLVal);
5985 } else
5986 CGF.EmitNullInitialization(DestPtr: FlagsLVal.getAddress(), Ty: FlagsLVal.getType());
5987 }
5988 if (Data.IsReductionWithTaskMod) {
5989 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
5990 // is_ws, int num, void *data);
5991 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
5992 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
5993 DestTy: CGM.IntTy, /*isSigned=*/true);
5994 llvm::Value *Args[] = {
5995 IdentTLoc, GTid,
5996 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Data.IsWorksharingReduction ? 1 : 0,
5997 /*isSigned=*/IsSigned: true),
5998 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
5999 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6000 V: TaskRedInput.getPointer(), DestTy: CGM.VoidPtrTy)};
6001 return CGF.EmitRuntimeCall(
6002 callee: OMPBuilder.getOrCreateRuntimeFunction(
6003 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_modifier_init),
6004 args: Args);
6005 }
6006 // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data);
6007 llvm::Value *Args[] = {
6008 CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc), DestTy: CGM.IntTy,
6009 /*isSigned=*/true),
6010 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6011 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: TaskRedInput.getPointer(),
6012 DestTy: CGM.VoidPtrTy)};
6013 return CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6014 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_init),
6015 args: Args);
6016}
6017
6018void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
6019 SourceLocation Loc,
6020 bool IsWorksharingReduction) {
6021 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6022 // is_ws, int num, void *data);
6023 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6024 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6025 DestTy: CGM.IntTy, /*isSigned=*/true);
6026 llvm::Value *Args[] = {IdentTLoc, GTid,
6027 llvm::ConstantInt::get(Ty: CGM.IntTy,
6028 V: IsWorksharingReduction ? 1 : 0,
6029 /*isSigned=*/IsSigned: true)};
6030 (void)CGF.EmitRuntimeCall(
6031 callee: OMPBuilder.getOrCreateRuntimeFunction(
6032 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_modifier_fini),
6033 args: Args);
6034}
6035
6036void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6037 SourceLocation Loc,
6038 ReductionCodeGen &RCG,
6039 unsigned N) {
6040 auto Sizes = RCG.getSizes(N);
6041 // Emit threadprivate global variable if the type is non-constant
6042 // (Sizes.second = nullptr).
6043 if (Sizes.second) {
6044 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(V: Sizes.second, DestTy: CGM.SizeTy,
6045 /*isSigned=*/false);
6046 Address SizeAddr = getAddrOfArtificialThreadPrivate(
6047 CGF, VarType: CGM.getContext().getSizeType(),
6048 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6049 CGF.Builder.CreateStore(Val: SizeVal, Addr: SizeAddr, /*IsVolatile=*/false);
6050 }
6051}
6052
6053Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6054 SourceLocation Loc,
6055 llvm::Value *ReductionsPtr,
6056 LValue SharedLVal) {
6057 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6058 // *d);
6059 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6060 DestTy: CGM.IntTy,
6061 /*isSigned=*/true),
6062 ReductionsPtr,
6063 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6064 V: SharedLVal.getPointer(CGF), DestTy: CGM.VoidPtrTy)};
6065 return Address(
6066 CGF.EmitRuntimeCall(
6067 callee: OMPBuilder.getOrCreateRuntimeFunction(
6068 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_get_th_data),
6069 args: Args),
6070 CGF.Int8Ty, SharedLVal.getAlignment());
6071}
6072
6073void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
6074 const OMPTaskDataTy &Data) {
6075 if (!CGF.HaveInsertPoint())
6076 return;
6077
6078 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
6079 // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder.
6080 OMPBuilder.createTaskwait(Loc: CGF.Builder);
6081 } else {
6082 llvm::Value *ThreadID = getThreadID(CGF, Loc);
6083 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
6084 auto &M = CGM.getModule();
6085 Address DependenciesArray = Address::invalid();
6086 llvm::Value *NumOfElements;
6087 std::tie(args&: NumOfElements, args&: DependenciesArray) =
6088 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
6089 if (!Data.Dependences.empty()) {
6090 llvm::Value *DepWaitTaskArgs[7];
6091 DepWaitTaskArgs[0] = UpLoc;
6092 DepWaitTaskArgs[1] = ThreadID;
6093 DepWaitTaskArgs[2] = NumOfElements;
6094 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
6095 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
6096 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6097 DepWaitTaskArgs[6] =
6098 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
6099
6100 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
6101
6102 // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
6103 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
6104 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
6105 // kmp_int32 has_no_wait); if dependence info is specified.
6106 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6107 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
6108 args: DepWaitTaskArgs);
6109
6110 } else {
6111
6112 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6113 // global_tid);
6114 llvm::Value *Args[] = {UpLoc, ThreadID};
6115 // Ignore return result until untied tasks are supported.
6116 CGF.EmitRuntimeCall(
6117 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_omp_taskwait),
6118 args: Args);
6119 }
6120 }
6121
6122 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
6123 Region->emitUntiedSwitch(CGF);
6124}
6125
6126void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6127 OpenMPDirectiveKind InnerKind,
6128 const RegionCodeGenTy &CodeGen,
6129 bool HasCancel) {
6130 if (!CGF.HaveInsertPoint())
6131 return;
6132 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
6133 InnerKind != OMPD_critical &&
6134 InnerKind != OMPD_master &&
6135 InnerKind != OMPD_masked);
6136 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6137}
6138
6139namespace {
6140enum RTCancelKind {
6141 CancelNoreq = 0,
6142 CancelParallel = 1,
6143 CancelLoop = 2,
6144 CancelSections = 3,
6145 CancelTaskgroup = 4
6146};
6147} // anonymous namespace
6148
6149static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6150 RTCancelKind CancelKind = CancelNoreq;
6151 if (CancelRegion == OMPD_parallel)
6152 CancelKind = CancelParallel;
6153 else if (CancelRegion == OMPD_for)
6154 CancelKind = CancelLoop;
6155 else if (CancelRegion == OMPD_sections)
6156 CancelKind = CancelSections;
6157 else {
6158 assert(CancelRegion == OMPD_taskgroup);
6159 CancelKind = CancelTaskgroup;
6160 }
6161 return CancelKind;
6162}
6163
6164void CGOpenMPRuntime::emitCancellationPointCall(
6165 CodeGenFunction &CGF, SourceLocation Loc,
6166 OpenMPDirectiveKind CancelRegion) {
6167 if (!CGF.HaveInsertPoint())
6168 return;
6169 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6170 // global_tid, kmp_int32 cncl_kind);
6171 if (auto *OMPRegionInfo =
6172 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6173 // For 'cancellation point taskgroup', the task region info may not have a
6174 // cancel. This may instead happen in another adjacent task.
6175 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6176 llvm::Value *Args[] = {
6177 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6178 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6179 // Ignore return result until untied tasks are supported.
6180 llvm::Value *Result = CGF.EmitRuntimeCall(
6181 callee: OMPBuilder.getOrCreateRuntimeFunction(
6182 M&: CGM.getModule(), FnID: OMPRTL___kmpc_cancellationpoint),
6183 args: Args);
6184 // if (__kmpc_cancellationpoint()) {
6185 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6186 // exit from construct;
6187 // }
6188 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6189 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6190 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6191 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6192 CGF.EmitBlock(BB: ExitBB);
6193 if (CancelRegion == OMPD_parallel)
6194 emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6195 // exit from construct;
6196 CodeGenFunction::JumpDest CancelDest =
6197 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6198 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6199 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6200 }
6201 }
6202}
6203
6204void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6205 const Expr *IfCond,
6206 OpenMPDirectiveKind CancelRegion) {
6207 if (!CGF.HaveInsertPoint())
6208 return;
6209 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6210 // kmp_int32 cncl_kind);
6211 auto &M = CGM.getModule();
6212 if (auto *OMPRegionInfo =
6213 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6214 auto &&ThenGen = [this, &M, Loc, CancelRegion,
6215 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
6216 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6217 llvm::Value *Args[] = {
6218 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6219 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6220 // Ignore return result until untied tasks are supported.
6221 llvm::Value *Result = CGF.EmitRuntimeCall(
6222 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_cancel), args: Args);
6223 // if (__kmpc_cancel()) {
6224 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6225 // exit from construct;
6226 // }
6227 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6228 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6229 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6230 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6231 CGF.EmitBlock(BB: ExitBB);
6232 if (CancelRegion == OMPD_parallel)
6233 RT.emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6234 // exit from construct;
6235 CodeGenFunction::JumpDest CancelDest =
6236 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6237 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6238 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6239 };
6240 if (IfCond) {
6241 emitIfClause(CGF, Cond: IfCond, ThenGen,
6242 ElseGen: [](CodeGenFunction &, PrePostActionTy &) {});
6243 } else {
6244 RegionCodeGenTy ThenRCG(ThenGen);
6245 ThenRCG(CGF);
6246 }
6247 }
6248}
6249
6250namespace {
6251/// Cleanup action for uses_allocators support.
6252class OMPUsesAllocatorsActionTy final : public PrePostActionTy {
6253 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators;
6254
6255public:
6256 OMPUsesAllocatorsActionTy(
6257 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
6258 : Allocators(Allocators) {}
6259 void Enter(CodeGenFunction &CGF) override {
6260 if (!CGF.HaveInsertPoint())
6261 return;
6262 for (const auto &AllocatorData : Allocators) {
6263 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(
6264 CGF, Allocator: AllocatorData.first, AllocatorTraits: AllocatorData.second);
6265 }
6266 }
6267 void Exit(CodeGenFunction &CGF) override {
6268 if (!CGF.HaveInsertPoint())
6269 return;
6270 for (const auto &AllocatorData : Allocators) {
6271 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,
6272 Allocator: AllocatorData.first);
6273 }
6274 }
6275};
6276} // namespace
6277
6278void CGOpenMPRuntime::emitTargetOutlinedFunction(
6279 const OMPExecutableDirective &D, StringRef ParentName,
6280 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6281 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6282 assert(!ParentName.empty() && "Invalid target entry parent name!");
6283 HasEmittedTargetRegion = true;
6284 SmallVector<std::pair<const Expr *, const Expr *>, 4> Allocators;
6285 for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) {
6286 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
6287 const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
6288 if (!D.AllocatorTraits)
6289 continue;
6290 Allocators.emplace_back(Args: D.Allocator, Args: D.AllocatorTraits);
6291 }
6292 }
6293 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
6294 CodeGen.setAction(UsesAllocatorAction);
6295 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6296 IsOffloadEntry, CodeGen);
6297}
6298
6299void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
6300 const Expr *Allocator,
6301 const Expr *AllocatorTraits) {
6302 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6303 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6304 // Use default memspace handle.
6305 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6306 llvm::Value *NumTraits = llvm::ConstantInt::get(
6307 Ty: CGF.IntTy, V: cast<ConstantArrayType>(
6308 Val: AllocatorTraits->getType()->getAsArrayTypeUnsafe())
6309 ->getSize()
6310 .getLimitedValue());
6311 LValue AllocatorTraitsLVal = CGF.EmitLValue(E: AllocatorTraits);
6312 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6313 Addr: AllocatorTraitsLVal.getAddress(), Ty: CGF.VoidPtrPtrTy, ElementTy: CGF.VoidPtrTy);
6314 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, T: CGF.getContext().VoidPtrTy,
6315 BaseInfo: AllocatorTraitsLVal.getBaseInfo(),
6316 TBAAInfo: AllocatorTraitsLVal.getTBAAInfo());
6317 llvm::Value *Traits = Addr.emitRawPointer(CGF);
6318
6319 llvm::Value *AllocatorVal =
6320 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6321 M&: CGM.getModule(), FnID: OMPRTL___kmpc_init_allocator),
6322 args: {ThreadId, MemSpaceHandle, NumTraits, Traits});
6323 // Store to allocator.
6324 CGF.EmitAutoVarAlloca(var: *cast<VarDecl>(
6325 Val: cast<DeclRefExpr>(Val: Allocator->IgnoreParenImpCasts())->getDecl()));
6326 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6327 AllocatorVal =
6328 CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: CGF.getContext().VoidPtrTy,
6329 DstTy: Allocator->getType(), Loc: Allocator->getExprLoc());
6330 CGF.EmitStoreOfScalar(value: AllocatorVal, lvalue: AllocatorLVal);
6331}
6332
6333void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
6334 const Expr *Allocator) {
6335 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6336 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6337 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6338 llvm::Value *AllocatorVal =
6339 CGF.EmitLoadOfScalar(lvalue: AllocatorLVal, Loc: Allocator->getExprLoc());
6340 AllocatorVal = CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: Allocator->getType(),
6341 DstTy: CGF.getContext().VoidPtrTy,
6342 Loc: Allocator->getExprLoc());
6343 (void)CGF.EmitRuntimeCall(
6344 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
6345 FnID: OMPRTL___kmpc_destroy_allocator),
6346 args: {ThreadId, AllocatorVal});
6347}
6348
6349void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams(
6350 const OMPExecutableDirective &D, CodeGenFunction &CGF,
6351 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6352 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6353 "invalid default attrs structure");
6354 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6355 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6356
6357 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: Attrs.MinTeams.front(),
6358 MaxTeamsVal);
6359 getNumThreadsExprForTargetDirective(CGF, D, UpperBound&: MaxThreadsVal,
6360 /*UpperBoundOnly=*/true);
6361
6362 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6363 for (auto *A : C->getAttrs()) {
6364 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6365 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6366 if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(Val: A))
6367 CGM.handleCUDALaunchBoundsAttr(F: nullptr, A: Attr, MaxThreadsVal: &AttrMaxThreadsVal,
6368 MinBlocksVal: &AttrMinBlocksVal, MaxClusterRankVal: &AttrMaxBlocksVal);
6369 else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(Val: A))
6370 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6371 F: nullptr, A: Attr, /*ReqdWGS=*/nullptr, MinThreadsVal: &AttrMinThreadsVal,
6372 MaxThreadsVal: &AttrMaxThreadsVal);
6373 else
6374 continue;
6375
6376 Attrs.MinThreads.front() =
6377 std::max(a: Attrs.MinThreads.front(), b: AttrMinThreadsVal);
6378 if (AttrMaxThreadsVal > 0)
6379 MaxThreadsVal = MaxThreadsVal > 0
6380 ? std::min(a: MaxThreadsVal, b: AttrMaxThreadsVal)
6381 : AttrMaxThreadsVal;
6382 Attrs.MinTeams.front() =
6383 std::max(a: Attrs.MinTeams.front(), b: AttrMinBlocksVal);
6384 if (AttrMaxBlocksVal > 0)
6385 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(a: MaxTeamsVal, b: AttrMaxBlocksVal)
6386 : AttrMaxBlocksVal;
6387 }
6388 }
6389}
6390
6391void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6392 const OMPExecutableDirective &D, StringRef ParentName,
6393 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6394 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6395
6396 llvm::TargetRegionEntryInfo EntryInfo =
6397 getEntryInfoFromPresumedLoc(CGM, OMPBuilder, BeginLoc: D.getBeginLoc(), ParentName);
6398
6399 CodeGenFunction CGF(CGM, true);
6400 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6401 [&CGF, &D, &CodeGen, this](StringRef EntryFnName) {
6402 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
6403
6404 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6405 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6406 if (CGM.getLangOpts().OpenMPIsTargetDevice && !isGPU())
6407 return CGF.GenerateOpenMPCapturedStmtFunctionAggregate(S: CS, D);
6408 return CGF.GenerateOpenMPCapturedStmtFunction(S: CS, D);
6409 };
6410
6411 cantFail(Err: OMPBuilder.emitTargetRegionFunction(
6412 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6413 OutlinedFnID));
6414
6415 if (!OutlinedFn)
6416 return;
6417
6418 CGM.getTargetCodeGenInfo().setTargetAttributes(D: nullptr, GV: OutlinedFn, M&: CGM);
6419
6420 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6421 for (auto *A : C->getAttrs()) {
6422 if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(Val: A))
6423 CGM.handleAMDGPUWavesPerEUAttr(F: OutlinedFn, A: Attr);
6424 }
6425 }
6426 registerVTable(D);
6427}
6428
6429/// Checks if the expression is constant or does not have non-trivial function
6430/// calls.
6431static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6432 // We can skip constant expressions.
6433 // We can skip expressions with trivial calls or simple expressions.
6434 return (E->isEvaluatable(Ctx, AllowSideEffects: Expr::SE_AllowUndefinedBehavior) ||
6435 !E->hasNonTrivialCall(Ctx)) &&
6436 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6437}
6438
6439const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,
6440 const Stmt *Body) {
6441 const Stmt *Child = Body->IgnoreContainers();
6442 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Val: Child)) {
6443 Child = nullptr;
6444 for (const Stmt *S : C->body()) {
6445 if (const auto *E = dyn_cast<Expr>(Val: S)) {
6446 if (isTrivial(Ctx, E))
6447 continue;
6448 }
6449 // Some of the statements can be ignored.
6450 if (isa<AsmStmt>(Val: S) || isa<NullStmt>(Val: S) || isa<OMPFlushDirective>(Val: S) ||
6451 isa<OMPBarrierDirective>(Val: S) || isa<OMPTaskyieldDirective>(Val: S))
6452 continue;
6453 // Analyze declarations.
6454 if (const auto *DS = dyn_cast<DeclStmt>(Val: S)) {
6455 if (llvm::all_of(Range: DS->decls(), P: [](const Decl *D) {
6456 if (isa<EmptyDecl>(Val: D) || isa<DeclContext>(Val: D) ||
6457 isa<TypeDecl>(Val: D) || isa<PragmaCommentDecl>(Val: D) ||
6458 isa<PragmaDetectMismatchDecl>(Val: D) || isa<UsingDecl>(Val: D) ||
6459 isa<UsingDirectiveDecl>(Val: D) ||
6460 isa<OMPDeclareReductionDecl>(Val: D) ||
6461 isa<OMPThreadPrivateDecl>(Val: D) || isa<OMPAllocateDecl>(Val: D))
6462 return true;
6463 const auto *VD = dyn_cast<VarDecl>(Val: D);
6464 if (!VD)
6465 return false;
6466 return VD->hasGlobalStorage() || !VD->isUsed();
6467 }))
6468 continue;
6469 }
6470 // Found multiple children - cannot get the one child only.
6471 if (Child)
6472 return nullptr;
6473 Child = S;
6474 }
6475 if (Child)
6476 Child = Child->IgnoreContainers();
6477 }
6478 return Child;
6479}
6480
6481const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(
6482 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6483 int32_t &MaxTeamsVal) {
6484
6485 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6486 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6487 "Expected target-based executable directive.");
6488 switch (DirectiveKind) {
6489 case OMPD_target: {
6490 const auto *CS = D.getInnermostCapturedStmt();
6491 const auto *Body =
6492 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6493 const Stmt *ChildStmt =
6494 CGOpenMPRuntime::getSingleCompoundChild(Ctx&: CGF.getContext(), Body);
6495 if (const auto *NestedDir =
6496 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
6497 if (isOpenMPTeamsDirective(DKind: NestedDir->getDirectiveKind())) {
6498 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6499 const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>()
6500 ->getNumTeams()
6501 .front();
6502 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6503 if (auto Constant =
6504 NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6505 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6506 return NumTeams;
6507 }
6508 MinTeamsVal = MaxTeamsVal = 0;
6509 return nullptr;
6510 }
6511 MinTeamsVal = MaxTeamsVal = 1;
6512 return nullptr;
6513 }
6514 // A value of -1 is used to check if we need to emit no teams region
6515 MinTeamsVal = MaxTeamsVal = -1;
6516 return nullptr;
6517 }
6518 case OMPD_target_teams_loop:
6519 case OMPD_target_teams:
6520 case OMPD_target_teams_distribute:
6521 case OMPD_target_teams_distribute_simd:
6522 case OMPD_target_teams_distribute_parallel_for:
6523 case OMPD_target_teams_distribute_parallel_for_simd: {
6524 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6525 const Expr *NumTeams =
6526 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front();
6527 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6528 if (auto Constant = NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6529 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6530 return NumTeams;
6531 }
6532 MinTeamsVal = MaxTeamsVal = 0;
6533 return nullptr;
6534 }
6535 case OMPD_target_parallel:
6536 case OMPD_target_parallel_for:
6537 case OMPD_target_parallel_for_simd:
6538 case OMPD_target_parallel_loop:
6539 case OMPD_target_simd:
6540 MinTeamsVal = MaxTeamsVal = 1;
6541 return nullptr;
6542 case OMPD_parallel:
6543 case OMPD_for:
6544 case OMPD_parallel_for:
6545 case OMPD_parallel_loop:
6546 case OMPD_parallel_master:
6547 case OMPD_parallel_sections:
6548 case OMPD_for_simd:
6549 case OMPD_parallel_for_simd:
6550 case OMPD_cancel:
6551 case OMPD_cancellation_point:
6552 case OMPD_ordered_standalone:
6553 case OMPD_ordered_blockassoc:
6554 case OMPD_threadprivate:
6555 case OMPD_allocate:
6556 case OMPD_task:
6557 case OMPD_simd:
6558 case OMPD_tile:
6559 case OMPD_unroll:
6560 case OMPD_sections:
6561 case OMPD_section:
6562 case OMPD_single:
6563 case OMPD_master:
6564 case OMPD_critical:
6565 case OMPD_taskyield:
6566 case OMPD_barrier:
6567 case OMPD_taskwait:
6568 case OMPD_taskgroup:
6569 case OMPD_atomic:
6570 case OMPD_flush:
6571 case OMPD_depobj:
6572 case OMPD_scan:
6573 case OMPD_teams:
6574 case OMPD_target_data:
6575 case OMPD_target_exit_data:
6576 case OMPD_target_enter_data:
6577 case OMPD_distribute:
6578 case OMPD_distribute_simd:
6579 case OMPD_distribute_parallel_for:
6580 case OMPD_distribute_parallel_for_simd:
6581 case OMPD_teams_distribute:
6582 case OMPD_teams_distribute_simd:
6583 case OMPD_teams_distribute_parallel_for:
6584 case OMPD_teams_distribute_parallel_for_simd:
6585 case OMPD_target_update:
6586 case OMPD_declare_simd:
6587 case OMPD_declare_variant:
6588 case OMPD_begin_declare_variant:
6589 case OMPD_end_declare_variant:
6590 case OMPD_declare_target:
6591 case OMPD_end_declare_target:
6592 case OMPD_declare_reduction:
6593 case OMPD_declare_mapper:
6594 case OMPD_taskloop:
6595 case OMPD_taskloop_simd:
6596 case OMPD_master_taskloop:
6597 case OMPD_master_taskloop_simd:
6598 case OMPD_parallel_master_taskloop:
6599 case OMPD_parallel_master_taskloop_simd:
6600 case OMPD_requires:
6601 case OMPD_metadirective:
6602 case OMPD_unknown:
6603 break;
6604 default:
6605 break;
6606 }
6607 llvm_unreachable("Unexpected directive kind.");
6608}
6609
6610llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective(
6611 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6612 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6613 "Clauses associated with the teams directive expected to be emitted "
6614 "only for the host!");
6615 CGBuilderTy &Bld = CGF.Builder;
6616 int32_t MinNT = -1, MaxNT = -1;
6617 const Expr *NumTeams =
6618 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: MinNT, MaxTeamsVal&: MaxNT);
6619 if (NumTeams != nullptr) {
6620 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6621
6622 switch (DirectiveKind) {
6623 case OMPD_target: {
6624 const auto *CS = D.getInnermostCapturedStmt();
6625 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6626 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6627 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6628 /*IgnoreResultAssign*/ true);
6629 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6630 /*isSigned=*/true);
6631 }
6632 case OMPD_target_teams:
6633 case OMPD_target_teams_distribute:
6634 case OMPD_target_teams_distribute_simd:
6635 case OMPD_target_teams_distribute_parallel_for:
6636 case OMPD_target_teams_distribute_parallel_for_simd: {
6637 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6638 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6639 /*IgnoreResultAssign*/ true);
6640 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6641 /*isSigned=*/true);
6642 }
6643 default:
6644 break;
6645 }
6646 }
6647
6648 assert(MinNT == MaxNT && "Num threads ranges require handling here.");
6649 return llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: MinNT);
6650}
6651
6652/// Merge the thread count upper bound \p Val into \p UpperBound.
6653///
6654/// \p UpperBound is -1 while no thread limiting clause has been seen, 0 once
6655/// one has been seen whose value is not known at compile time, and otherwise
6656/// the smallest constant bound found so far.
6657///
6658/// Thread limiting clauses compose by taking the minimum, so a constant bound
6659/// stays valid whatever the clauses that are not compile time constants
6660/// evaluate to. That makes it correct to replace the 0 marker with \p Val, and
6661/// necessary to keep a clause from raising a smaller bound found earlier.
6662static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val) {
6663 UpperBound = UpperBound > 0 ? std::min(a: UpperBound, b: Val) : Val;
6664}
6665
6666/// Check for a num threads constant value (stored in \p DefaultVal), or
6667/// expression (stored in \p E). If the value is conditional (via an if-clause),
6668/// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are
6669/// nullptr, no expression evaluation is perfomed.
6670static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6671 const Expr **E, int32_t &UpperBound,
6672 bool UpperBoundOnly, llvm::Value **CondVal) {
6673 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6674 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6675 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6676 if (!Dir)
6677 return;
6678
6679 if (isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6680 // Handle if clause. If if clause present, the number of threads is
6681 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6682 if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) {
6683 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6684 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6685 const OMPIfClause *IfClause = nullptr;
6686 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6687 if (C->getNameModifier() == OMPD_unknown ||
6688 C->getNameModifier() == OMPD_parallel) {
6689 IfClause = C;
6690 break;
6691 }
6692 }
6693 if (IfClause) {
6694 const Expr *CondExpr = IfClause->getCondition();
6695 bool Result;
6696 if (CondExpr->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6697 if (!Result) {
6698 UpperBound = 1;
6699 return;
6700 }
6701 } else {
6702 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());
6703 if (const auto *PreInit =
6704 cast_or_null<DeclStmt>(Val: IfClause->getPreInitStmt())) {
6705 for (const auto *I : PreInit->decls()) {
6706 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6707 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6708 } else {
6709 CodeGenFunction::AutoVarEmission Emission =
6710 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6711 CGF.EmitAutoVarCleanups(emission: Emission);
6712 }
6713 }
6714 *CondVal = CGF.EvaluateExprAsBool(E: CondExpr);
6715 }
6716 }
6717 }
6718 }
6719 // Check the value of num_threads clause iff if clause was not specified
6720 // or is not evaluated to false.
6721 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6722 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6723 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6724 const auto *NumThreadsClause =
6725 Dir->getSingleClause<OMPNumThreadsClause>();
6726 const Expr *NTExpr = NumThreadsClause->getNumThreads().front();
6727 if (NTExpr->isIntegerConstantExpr(Ctx: CGF.getContext()))
6728 if (auto Constant = NTExpr->getIntegerConstantExpr(Ctx: CGF.getContext()))
6729 mergeThreadCountUpperBound(
6730 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
6731 // If we haven't found a upper bound, remember we saw a thread limiting
6732 // clause.
6733 if (UpperBound == -1)
6734 UpperBound = 0;
6735 if (!E)
6736 return;
6737 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6738 if (const auto *PreInit =
6739 cast_or_null<DeclStmt>(Val: NumThreadsClause->getPreInitStmt())) {
6740 for (const auto *I : PreInit->decls()) {
6741 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6742 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6743 } else {
6744 CodeGenFunction::AutoVarEmission Emission =
6745 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6746 CGF.EmitAutoVarCleanups(emission: Emission);
6747 }
6748 }
6749 }
6750 *E = NTExpr;
6751 }
6752 return;
6753 }
6754 if (isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6755 UpperBound = 1;
6756}
6757
6758const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective(
6759 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6760 bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) {
6761 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6762 "Clauses associated with the teams directive expected to be emitted "
6763 "only for the host!");
6764 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6765 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6766 "Expected target-based executable directive.");
6767
6768 const Expr *NT = nullptr;
6769 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6770
6771 auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) {
6772 if (E->isIntegerConstantExpr(Ctx: CGF.getContext())) {
6773 if (auto Constant = E->getIntegerConstantExpr(Ctx: CGF.getContext()))
6774 mergeThreadCountUpperBound(
6775 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
6776 }
6777 // If we haven't found a upper bound, remember we saw a thread limiting
6778 // clause.
6779 if (UpperBound == -1)
6780 UpperBound = 0;
6781 if (EPtr)
6782 *EPtr = E;
6783 };
6784
6785 auto ReturnSequential = [&]() {
6786 UpperBound = 1;
6787 return NT;
6788 };
6789
6790 switch (DirectiveKind) {
6791 case OMPD_target: {
6792 const CapturedStmt *CS = D.getInnermostCapturedStmt();
6793 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6794 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6795 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6796 // TODO: The standard is not clear how to resolve two thread limit clauses,
6797 // let's pick the teams one if it's present, otherwise the target one.
6798 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6799 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6800 if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) {
6801 ThreadLimitClause = TLC;
6802 if (ThreadLimitExpr) {
6803 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6804 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6805 CodeGenFunction::LexicalScope Scope(
6806 CGF,
6807 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
6808 if (const auto *PreInit =
6809 cast_or_null<DeclStmt>(Val: ThreadLimitClause->getPreInitStmt())) {
6810 for (const auto *I : PreInit->decls()) {
6811 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6812 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6813 } else {
6814 CodeGenFunction::AutoVarEmission Emission =
6815 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6816 CGF.EmitAutoVarCleanups(emission: Emission);
6817 }
6818 }
6819 }
6820 }
6821 }
6822 }
6823 if (ThreadLimitClause)
6824 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6825 ThreadLimitExpr);
6826 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6827 if (isOpenMPTeamsDirective(DKind: Dir->getDirectiveKind()) &&
6828 !isOpenMPDistributeDirective(DKind: Dir->getDirectiveKind())) {
6829 CS = Dir->getInnermostCapturedStmt();
6830 // Now that the 'teams' level has been peeled off, the remainder is
6831 // shaped like a 'target teams' region, so pick up the num_threads of
6832 // the directive nested in it the same way the OMPD_target_teams case
6833 // below does. Without this the upper bound of a construct written as
6834 // 'target' / 'teams' / 'distribute parallel for' would stay at the
6835 // default, while every combined spelling of the same construct honors
6836 // the clause. Only the bound is taken here: passing null for the
6837 // expression and the condition keeps this from emitting anything, so
6838 // the value the host passes to the kernel launch is left as it was.
6839 getNumThreads(CGF, CS, /*E=*/nullptr, UpperBound, UpperBoundOnly,
6840 /*CondVal=*/nullptr);
6841 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6842 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6843 Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6844 }
6845 if (Dir && isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6846 CS = Dir->getInnermostCapturedStmt();
6847 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6848 } else if (Dir && isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6849 return ReturnSequential();
6850 }
6851 return NT;
6852 }
6853 case OMPD_target_teams: {
6854 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6855 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6856 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6857 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6858 ThreadLimitExpr);
6859 }
6860 const CapturedStmt *CS = D.getInnermostCapturedStmt();
6861 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6862 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6863 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6864 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6865 if (Dir->getDirectiveKind() == OMPD_distribute) {
6866 CS = Dir->getInnermostCapturedStmt();
6867 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6868 }
6869 }
6870 return NT;
6871 }
6872 case OMPD_target_teams_distribute:
6873 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6874 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6875 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6876 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6877 ThreadLimitExpr);
6878 }
6879 getNumThreads(CGF, CS: D.getInnermostCapturedStmt(), E: NTPtr, UpperBound,
6880 UpperBoundOnly, CondVal);
6881 return NT;
6882 case OMPD_target_teams_loop:
6883 case OMPD_target_parallel_loop:
6884 case OMPD_target_parallel:
6885 case OMPD_target_parallel_for:
6886 case OMPD_target_parallel_for_simd:
6887 case OMPD_target_teams_distribute_parallel_for:
6888 case OMPD_target_teams_distribute_parallel_for_simd: {
6889 if (CondVal && D.hasClausesOfKind<OMPIfClause>()) {
6890 const OMPIfClause *IfClause = nullptr;
6891 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
6892 if (C->getNameModifier() == OMPD_unknown ||
6893 C->getNameModifier() == OMPD_parallel) {
6894 IfClause = C;
6895 break;
6896 }
6897 }
6898 if (IfClause) {
6899 const Expr *Cond = IfClause->getCondition();
6900 bool Result;
6901 if (Cond->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6902 if (!Result)
6903 return ReturnSequential();
6904 } else {
6905 CodeGenFunction::RunCleanupsScope Scope(CGF);
6906 *CondVal = CGF.EvaluateExprAsBool(E: Cond);
6907 }
6908 }
6909 }
6910 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6911 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6912 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6913 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6914 ThreadLimitExpr);
6915 }
6916 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
6917 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6918 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
6919 CheckForConstExpr(NumThreadsClause->getNumThreads().front(), nullptr);
6920 return NumThreadsClause->getNumThreads().front();
6921 }
6922 return NT;
6923 }
6924 case OMPD_target_teams_distribute_simd:
6925 case OMPD_target_simd:
6926 return ReturnSequential();
6927 default:
6928 break;
6929 }
6930 llvm_unreachable("Unsupported directive kind.");
6931}
6932
6933llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective(
6934 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6935 llvm::Value *NumThreadsVal = nullptr;
6936 llvm::Value *CondVal = nullptr;
6937 llvm::Value *ThreadLimitVal = nullptr;
6938 const Expr *ThreadLimitExpr = nullptr;
6939 int32_t UpperBound = -1;
6940
6941 const Expr *NT = getNumThreadsExprForTargetDirective(
6942 CGF, D, UpperBound, /* UpperBoundOnly */ false, CondVal: &CondVal,
6943 ThreadLimitExpr: &ThreadLimitExpr);
6944
6945 // Thread limit expressions are used below, emit them.
6946 if (ThreadLimitExpr) {
6947 ThreadLimitVal =
6948 CGF.EmitScalarExpr(E: ThreadLimitExpr, /*IgnoreResultAssign=*/true);
6949 ThreadLimitVal = CGF.Builder.CreateIntCast(V: ThreadLimitVal, DestTy: CGF.Int32Ty,
6950 /*isSigned=*/false);
6951 }
6952
6953 // Generate the num teams expression.
6954 if (UpperBound == 1) {
6955 NumThreadsVal = CGF.Builder.getInt32(C: UpperBound);
6956 } else if (NT) {
6957 NumThreadsVal = CGF.EmitScalarExpr(E: NT, /*IgnoreResultAssign=*/true);
6958 NumThreadsVal = CGF.Builder.CreateIntCast(V: NumThreadsVal, DestTy: CGF.Int32Ty,
6959 /*isSigned=*/false);
6960 } else if (ThreadLimitVal) {
6961 // If we do not have a num threads value but a thread limit, replace the
6962 // former with the latter. We know handled the thread limit expression.
6963 NumThreadsVal = ThreadLimitVal;
6964 ThreadLimitVal = nullptr;
6965 } else {
6966 // Default to "0" which means runtime choice.
6967 assert(!ThreadLimitVal && "Default not applicable with thread limit value");
6968 NumThreadsVal = CGF.Builder.getInt32(C: 0);
6969 }
6970
6971 // Handle if clause. If if clause present, the number of threads is
6972 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6973 if (CondVal) {
6974 CodeGenFunction::RunCleanupsScope Scope(CGF);
6975 NumThreadsVal = CGF.Builder.CreateSelect(C: CondVal, True: NumThreadsVal,
6976 False: CGF.Builder.getInt32(C: 1));
6977 }
6978
6979 // If the thread limit and num teams expression were present, take the
6980 // minimum.
6981 if (ThreadLimitVal) {
6982 NumThreadsVal = CGF.Builder.CreateSelect(
6983 C: CGF.Builder.CreateICmpULT(LHS: ThreadLimitVal, RHS: NumThreadsVal),
6984 True: ThreadLimitVal, False: NumThreadsVal);
6985 }
6986
6987 return NumThreadsVal;
6988}
6989
6990namespace {
6991LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
6992
6993// Utility to handle information from clauses associated with a given
6994// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
6995// It provides a convenient interface to obtain the information and generate
6996// code for that information.
6997class MappableExprsHandler {
6998public:
6999 /// Custom comparator for attach-pointer expressions that compares them by
7000 /// complexity (i.e. their component-depth) first, then by the order in which
7001 /// they were computed by collectAttachPtrExprInfo(), if they are semantically
7002 /// different.
7003 struct AttachPtrExprComparator {
7004 const MappableExprsHandler &Handler;
7005 // Cache of previous equality comparison results.
7006 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool>
7007 CachedEqualityComparisons;
7008
7009 AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {}
7010 AttachPtrExprComparator() = delete;
7011
7012 // Return true iff LHS is "less than" RHS.
7013 bool operator()(const Expr *LHS, const Expr *RHS) const {
7014 if (LHS == RHS)
7015 return false;
7016
7017 // First, compare by complexity (depth)
7018 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(Val: LHS);
7019 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(Val: RHS);
7020
7021 std::optional<size_t> DepthLHS =
7022 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
7023 : std::nullopt;
7024 std::optional<size_t> DepthRHS =
7025 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
7026 : std::nullopt;
7027
7028 // std::nullopt (no attach pointer) has lowest complexity
7029 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
7030 // Both have same complexity, now check semantic equality
7031 if (areEqual(LHS, RHS))
7032 return false;
7033 // Different semantically, compare by computation order
7034 return wasComputedBefore(LHS, RHS);
7035 }
7036 if (!DepthLHS.has_value())
7037 return true; // LHS has lower complexity
7038 if (!DepthRHS.has_value())
7039 return false; // RHS has lower complexity
7040
7041 // Both have values, compare by depth (lower depth = lower complexity)
7042 if (DepthLHS.value() != DepthRHS.value())
7043 return DepthLHS.value() < DepthRHS.value();
7044
7045 // Same complexity, now check semantic equality
7046 if (areEqual(LHS, RHS))
7047 return false;
7048 // Different semantically, compare by computation order
7049 return wasComputedBefore(LHS, RHS);
7050 }
7051
7052 public:
7053 /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached
7054 /// results, if available, otherwise does a recursive semantic comparison.
7055 bool areEqual(const Expr *LHS, const Expr *RHS) const {
7056 // Check cache first for faster lookup
7057 const auto CachedResultIt = CachedEqualityComparisons.find(Val: {LHS, RHS});
7058 if (CachedResultIt != CachedEqualityComparisons.end())
7059 return CachedResultIt->second;
7060
7061 bool ComparisonResult = areSemanticallyEqual(LHS, RHS);
7062
7063 // Cache the result for future lookups (both orders since semantic
7064 // equality is commutative)
7065 CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult;
7066 CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult;
7067 return ComparisonResult;
7068 }
7069
7070 /// Compare the two attach-ptr expressions by their computation order.
7071 /// Returns true iff LHS was computed before RHS by
7072 /// collectAttachPtrExprInfo().
7073 bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const {
7074 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(Val: LHS);
7075 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(Val: RHS);
7076
7077 return OrderLHS < OrderRHS;
7078 }
7079
7080 private:
7081 /// Helper function to compare attach-pointer expressions semantically.
7082 /// This function handles various expression types that can be part of an
7083 /// attach-pointer.
7084 /// TODO: Not urgent, but we should ideally return true when comparing
7085 /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc.
7086 bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const {
7087 if (LHS == RHS)
7088 return true;
7089
7090 // If only one is null, they aren't equal
7091 if (!LHS || !RHS)
7092 return false;
7093
7094 ASTContext &Ctx = Handler.CGF.getContext();
7095 // Strip away parentheses and no-op casts to get to the core expression
7096 LHS = LHS->IgnoreParenNoopCasts(Ctx);
7097 RHS = RHS->IgnoreParenNoopCasts(Ctx);
7098
7099 // Direct pointer comparison of the underlying expressions
7100 if (LHS == RHS)
7101 return true;
7102
7103 // Check if the expression classes match
7104 if (LHS->getStmtClass() != RHS->getStmtClass())
7105 return false;
7106
7107 // Handle DeclRefExpr (variable references)
7108 if (const auto *LD = dyn_cast<DeclRefExpr>(Val: LHS)) {
7109 const auto *RD = dyn_cast<DeclRefExpr>(Val: RHS);
7110 if (!RD)
7111 return false;
7112 return LD->getDecl()->getCanonicalDecl() ==
7113 RD->getDecl()->getCanonicalDecl();
7114 }
7115
7116 // Handle ArraySubscriptExpr (array indexing like a[i])
7117 if (const auto *LA = dyn_cast<ArraySubscriptExpr>(Val: LHS)) {
7118 const auto *RA = dyn_cast<ArraySubscriptExpr>(Val: RHS);
7119 if (!RA)
7120 return false;
7121 return areSemanticallyEqual(LHS: LA->getBase(), RHS: RA->getBase()) &&
7122 areSemanticallyEqual(LHS: LA->getIdx(), RHS: RA->getIdx());
7123 }
7124
7125 // Handle MemberExpr (member access like s.m or p->m)
7126 if (const auto *LM = dyn_cast<MemberExpr>(Val: LHS)) {
7127 const auto *RM = dyn_cast<MemberExpr>(Val: RHS);
7128 if (!RM)
7129 return false;
7130 if (LM->getMemberDecl()->getCanonicalDecl() !=
7131 RM->getMemberDecl()->getCanonicalDecl())
7132 return false;
7133 return areSemanticallyEqual(LHS: LM->getBase(), RHS: RM->getBase());
7134 }
7135
7136 // Handle UnaryOperator (unary operations like *p, &x, etc.)
7137 if (const auto *LU = dyn_cast<UnaryOperator>(Val: LHS)) {
7138 const auto *RU = dyn_cast<UnaryOperator>(Val: RHS);
7139 if (!RU)
7140 return false;
7141 if (LU->getOpcode() != RU->getOpcode())
7142 return false;
7143 return areSemanticallyEqual(LHS: LU->getSubExpr(), RHS: RU->getSubExpr());
7144 }
7145
7146 // Handle BinaryOperator (binary operations like p + offset)
7147 if (const auto *LB = dyn_cast<BinaryOperator>(Val: LHS)) {
7148 const auto *RB = dyn_cast<BinaryOperator>(Val: RHS);
7149 if (!RB)
7150 return false;
7151 if (LB->getOpcode() != RB->getOpcode())
7152 return false;
7153 return areSemanticallyEqual(LHS: LB->getLHS(), RHS: RB->getLHS()) &&
7154 areSemanticallyEqual(LHS: LB->getRHS(), RHS: RB->getRHS());
7155 }
7156
7157 // Handle ArraySectionExpr (array sections like a[0:1])
7158 // Attach pointers should not contain array-sections, but currently we
7159 // don't emit an error.
7160 if (const auto *LAS = dyn_cast<ArraySectionExpr>(Val: LHS)) {
7161 const auto *RAS = dyn_cast<ArraySectionExpr>(Val: RHS);
7162 if (!RAS)
7163 return false;
7164 return areSemanticallyEqual(LHS: LAS->getBase(), RHS: RAS->getBase()) &&
7165 areSemanticallyEqual(LHS: LAS->getLowerBound(),
7166 RHS: RAS->getLowerBound()) &&
7167 areSemanticallyEqual(LHS: LAS->getLength(), RHS: RAS->getLength());
7168 }
7169
7170 // Handle CastExpr (explicit casts)
7171 if (const auto *LC = dyn_cast<CastExpr>(Val: LHS)) {
7172 const auto *RC = dyn_cast<CastExpr>(Val: RHS);
7173 if (!RC)
7174 return false;
7175 if (LC->getCastKind() != RC->getCastKind())
7176 return false;
7177 return areSemanticallyEqual(LHS: LC->getSubExpr(), RHS: RC->getSubExpr());
7178 }
7179
7180 // Handle CXXThisExpr (this pointer)
7181 if (isa<CXXThisExpr>(Val: LHS) && isa<CXXThisExpr>(Val: RHS))
7182 return true;
7183
7184 // Handle IntegerLiteral (integer constants)
7185 if (const auto *LI = dyn_cast<IntegerLiteral>(Val: LHS)) {
7186 const auto *RI = dyn_cast<IntegerLiteral>(Val: RHS);
7187 if (!RI)
7188 return false;
7189 return LI->getValue() == RI->getValue();
7190 }
7191
7192 // Handle CharacterLiteral (character constants)
7193 if (const auto *LC = dyn_cast<CharacterLiteral>(Val: LHS)) {
7194 const auto *RC = dyn_cast<CharacterLiteral>(Val: RHS);
7195 if (!RC)
7196 return false;
7197 return LC->getValue() == RC->getValue();
7198 }
7199
7200 // Handle FloatingLiteral (floating point constants)
7201 if (const auto *LF = dyn_cast<FloatingLiteral>(Val: LHS)) {
7202 const auto *RF = dyn_cast<FloatingLiteral>(Val: RHS);
7203 if (!RF)
7204 return false;
7205 // Use bitwise comparison for floating point literals
7206 return LF->getValue().bitwiseIsEqual(RHS: RF->getValue());
7207 }
7208
7209 // Handle StringLiteral (string constants)
7210 if (const auto *LS = dyn_cast<StringLiteral>(Val: LHS)) {
7211 const auto *RS = dyn_cast<StringLiteral>(Val: RHS);
7212 if (!RS)
7213 return false;
7214 return LS->getString() == RS->getString();
7215 }
7216
7217 // Handle CXXNullPtrLiteralExpr (nullptr)
7218 if (isa<CXXNullPtrLiteralExpr>(Val: LHS) && isa<CXXNullPtrLiteralExpr>(Val: RHS))
7219 return true;
7220
7221 // Handle CXXBoolLiteralExpr (true/false)
7222 if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(Val: LHS)) {
7223 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(Val: RHS);
7224 if (!RB)
7225 return false;
7226 return LB->getValue() == RB->getValue();
7227 }
7228
7229 // Fallback for other forms - use the existing comparison method
7230 return Expr::isSameComparisonOperand(E1: LHS, E2: RHS);
7231 }
7232 };
7233
7234 /// Get the offset of the OMP_MAP_MEMBER_OF field.
7235 static unsigned getFlagMemberOffset() {
7236 unsigned Offset = 0;
7237 for (uint64_t Remain =
7238 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
7239 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7240 !(Remain & 1); Remain = Remain >> 1)
7241 Offset++;
7242 return Offset;
7243 }
7244
7245 /// Class that holds debugging information for a data mapping to be passed to
7246 /// the runtime library.
7247 class MappingExprInfo {
7248 /// The variable declaration used for the data mapping.
7249 const ValueDecl *MapDecl = nullptr;
7250 /// The original expression used in the map clause, or null if there is
7251 /// none.
7252 const Expr *MapExpr = nullptr;
7253
7254 public:
7255 MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr)
7256 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7257
7258 const ValueDecl *getMapDecl() const { return MapDecl; }
7259 const Expr *getMapExpr() const { return MapExpr; }
7260 };
7261
7262 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7263 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7264 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7265 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7266 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7267 using MapNonContiguousArrayTy =
7268 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7269 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7270 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7271 using MapData =
7272 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7273 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,
7274 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;
7275 using MapDataArrayTy = SmallVector<MapData, 4>;
7276
7277 /// This structure contains combined information generated for mappable
7278 /// clauses, including base pointers, pointers, sizes, map types, user-defined
7279 /// mappers, and non-contiguous information.
7280 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7281 MapExprsArrayTy Exprs;
7282 MapValueDeclsArrayTy Mappers;
7283 MapValueDeclsArrayTy DevicePtrDecls;
7284
7285 /// Append arrays in \a CurInfo.
7286 void append(MapCombinedInfoTy &CurInfo) {
7287 Exprs.append(in_start: CurInfo.Exprs.begin(), in_end: CurInfo.Exprs.end());
7288 DevicePtrDecls.append(in_start: CurInfo.DevicePtrDecls.begin(),
7289 in_end: CurInfo.DevicePtrDecls.end());
7290 Mappers.append(in_start: CurInfo.Mappers.begin(), in_end: CurInfo.Mappers.end());
7291 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7292 }
7293 };
7294
7295 /// Map between a struct and the its lowest & highest elements which have been
7296 /// mapped.
7297 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7298 /// HE(FieldIndex, Pointer)}
7299 struct StructRangeInfoTy {
7300 MapCombinedInfoTy PreliminaryMapData;
7301 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7302 0, Address::invalid()};
7303 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7304 0, Address::invalid()};
7305 Address Base = Address::invalid();
7306 Address LB = Address::invalid();
7307 bool IsArraySection = false;
7308 bool HasCompleteRecord = false;
7309 };
7310
7311 /// A struct to store the attach pointer and pointee information, to be used
7312 /// when emitting an attach entry.
7313 struct AttachInfoTy {
7314 Address AttachPtrAddr = Address::invalid();
7315 Address AttachPteeAddr = Address::invalid();
7316 const ValueDecl *AttachPtrDecl = nullptr;
7317 const Expr *AttachMapExpr = nullptr;
7318
7319 bool isValid() const {
7320 return AttachPtrAddr.isValid() && AttachPteeAddr.isValid();
7321 }
7322 };
7323
7324 /// Check if there's any component list where the attach pointer expression
7325 /// matches the given captured variable.
7326 bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const {
7327 for (const auto &AttachEntry : AttachPtrExprMap) {
7328 if (AttachEntry.second) {
7329 // Check if the attach pointer expression is a DeclRefExpr that
7330 // references the captured variable
7331 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: AttachEntry.second))
7332 if (DRE->getDecl() == VD)
7333 return true;
7334 }
7335 }
7336 return false;
7337 }
7338
7339 /// Get the previously-cached attach pointer for a component list, if-any.
7340 const Expr *getAttachPtrExpr(
7341 OMPClauseMappableExprCommon::MappableExprComponentListRef Components)
7342 const {
7343 const auto It = AttachPtrExprMap.find(Val: Components);
7344 if (It != AttachPtrExprMap.end())
7345 return It->second;
7346
7347 return nullptr;
7348 }
7349
7350private:
7351 /// Kind that defines how a device pointer has to be returned.
7352 struct MapInfo {
7353 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7354 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
7355 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7356 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7357 bool ReturnDevicePointer = false;
7358 bool IsImplicit = false;
7359 const ValueDecl *Mapper = nullptr;
7360 const Expr *VarRef = nullptr;
7361 bool ForDeviceAddr = false;
7362 bool HasUdpFbNullify = false;
7363
7364 MapInfo() = default;
7365 MapInfo(
7366 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7367 OpenMPMapClauseKind MapType,
7368 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7369 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7370 bool ReturnDevicePointer, bool IsImplicit,
7371 const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr,
7372 bool ForDeviceAddr = false, bool HasUdpFbNullify = false)
7373 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7374 MotionModifiers(MotionModifiers),
7375 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7376 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr),
7377 HasUdpFbNullify(HasUdpFbNullify) {}
7378 };
7379
7380 /// The target directive from where the mappable clauses were extracted. It
7381 /// is either a executable directive or a user-defined mapper directive.
7382 llvm::PointerUnion<const OMPExecutableDirective *,
7383 const OMPDeclareMapperDecl *>
7384 CurDir;
7385
7386 /// Function the directive is being generated for.
7387 CodeGenFunction &CGF;
7388
7389 /// Set of all first private variables in the current directive.
7390 /// bool data is set to true if the variable is implicitly marked as
7391 /// firstprivate, false otherwise.
7392 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7393
7394 /// Set of defaultmap clause kinds that use firstprivate behavior.
7395 llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
7396
7397 /// Map between device pointer declarations and their expression components.
7398 /// The key value for declarations in 'this' is null.
7399 llvm::DenseMap<
7400 const ValueDecl *,
7401 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7402 DevPointersMap;
7403
7404 /// Map between device addr declarations and their expression components.
7405 /// The key value for declarations in 'this' is null.
7406 llvm::DenseMap<
7407 const ValueDecl *,
7408 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7409 HasDevAddrsMap;
7410
7411 /// Map between lambda declarations and their map type.
7412 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7413
7414 /// Map from component lists to their attach pointer expressions.
7415 llvm::DenseMap<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7416 const Expr *>
7417 AttachPtrExprMap;
7418
7419 /// Map from attach pointer expressions to their component depth.
7420 /// nullptr key has std::nullopt depth. This can be used to order attach-ptr
7421 /// expressions with increasing/decreasing depth.
7422 /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`.
7423 /// TODO: Not urgent, but we should ideally use the number of pointer
7424 /// dereferences in an expr as an indicator of its complexity, instead of the
7425 /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`,
7426 /// `*(p + 5 + 5)` together.
7427 llvm::DenseMap<const Expr *, std::optional<size_t>>
7428 AttachPtrComponentDepthMap = {{nullptr, std::nullopt}};
7429
7430 /// Map from attach pointer expressions to the order they were computed in, in
7431 /// collectAttachPtrExprInfo().
7432 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7433 {nullptr, 0}};
7434
7435 /// An instance of attach-ptr-expr comparator that can be used throughout the
7436 /// lifetime of this handler.
7437 AttachPtrExprComparator AttachPtrComparator;
7438
7439 llvm::Value *getExprTypeSize(const Expr *E) const {
7440 QualType ExprTy = E->getType().getCanonicalType();
7441
7442 // Calculate the size for array shaping expression.
7443 if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(Val: E)) {
7444 llvm::Value *Size =
7445 CGF.getTypeSize(Ty: OAE->getBase()->getType()->getPointeeType());
7446 for (const Expr *SE : OAE->getDimensions()) {
7447 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
7448 Sz = CGF.EmitScalarConversion(Src: Sz, SrcTy: SE->getType(),
7449 DstTy: CGF.getContext().getSizeType(),
7450 Loc: SE->getExprLoc());
7451 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: Sz);
7452 }
7453 return Size;
7454 }
7455
7456 // Reference types are ignored for mapping purposes.
7457 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7458 ExprTy = RefTy->getPointeeType().getCanonicalType();
7459
7460 // Given that an array section is considered a built-in type, we need to
7461 // do the calculation based on the length of the section instead of relying
7462 // on CGF.getTypeSize(E->getType()).
7463 if (const auto *OAE = dyn_cast<ArraySectionExpr>(Val: E)) {
7464 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(
7465 Base: OAE->getBase()->IgnoreParenImpCasts())
7466 .getCanonicalType();
7467
7468 // If there is no length associated with the expression and lower bound is
7469 // not specified too, that means we are using the whole length of the
7470 // base.
7471 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7472 !OAE->getLowerBound())
7473 return CGF.getTypeSize(Ty: BaseTy);
7474
7475 llvm::Value *ElemSize;
7476 if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7477 ElemSize = CGF.getTypeSize(Ty: PTy->getPointeeType().getCanonicalType());
7478 } else {
7479 const auto *ATy = cast<ArrayType>(Val: BaseTy.getTypePtr());
7480 assert(ATy && "Expecting array type if not a pointer type.");
7481 ElemSize = CGF.getTypeSize(Ty: ATy->getElementType().getCanonicalType());
7482 }
7483
7484 // If we don't have a length at this point, that is because we have an
7485 // array section with a single element.
7486 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7487 return ElemSize;
7488
7489 if (const Expr *LenExpr = OAE->getLength()) {
7490 llvm::Value *LengthVal = CGF.EmitScalarExpr(E: LenExpr);
7491 LengthVal = CGF.EmitScalarConversion(Src: LengthVal, SrcTy: LenExpr->getType(),
7492 DstTy: CGF.getContext().getSizeType(),
7493 Loc: LenExpr->getExprLoc());
7494 return CGF.Builder.CreateNUWMul(LHS: LengthVal, RHS: ElemSize);
7495 }
7496 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7497 OAE->getLowerBound() && "expected array_section[lb:].");
7498 // Size = sizetype - lb * elemtype;
7499 llvm::Value *LengthVal = CGF.getTypeSize(Ty: BaseTy);
7500 llvm::Value *LBVal = CGF.EmitScalarExpr(E: OAE->getLowerBound());
7501 LBVal = CGF.EmitScalarConversion(Src: LBVal, SrcTy: OAE->getLowerBound()->getType(),
7502 DstTy: CGF.getContext().getSizeType(),
7503 Loc: OAE->getLowerBound()->getExprLoc());
7504 LBVal = CGF.Builder.CreateNUWMul(LHS: LBVal, RHS: ElemSize);
7505 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LHS: LengthVal, RHS: LBVal);
7506 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LHS: LengthVal, RHS: LBVal);
7507 LengthVal = CGF.Builder.CreateSelect(
7508 C: Cmp, True: TrueVal, False: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0));
7509 return LengthVal;
7510 }
7511 return CGF.getTypeSize(Ty: ExprTy);
7512 }
7513
7514 /// Return the corresponding bits for a given map clause modifier. Add
7515 /// a flag marking the map as a pointer if requested. Add a flag marking the
7516 /// map as the first one of a series of maps that relate to the same map
7517 /// expression.
7518 OpenMPOffloadMappingFlags getMapTypeBits(
7519 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7520 ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,
7521 bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const {
7522 OpenMPOffloadMappingFlags Bits =
7523 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7524 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7525 switch (MapType) {
7526 case OMPC_MAP_alloc:
7527 case OMPC_MAP_release:
7528 // alloc and release is the default behavior in the runtime library, i.e.
7529 // if we don't pass any bits alloc/release that is what the runtime is
7530 // going to do. Therefore, we don't need to signal anything for these two
7531 // type modifiers.
7532 break;
7533 case OMPC_MAP_to:
7534 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7535 break;
7536 case OMPC_MAP_from:
7537 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7538 break;
7539 case OMPC_MAP_tofrom:
7540 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7541 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7542 break;
7543 case OMPC_MAP_delete:
7544 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7545 break;
7546 case OMPC_MAP_unknown:
7547 llvm_unreachable("Unexpected map type!");
7548 }
7549 if (AddPtrFlag)
7550 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7551 if (AddIsTargetParamFlag)
7552 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7553 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_always))
7554 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7555 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_close))
7556 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7557 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_present) ||
7558 llvm::is_contained(Range&: MotionModifiers, Element: OMPC_MOTION_MODIFIER_present))
7559 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7560 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_ompx_hold))
7561 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7562 if (IsNonContiguous)
7563 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7564 return Bits;
7565 }
7566
7567 /// Return true if the provided expression is a final array section. A
7568 /// final array section, is one whose length can't be proved to be one.
7569 bool isFinalArraySectionExpression(const Expr *E) const {
7570 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E);
7571
7572 // It is not an array section and therefore not a unity-size one.
7573 if (!OASE)
7574 return false;
7575
7576 // An array section with no colon always refer to a single element.
7577 if (OASE->getColonLocFirst().isInvalid())
7578 return false;
7579
7580 const Expr *Length = OASE->getLength();
7581
7582 // If we don't have a length we have to check if the array has size 1
7583 // for this dimension. Also, we should always expect a length if the
7584 // base type is pointer.
7585 if (!Length) {
7586 QualType BaseQTy = ArraySectionExpr::getBaseOriginalType(
7587 Base: OASE->getBase()->IgnoreParenImpCasts())
7588 .getCanonicalType();
7589 if (const auto *ATy = dyn_cast<ConstantArrayType>(Val: BaseQTy.getTypePtr()))
7590 return ATy->getSExtSize() != 1;
7591 // If we don't have a constant dimension length, we have to consider
7592 // the current section as having any size, so it is not necessarily
7593 // unitary. If it happen to be unity size, that's user fault.
7594 return true;
7595 }
7596
7597 // Check if the length evaluates to 1.
7598 Expr::EvalResult Result;
7599 if (!Length->EvaluateAsInt(Result, Ctx: CGF.getContext()))
7600 return true; // Can have more that size 1.
7601
7602 llvm::APSInt ConstLength = Result.Val.getInt();
7603 return ConstLength.getSExtValue() != 1;
7604 }
7605
7606 /// Emit an attach entry into \p CombinedInfo, using the information from \p
7607 /// AttachInfo. For example, for a map of form `int *p; ... map(p[1:10])`,
7608 /// an attach entry has the following form:
7609 /// &p, &p[1], sizeof(void*), ATTACH
7610 void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo,
7611 const AttachInfoTy &AttachInfo) const {
7612 assert(AttachInfo.isValid() &&
7613 "Expected valid attach pointer/pointee information!");
7614
7615 // Size is the size of the pointer itself - use pointer size, not BaseDecl
7616 // size
7617 llvm::Value *PointerSize = CGF.Builder.CreateIntCast(
7618 V: llvm::ConstantInt::get(
7619 Ty: CGF.CGM.SizeTy, V: CGF.getContext()
7620 .getTypeSizeInChars(T: CGF.getContext().VoidPtrTy)
7621 .getQuantity()),
7622 DestTy: CGF.Int64Ty, /*isSigned=*/true);
7623
7624 CombinedInfo.Exprs.emplace_back(Args: AttachInfo.AttachPtrDecl,
7625 Args: AttachInfo.AttachMapExpr);
7626 CombinedInfo.BasePointers.push_back(
7627 Elt: AttachInfo.AttachPtrAddr.emitRawPointer(CGF));
7628 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7629 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7630 CombinedInfo.Pointers.push_back(
7631 Elt: AttachInfo.AttachPteeAddr.emitRawPointer(CGF));
7632 CombinedInfo.Sizes.push_back(Elt: PointerSize);
7633 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
7634 // ATTACH entries themselves don't "have" a base attach-ptr.
7635 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7636 CombinedInfo.Mappers.push_back(Elt: nullptr);
7637 CombinedInfo.NonContigInfo.Dims.push_back(Elt: 1);
7638 }
7639
7640 /// A helper class to copy structures with overlapped elements, i.e. those
7641 /// which have mappings of both "s" and "s.mem". Consecutive elements that
7642 /// are not explicitly copied have mapping nodes synthesized for them,
7643 /// taking care to avoid generating zero-sized copies.
7644 class CopyOverlappedEntryGaps {
7645 CodeGenFunction &CGF;
7646 MapCombinedInfoTy &CombinedInfo;
7647 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7648 const ValueDecl *MapDecl = nullptr;
7649 const Expr *MapExpr = nullptr;
7650 Address BP = Address::invalid();
7651 bool IsNonContiguous = false;
7652 uint64_t DimSize = 0;
7653 // These elements track the position as the struct is iterated over
7654 // (in order of increasing element address).
7655 const RecordDecl *LastParent = nullptr;
7656 uint64_t Cursor = 0;
7657 unsigned LastIndex = -1u;
7658 Address LB = Address::invalid();
7659
7660 public:
7661 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7662 MapCombinedInfoTy &CombinedInfo,
7663 OpenMPOffloadMappingFlags Flags,
7664 const ValueDecl *MapDecl, const Expr *MapExpr,
7665 Address BP, Address LB, bool IsNonContiguous,
7666 uint64_t DimSize)
7667 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7668 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7669 DimSize(DimSize), LB(LB) {}
7670
7671 void processField(
7672 const OMPClauseMappableExprCommon::MappableComponent &MC,
7673 const FieldDecl *FD,
7674 llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)>
7675 EmitMemberExprBase) {
7676 const RecordDecl *RD = FD->getParent();
7677 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(D: RD);
7678 uint64_t FieldOffset = RL.getFieldOffset(FieldNo: FD->getFieldIndex());
7679 uint64_t FieldSize =
7680 CGF.getContext().getTypeSize(T: FD->getType().getCanonicalType());
7681 Address ComponentLB = Address::invalid();
7682
7683 if (FD->getType()->isLValueReferenceType()) {
7684 const auto *ME = cast<MemberExpr>(Val: MC.getAssociatedExpression());
7685 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7686 ComponentLB =
7687 CGF.EmitLValueForFieldInitialization(Base: BaseLVal, Field: FD).getAddress();
7688 } else {
7689 ComponentLB =
7690 CGF.EmitOMPSharedLValue(E: MC.getAssociatedExpression()).getAddress();
7691 }
7692
7693 if (!LastParent)
7694 LastParent = RD;
7695 if (FD->getParent() == LastParent) {
7696 if (FD->getFieldIndex() != LastIndex + 1)
7697 copyUntilField(FD, ComponentLB);
7698 } else {
7699 LastParent = FD->getParent();
7700 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7701 copyUntilField(FD, ComponentLB);
7702 }
7703 Cursor = FieldOffset + FieldSize;
7704 LastIndex = FD->getFieldIndex();
7705 LB = CGF.Builder.CreateConstGEP(Addr: ComponentLB, Index: 1);
7706 }
7707
7708 void copyUntilField(const FieldDecl *FD, Address ComponentLB) {
7709 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7710 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7711 llvm::Value *Size = CGF.Builder.CreatePtrDiff(LHS: ComponentLBPtr, RHS: LBPtr);
7712 copySizedChunk(Base: LBPtr, Size);
7713 }
7714
7715 void copyUntilEnd(Address HB) {
7716 if (LastParent) {
7717 const ASTRecordLayout &RL =
7718 CGF.getContext().getASTRecordLayout(D: LastParent);
7719 if ((uint64_t)CGF.getContext().toBits(CharSize: RL.getSize()) <= Cursor)
7720 return;
7721 }
7722 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7723 llvm::Value *Size = CGF.Builder.CreatePtrDiff(
7724 LHS: CGF.Builder.CreateConstGEP(Addr: HB, Index: 1).emitRawPointer(CGF), RHS: LBPtr);
7725 copySizedChunk(Base: LBPtr, Size);
7726 }
7727
7728 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7729 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
7730 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
7731 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7732 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7733 CombinedInfo.Pointers.push_back(Elt: Base);
7734 CombinedInfo.Sizes.push_back(
7735 Elt: CGF.Builder.CreateIntCast(V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/false));
7736 CombinedInfo.Types.push_back(Elt: Flags);
7737 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7738 CombinedInfo.Mappers.push_back(Elt: nullptr);
7739 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize : 1);
7740 }
7741 };
7742
7743 /// Generate the base pointers, section pointers, sizes, map type bits, and
7744 /// user-defined mappers (all included in \a CombinedInfo) for the provided
7745 /// map type, map or motion modifiers, and expression components.
7746 /// \a IsFirstComponent should be set to true if the provided set of
7747 /// components is the first associated with a capture.
7748 void generateInfoForComponentList(
7749 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7750 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7751 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7752 MapCombinedInfoTy &CombinedInfo,
7753 MapCombinedInfoTy &StructBaseCombinedInfo,
7754 StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo,
7755 bool IsFirstComponentList, bool IsImplicit,
7756 bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr,
7757 bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr,
7758 const Expr *MapExpr = nullptr,
7759 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7760 OverlappedElements = {}) const {
7761
7762 // The following summarizes what has to be generated for each map and the
7763 // types below. The generated information is expressed in this order:
7764 // base pointer, section pointer, size, flags
7765 // (to add to the ones that come from the map type and modifier).
7766 // Entries annotated with (+) are only generated for "target" constructs,
7767 // and only if the variable at the beginning of the expression is used in
7768 // the region.
7769 //
7770 // double d;
7771 // int i[100];
7772 // float *p;
7773 // int **a = &i;
7774 //
7775 // struct S1 {
7776 // int i;
7777 // float f[50];
7778 // }
7779 // struct S2 {
7780 // int i;
7781 // float f[50];
7782 // S1 s;
7783 // double *p;
7784 // double *&pref;
7785 // struct S2 *ps;
7786 // int &ref;
7787 // }
7788 // S2 s;
7789 // S2 *ps;
7790 //
7791 // map(d)
7792 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
7793 //
7794 // map(i)
7795 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
7796 //
7797 // map(i[1:23])
7798 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
7799 //
7800 // map(p)
7801 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7802 //
7803 // map(p[1:24])
7804 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM // map pointee
7805 // &p, &p[1], sizeof(void*), ATTACH // attach pointer/pointee, if both
7806 // // are present, and either is new
7807 //
7808 // map(([22])p)
7809 // p, p, 22*sizeof(float), TARGET_PARAM | TO | FROM
7810 // &p, p, sizeof(void*), ATTACH
7811 //
7812 // map((*a)[0:3])
7813 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
7814 // (*a)[0], &(*a)[0], 3 * sizeof(int), TO | FROM
7815 // &(*a), &(*a)[0], sizeof(void*), ATTACH
7816 // (+) Only on target, if a is used in the region
7817 // Note: Since the attach base-pointer is `*a`, which is not a scalar
7818 // variable, it doesn't determine the clause on `a`. `a` is mapped using
7819 // a zero-length-array-section map by generateDefaultMapInfo, if it is
7820 // referenced in the target region, because it is a pointer.
7821 //
7822 // map(**a)
7823 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
7824 // &(*a)[0], &(*a)[0], sizeof(int), TO | FROM
7825 // &(*a), &(*a)[0], sizeof(void*), ATTACH
7826 // (+) Only on target, if a is used in the region
7827 //
7828 // map(s)
7829 // FIXME: This needs to also imply map(ref_ptr_ptee: s.ref), since the
7830 // effect is supposed to be same as if the user had a map for every element
7831 // of the struct. We currently do a shallow-map of s.
7832 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
7833 //
7834 // map(s.i)
7835 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
7836 //
7837 // map(s.s.f)
7838 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7839 //
7840 // map(s.p)
7841 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
7842 //
7843 // map(to: s.p[:22])
7844 // &s, &(s.p), sizeof(double*), TARGET_PARAM | IMPLICIT // (+)
7845 // &(s.p[0]), &(s.p[0]), 22 * sizeof(double*), TO | FROM
7846 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7847 //
7848 // map(to: s.ref)
7849 // &s, &(ptr(s.ref)), sizeof(int*), TARGET_PARAM (*)
7850 // &s, &(ptee(s.ref)), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
7851 // (*) alloc space for struct members, only this is a target parameter.
7852 // (**) map the pointer (nothing to be mapped in this example) (the compiler
7853 // optimizes this entry out, same in the examples below)
7854 // (***) map the pointee (map: to)
7855 // Note: ptr(s.ref) represents the referring pointer of s.ref
7856 // ptee(s.ref) represents the referenced pointee of s.ref
7857 //
7858 // map(to: s.pref)
7859 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM
7860 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO
7861 //
7862 // map(to: s.pref[:22])
7863 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | IMPLICIT // (+)
7864 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO |
7865 // FROM | IMPLICIT // (+)
7866 // &(ptee(s.pref)[0]), &(ptee(s.pref)[0]), 22 * sizeof(double), TO
7867 // &(ptee(s.pref)), &(ptee(s.pref)[0]), sizeof(void*), ATTACH
7868 //
7869 // map(s.ps)
7870 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7871 //
7872 // map(from: s.ps->s.i)
7873 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7874 // &(s.ps[0]), &(s.ps->s.i), sizeof(int), FROM
7875 // &(s.ps), &(s.ps->s.i), sizeof(void*), ATTACH
7876 //
7877 // map(to: s.ps->ps)
7878 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7879 // &(s.ps[0]), &(s.ps->ps), sizeof(S2*), TO
7880 // &(s.ps), &(s.ps->ps), sizeof(void*), ATTACH
7881 //
7882 // map(s.ps->ps->ps)
7883 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7884 // &(s.ps->ps[0]), &(s.ps->ps->ps), sizeof(S2*), TO
7885 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(void*), ATTACH
7886 //
7887 // map(to: s.ps->ps->s.f[:22])
7888 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7889 // &(s.ps->ps[0]), &(s.ps->ps->s.f[0]), 22*sizeof(float), TO
7890 // &(s.ps->ps), &(s.ps->ps->s.f[0]), sizeof(void*), ATTACH
7891 //
7892 // map(ps)
7893 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
7894 //
7895 // map(ps->i)
7896 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
7897 // &ps, &(ps->i), sizeof(void*), ATTACH
7898 //
7899 // map(ps->s.f)
7900 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7901 // &ps, &(ps->s.f[0]), sizeof(ps), ATTACH
7902 //
7903 // map(from: ps->p)
7904 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
7905 // &ps, &(ps->p), sizeof(ps), ATTACH
7906 //
7907 // map(to: ps->p[:22])
7908 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
7909 // &(ps->p[0]), &(ps->p[0]), 22*sizeof(double), TO
7910 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7911 //
7912 // map(ps->ps)
7913 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7914 // &ps, &(ps->ps), sizeof(ps), ATTACH
7915 //
7916 // map(from: ps->ps->s.i)
7917 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
7918 // &(ps->ps[0]), &(ps->ps->s.i), sizeof(int), FROM
7919 // &(ps->ps), &(ps->ps->s.i), sizeof(void*), ATTACH
7920 //
7921 // map(from: ps->ps->ps)
7922 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7923 // &(ps->ps[0]), &(ps->ps->ps), sizeof(S2*), FROM
7924 // &(ps->ps), &(ps->ps->ps), sizeof(void*), ATTACH
7925 //
7926 // map(ps->ps->ps->ps)
7927 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7928 // &(ps->ps->ps[0]), &(ps->ps->ps->ps), sizeof(S2*), FROM
7929 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(void*), ATTACH
7930 //
7931 // map(to: ps->ps->ps->s.f[:22])
7932 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7933 // &(ps->ps->ps[0]), &(ps->ps->ps->s.f[0]), 22*sizeof(float), TO
7934 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), sizeof(void*), ATTACH
7935 //
7936 // map(to: s.f[:22]) map(from: s.p[:33])
7937 // On target, and if s is used in the region:
7938 //
7939 // &s, &(s.f[0]), 50*sizeof(float) +
7940 // sizeof(struct S1) +
7941 // sizeof(double*) (**), TARGET_PARAM
7942 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
7943 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) | TO |
7944 // FROM | IMPLICIT
7945 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
7946 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7947 // (**) allocate contiguous space needed to fit all mapped members even if
7948 // we allocate space for members not mapped (in this example,
7949 // s.f[22..49] and s.s are not mapped, yet we must allocate space for
7950 // them as well because they fall between &s.f[0] and &s.p)
7951 //
7952 // On other constructs, and, if s is not used in the region, on target:
7953 // &s, &(s.f[0]), 22*sizeof(float), TO
7954 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
7955 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7956 //
7957 // map(from: s.f[:22]) map(to: ps->p[:33])
7958 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
7959 // &ps[0], &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7960 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
7961 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7962 //
7963 // map(from: s.f[:22], s.s) map(to: ps->p[:33])
7964 // &s, &(s.f[0]), 50*sizeof(float) +
7965 // sizeof(struct S1), TARGET_PARAM
7966 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
7967 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
7968 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7969 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
7970 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7971 //
7972 // map(p[:100], p)
7973 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7974 // p, &p[0], 100*sizeof(float), TO | FROM
7975 // &p, &p[0], sizeof(float*), ATTACH
7976
7977 // Track if the map information being generated is the first for a capture.
7978 bool IsCaptureFirstInfo = IsFirstComponentList;
7979 // When the variable is on a declare target link or in a to clause with
7980 // unified memory, a reference is needed to hold the host/device address
7981 // of the variable.
7982 bool RequiresReference = false;
7983
7984 // Scan the components from the base to the complete expression.
7985 auto CI = Components.rbegin();
7986 auto CE = Components.rend();
7987 auto I = CI;
7988
7989 // Track if the map information being generated is the first for a list of
7990 // components.
7991 bool IsExpressionFirstInfo = true;
7992 bool FirstPointerInComplexData = false;
7993 Address BP = Address::invalid();
7994 Address FinalLowestElem = Address::invalid();
7995 const Expr *AssocExpr = I->getAssociatedExpression();
7996 const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr);
7997 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
7998 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(Val: AssocExpr);
7999
8000 // Get the pointer-attachment base-pointer for the given list, if any.
8001 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
8002 auto [AttachPtrAddr, AttachPteeBaseAddr] =
8003 getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF);
8004
8005 bool HasAttachPtr = AttachPtrExpr != nullptr;
8006 bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr;
8007 bool SeenAttachPtr = FirstComponentIsForAttachPtr;
8008
8009 if (FirstComponentIsForAttachPtr) {
8010 // No need to process AttachPtr here. It will be processed at the end
8011 // after we have computed the pointee's address.
8012 ++I;
8013 } else if (isa<MemberExpr>(Val: AssocExpr)) {
8014 // The base is the 'this' pointer. The content of the pointer is going
8015 // to be the base of the field being mapped.
8016 BP = CGF.LoadCXXThisAddress();
8017 } else if ((AE && isa<CXXThisExpr>(Val: AE->getBase()->IgnoreParenImpCasts())) ||
8018 (OASE &&
8019 isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))) {
8020 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8021 } else if (OAShE &&
8022 isa<CXXThisExpr>(Val: OAShE->getBase()->IgnoreParenCasts())) {
8023 BP = Address(
8024 CGF.EmitScalarExpr(E: OAShE->getBase()),
8025 CGF.ConvertTypeForMem(T: OAShE->getBase()->getType()->getPointeeType()),
8026 CGF.getContext().getTypeAlignInChars(T: OAShE->getBase()->getType()));
8027 } else {
8028 // The base is the reference to the variable.
8029 // BP = &Var.
8030 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8031 if (const auto *VD =
8032 dyn_cast_or_null<VarDecl>(Val: I->getAssociatedDeclaration())) {
8033 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8034 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
8035 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
8036 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
8037 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
8038 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
8039 RequiresReference = true;
8040 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
8041 }
8042 }
8043 }
8044
8045 // If the variable is a pointer and is being dereferenced (i.e. is not
8046 // the last component), the base has to be the pointer itself, not its
8047 // reference. References are ignored for mapping purposes.
8048 QualType Ty =
8049 I->getAssociatedDeclaration()->getType().getNonReferenceType();
8050 if (Ty->isAnyPointerType() && std::next(x: I) != CE) {
8051 // No need to generate individual map information for the pointer, it
8052 // can be associated with the combined storage if shared memory mode is
8053 // active or the base declaration is not global variable.
8054 const auto *VD = dyn_cast<VarDecl>(Val: I->getAssociatedDeclaration());
8055 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8056 !VD || VD->hasLocalStorage() || HasAttachPtr)
8057 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8058 else
8059 FirstPointerInComplexData = true;
8060 ++I;
8061 }
8062 }
8063
8064 // Track whether a component of the list should be marked as MEMBER_OF some
8065 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
8066 // in a component list should be marked as MEMBER_OF, all subsequent entries
8067 // do not belong to the base struct. E.g.
8068 // struct S2 s;
8069 // s.ps->ps->ps->f[:]
8070 // (1) (2) (3) (4)
8071 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
8072 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
8073 // is the pointee of ps(2) which is not member of struct s, so it should not
8074 // be marked as such (it is still PTR_AND_OBJ).
8075 // The variable is initialized to false so that PTR_AND_OBJ entries which
8076 // are not struct members are not considered (e.g. array of pointers to
8077 // data).
8078 bool ShouldBeMemberOf = false;
8079
8080 // Variable keeping track of whether or not we have encountered a component
8081 // in the component list which is a member expression. Useful when we have a
8082 // pointer or a final array section, in which case it is the previous
8083 // component in the list which tells us whether we have a member expression.
8084 // E.g. X.f[:]
8085 // While processing the final array section "[:]" it is "f" which tells us
8086 // whether we are dealing with a member of a declared struct.
8087 const MemberExpr *EncounteredME = nullptr;
8088
8089 // Track for the total number of dimension. Start from one for the dummy
8090 // dimension.
8091 uint64_t DimSize = 1;
8092
8093 // Detects non-contiguous updates due to strided accesses.
8094 // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set
8095 // correctly when generating information to be passed to the runtime. The
8096 // flag is set to true if any array section has a stride not equal to 1, or
8097 // if the stride is not a constant expression (conservatively assumed
8098 // non-contiguous).
8099 bool IsNonContiguous =
8100 CombinedInfo.NonContigInfo.IsNonContiguous ||
8101 any_of(Range&: Components, P: [&](const auto &Component) {
8102 const auto *OASE =
8103 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
8104 if (!OASE)
8105 return false;
8106
8107 const Expr *StrideExpr = OASE->getStride();
8108 if (!StrideExpr)
8109 return false;
8110
8111 assert(StrideExpr->getType()->isIntegerType() &&
8112 "Stride expression must be of integer type");
8113
8114 // If stride is not evaluatable as a constant, treat as
8115 // non-contiguous.
8116 const auto Constant =
8117 StrideExpr->getIntegerConstantExpr(Ctx: CGF.getContext());
8118 if (!Constant)
8119 return true;
8120
8121 // Treat non-unitary strides as non-contiguous.
8122 return !Constant->isOne();
8123 });
8124
8125 bool IsPrevMemberReference = false;
8126
8127 bool IsPartialMapped =
8128 !PartialStruct.PreliminaryMapData.BasePointers.empty();
8129
8130 // We need to check if we will be encountering any MEs. If we do not
8131 // encounter any ME expression it means we will be mapping the whole struct.
8132 // In that case we need to skip adding an entry for the struct to the
8133 // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo
8134 // list only when generating all info for clauses.
8135 bool IsMappingWholeStruct = true;
8136 if (!GenerateAllInfoForClauses) {
8137 IsMappingWholeStruct = false;
8138 } else {
8139 for (auto TempI = I; TempI != CE; ++TempI) {
8140 const MemberExpr *PossibleME =
8141 dyn_cast<MemberExpr>(Val: TempI->getAssociatedExpression());
8142 if (PossibleME) {
8143 IsMappingWholeStruct = false;
8144 break;
8145 }
8146 }
8147 }
8148
8149 bool SeenFirstNonBinOpExprAfterAttachPtr = false;
8150 for (; I != CE; ++I) {
8151 // If we have a valid attach-ptr, we skip processing all components until
8152 // after the attach-ptr.
8153 if (HasAttachPtr && !SeenAttachPtr) {
8154 SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr;
8155 continue;
8156 }
8157
8158 // After finding the attach pointer, skip binary-ops, to skip past
8159 // expressions like (p + 10), for a map like map(*(p + 10)), where p is
8160 // the attach-ptr.
8161 if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) {
8162 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8163 if (BO)
8164 continue;
8165
8166 // Found the first non-binary-operator component after attach
8167 SeenFirstNonBinOpExprAfterAttachPtr = true;
8168 BP = AttachPteeBaseAddr;
8169 }
8170
8171 // If the current component is member of a struct (parent struct) mark it.
8172 if (!EncounteredME) {
8173 EncounteredME = dyn_cast<MemberExpr>(Val: I->getAssociatedExpression());
8174 // If we encounter a PTR_AND_OBJ entry from now on it should be marked
8175 // as MEMBER_OF the parent struct.
8176 if (EncounteredME) {
8177 ShouldBeMemberOf = true;
8178 // Do not emit as complex pointer if this is actually not array-like
8179 // expression.
8180 if (FirstPointerInComplexData) {
8181 QualType Ty = std::prev(x: I)
8182 ->getAssociatedDeclaration()
8183 ->getType()
8184 .getNonReferenceType();
8185 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8186 FirstPointerInComplexData = false;
8187 }
8188 }
8189 }
8190
8191 auto Next = std::next(x: I);
8192
8193 // We need to generate the addresses and sizes if this is the last
8194 // component, if the component is a pointer or if it is an array section
8195 // whose length can't be proved to be one. If this is a pointer, it
8196 // becomes the base address for the following components.
8197
8198 // A final array section, is one whose length can't be proved to be one.
8199 // If the map item is non-contiguous then we don't treat any array section
8200 // as final array section.
8201 bool IsFinalArraySection =
8202 !IsNonContiguous &&
8203 isFinalArraySectionExpression(E: I->getAssociatedExpression());
8204
8205 // If we have a declaration for the mapping use that, otherwise use
8206 // the base declaration of the map clause.
8207 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
8208 ? I->getAssociatedDeclaration()
8209 : BaseDecl;
8210 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
8211 : MapExpr;
8212
8213 // Get information on whether the element is a pointer. Have to do a
8214 // special treatment for array sections given that they are built-in
8215 // types.
8216 const auto *OASE =
8217 dyn_cast<ArraySectionExpr>(Val: I->getAssociatedExpression());
8218 const auto *OAShE =
8219 dyn_cast<OMPArrayShapingExpr>(Val: I->getAssociatedExpression());
8220 const auto *UO = dyn_cast<UnaryOperator>(Val: I->getAssociatedExpression());
8221 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8222 bool IsPointer =
8223 OAShE ||
8224 (OASE && ArraySectionExpr::getBaseOriginalType(Base: OASE)
8225 .getCanonicalType()
8226 ->isAnyPointerType()) ||
8227 I->getAssociatedExpression()->getType()->isAnyPointerType();
8228 bool IsMemberReference = isa<MemberExpr>(Val: I->getAssociatedExpression()) &&
8229 MapDecl &&
8230 MapDecl->getType()->isLValueReferenceType();
8231 bool IsNonDerefPointer = IsPointer &&
8232 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
8233 !IsNonContiguous;
8234
8235 if (OASE)
8236 ++DimSize;
8237
8238 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
8239 IsFinalArraySection) {
8240 // If this is not the last component, we expect the pointer to be
8241 // associated with an array expression or member expression.
8242 assert((Next == CE ||
8243 isa<MemberExpr>(Next->getAssociatedExpression()) ||
8244 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
8245 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
8246 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
8247 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
8248 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
8249 "Unexpected expression");
8250
8251 Address LB = Address::invalid();
8252 Address LowestElem = Address::invalid();
8253 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
8254 const MemberExpr *E) {
8255 const Expr *BaseExpr = E->getBase();
8256 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a
8257 // scalar.
8258 LValue BaseLV;
8259 if (E->isArrow()) {
8260 LValueBaseInfo BaseInfo;
8261 TBAAAccessInfo TBAAInfo;
8262 Address Addr =
8263 CGF.EmitPointerWithAlignment(Addr: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
8264 QualType PtrTy = BaseExpr->getType()->getPointeeType();
8265 BaseLV = CGF.MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
8266 } else {
8267 BaseLV = CGF.EmitOMPSharedLValue(E: BaseExpr);
8268 }
8269 return BaseLV;
8270 };
8271 if (OAShE) {
8272 LowestElem = LB =
8273 Address(CGF.EmitScalarExpr(E: OAShE->getBase()),
8274 CGF.ConvertTypeForMem(
8275 T: OAShE->getBase()->getType()->getPointeeType()),
8276 CGF.getContext().getTypeAlignInChars(
8277 T: OAShE->getBase()->getType()));
8278 } else if (IsMemberReference) {
8279 const auto *ME = cast<MemberExpr>(Val: I->getAssociatedExpression());
8280 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
8281 LowestElem = CGF.EmitLValueForFieldInitialization(
8282 Base: BaseLVal, Field: cast<FieldDecl>(Val: MapDecl))
8283 .getAddress();
8284 LB = CGF.EmitLoadOfReferenceLValue(RefAddr: LowestElem, RefTy: MapDecl->getType())
8285 .getAddress();
8286 } else {
8287 LowestElem = LB =
8288 CGF.EmitOMPSharedLValue(E: I->getAssociatedExpression())
8289 .getAddress();
8290 }
8291
8292 // Save the final LowestElem, to use it as the pointee in attach maps,
8293 // if emitted.
8294 if (Next == CE)
8295 FinalLowestElem = LowestElem;
8296
8297 // If this component is a pointer inside the base struct then we don't
8298 // need to create any entry for it - it will be combined with the object
8299 // it is pointing to into a single PTR_AND_OBJ entry.
8300 bool IsMemberPointerOrAddr =
8301 EncounteredME &&
8302 (((IsPointer || ForDeviceAddr) &&
8303 I->getAssociatedExpression() == EncounteredME) ||
8304 (IsPrevMemberReference && !IsPointer) ||
8305 (IsMemberReference && Next != CE &&
8306 !Next->getAssociatedExpression()->getType()->isPointerType()));
8307 if (!OverlappedElements.empty() && Next == CE) {
8308 // Handle base element with the info for overlapped elements.
8309 assert(!PartialStruct.Base.isValid() && "The base element is set.");
8310 assert(!IsPointer &&
8311 "Unexpected base element with the pointer type.");
8312 // Mark the whole struct as the struct that requires allocation on the
8313 // device.
8314 PartialStruct.LowestElem = {0, LowestElem};
8315 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
8316 T: I->getAssociatedExpression()->getType());
8317 Address HB = CGF.Builder.CreateConstGEP(
8318 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8319 Addr: LowestElem, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty),
8320 Index: TypeSize.getQuantity() - 1);
8321 PartialStruct.HighestElem = {
8322 std::numeric_limits<decltype(
8323 PartialStruct.HighestElem.first)>::max(),
8324 HB};
8325 PartialStruct.Base = BP;
8326 PartialStruct.LB = LB;
8327 assert(
8328 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8329 "Overlapped elements must be used only once for the variable.");
8330 std::swap(a&: PartialStruct.PreliminaryMapData, b&: CombinedInfo);
8331 // Emit data for non-overlapped data.
8332 OpenMPOffloadMappingFlags Flags =
8333 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8334 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8335 /*AddPtrFlag=*/false,
8336 /*AddIsTargetParamFlag=*/false, IsNonContiguous);
8337 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8338 MapExpr, BP, LB, IsNonContiguous,
8339 DimSize);
8340 // Do bitcopy of all non-overlapped structure elements.
8341 for (OMPClauseMappableExprCommon::MappableExprComponentListRef
8342 Component : OverlappedElements) {
8343 for (const OMPClauseMappableExprCommon::MappableComponent &MC :
8344 Component) {
8345 if (const ValueDecl *VD = MC.getAssociatedDeclaration()) {
8346 if (const auto *FD = dyn_cast<FieldDecl>(Val: VD)) {
8347 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8348 }
8349 }
8350 }
8351 }
8352 CopyGaps.copyUntilEnd(HB);
8353 break;
8354 }
8355 llvm::Value *Size = getExprTypeSize(E: I->getAssociatedExpression());
8356 // Skip adding an entry in the CurInfo of this combined entry if the
8357 // whole struct is currently being mapped. The struct needs to be added
8358 // in the first position before any data internal to the struct is being
8359 // mapped.
8360 // Skip adding an entry in the CurInfo of this combined entry if the
8361 // PartialStruct.PreliminaryMapData.BasePointers has been mapped.
8362 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8363 (Next == CE && MapType != OMPC_MAP_unknown)) {
8364 if (!IsMappingWholeStruct) {
8365 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8366 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
8367 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8368 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8369 CombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8370 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8371 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8372 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize
8373 : 1);
8374 } else {
8375 StructBaseCombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8376 StructBaseCombinedInfo.BasePointers.push_back(
8377 Elt: BP.emitRawPointer(CGF));
8378 StructBaseCombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8379 StructBaseCombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8380 StructBaseCombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8381 StructBaseCombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8382 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8383 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8384 Elt: IsNonContiguous ? DimSize : 1);
8385 }
8386
8387 // If Mapper is valid, the last component inherits the mapper.
8388 bool HasMapper = Mapper && Next == CE;
8389 if (!IsMappingWholeStruct)
8390 CombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper : nullptr);
8391 else
8392 StructBaseCombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper
8393 : nullptr);
8394
8395 // We need to add a pointer flag for each map that comes from the
8396 // same expression except for the first one. We also need to signal
8397 // this map is the first one that relates with the current capture
8398 // (there is a set of entries for each capture).
8399 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
8400 MapType, MapModifiers, MotionModifiers, IsImplicit,
8401 AddPtrFlag: !IsExpressionFirstInfo || RequiresReference ||
8402 FirstPointerInComplexData || IsMemberReference,
8403 AddIsTargetParamFlag: IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
8404
8405 if (!IsExpressionFirstInfo || IsMemberReference) {
8406 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
8407 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
8408 if (IsPointer || (IsMemberReference && Next != CE))
8409 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8410 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8411 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8412 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8413 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8414
8415 if (ShouldBeMemberOf) {
8416 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
8417 // should be later updated with the correct value of MEMBER_OF.
8418 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8419 // From now on, all subsequent PTR_AND_OBJ entries should not be
8420 // marked as MEMBER_OF.
8421 ShouldBeMemberOf = false;
8422 }
8423 }
8424
8425 if (!IsMappingWholeStruct) {
8426 CombinedInfo.Types.push_back(Elt: Flags);
8427 // HasAttachPtr marks pointee entries, which have a base attach-ptr.
8428 CombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8429 } else {
8430 StructBaseCombinedInfo.Types.push_back(Elt: Flags);
8431 StructBaseCombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8432 }
8433 }
8434
8435 // If we have encountered a member expression so far, keep track of the
8436 // mapped member. If the parent is "*this", then the value declaration
8437 // is nullptr.
8438 if (EncounteredME) {
8439 const auto *FD = cast<FieldDecl>(Val: EncounteredME->getMemberDecl());
8440 unsigned FieldIndex = FD->getFieldIndex();
8441
8442 // Update info about the lowest and highest elements for this struct
8443 if (!PartialStruct.Base.isValid()) {
8444 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8445 if (IsFinalArraySection && OASE) {
8446 Address HB =
8447 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8448 .getAddress();
8449 PartialStruct.HighestElem = {FieldIndex, HB};
8450 } else {
8451 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8452 }
8453 PartialStruct.Base = BP;
8454 PartialStruct.LB = BP;
8455 } else if (FieldIndex < PartialStruct.LowestElem.first) {
8456 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8457 } else if (FieldIndex > PartialStruct.HighestElem.first) {
8458 if (IsFinalArraySection && OASE) {
8459 Address HB =
8460 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8461 .getAddress();
8462 PartialStruct.HighestElem = {FieldIndex, HB};
8463 } else {
8464 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8465 }
8466 }
8467 }
8468
8469 // Need to emit combined struct for array sections.
8470 if (IsFinalArraySection || IsNonContiguous)
8471 PartialStruct.IsArraySection = true;
8472
8473 // If we have a final array section, we are done with this expression.
8474 if (IsFinalArraySection)
8475 break;
8476
8477 // The pointer becomes the base for the next element.
8478 if (Next != CE)
8479 BP = IsMemberReference ? LowestElem : LB;
8480 if (!IsPartialMapped)
8481 IsExpressionFirstInfo = false;
8482 IsCaptureFirstInfo = false;
8483 FirstPointerInComplexData = false;
8484 IsPrevMemberReference = IsMemberReference;
8485 } else if (FirstPointerInComplexData) {
8486 QualType Ty = Components.rbegin()
8487 ->getAssociatedDeclaration()
8488 ->getType()
8489 .getNonReferenceType();
8490 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8491 FirstPointerInComplexData = false;
8492 }
8493 }
8494 // If ran into the whole component - allocate the space for the whole
8495 // record.
8496 if (!EncounteredME)
8497 PartialStruct.HasCompleteRecord = true;
8498
8499 // Populate ATTACH information for later processing by emitAttachEntry.
8500 if (shouldEmitAttachEntry(PointerExpr: AttachPtrExpr, MapBaseDecl: BaseDecl, CGF, CurDir)) {
8501 AttachInfo.AttachPtrAddr = AttachPtrAddr;
8502 AttachInfo.AttachPteeAddr = FinalLowestElem;
8503 AttachInfo.AttachPtrDecl = BaseDecl;
8504 AttachInfo.AttachMapExpr = MapExpr;
8505 }
8506
8507 if (!IsNonContiguous)
8508 return;
8509
8510 const ASTContext &Context = CGF.getContext();
8511
8512 // For supporting stride in array section, we need to initialize the first
8513 // dimension size as 1, first offset as 0, and first count as 1
8514 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 0)};
8515 MapValuesArrayTy CurCounts;
8516 MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8517 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8518 uint64_t ElementTypeSize;
8519
8520 // Collect Size information for each dimension and get the element size as
8521 // the first Stride. For example, for `int arr[10][10]`, the DimSizes
8522 // should be [10, 10] and the first stride is 4 btyes.
8523 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8524 Components) {
8525 const Expr *AssocExpr = Component.getAssociatedExpression();
8526 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8527
8528 if (!OASE)
8529 continue;
8530
8531 QualType Ty = ArraySectionExpr::getBaseOriginalType(Base: OASE->getBase());
8532 auto *CAT = Context.getAsConstantArrayType(T: Ty);
8533 auto *VAT = Context.getAsVariableArrayType(T: Ty);
8534
8535 // We need all the dimension size except for the last dimension.
8536 assert((VAT || CAT || &Component == &*Components.begin()) &&
8537 "Should be either ConstantArray or VariableArray if not the "
8538 "first Component");
8539
8540 // Get element size if CurCounts is empty.
8541 if (CurCounts.empty()) {
8542 const Type *ElementType = nullptr;
8543 if (CAT)
8544 ElementType = CAT->getElementType().getTypePtr();
8545 else if (VAT)
8546 ElementType = VAT->getElementType().getTypePtr();
8547 else if (&Component == &*Components.begin()) {
8548 // If the base is a raw pointer (e.g. T *data with data[a:b:c]),
8549 // there was no earlier CAT/VAT/array handling to establish
8550 // ElementType. Capture the pointee type now so that subsequent
8551 // components (offset/length/stride) have a concrete element type to
8552 // work with. This makes pointer-backed sections behave consistently
8553 // with CAT/VAT/array bases.
8554 if (const auto *PtrType = Ty->getAs<PointerType>())
8555 ElementType = PtrType->getPointeeType().getTypePtr();
8556 } else {
8557 // Any component after the first should never have a raw pointer type;
8558 // by this point. ElementType must already be known (set above or in
8559 // prior array / CAT / VAT handling).
8560 assert(!Ty->isPointerType() &&
8561 "Non-first components should not be raw pointers");
8562 }
8563
8564 // At this stage, if ElementType was a base pointer and we are in the
8565 // first iteration, it has been computed.
8566 if (ElementType) {
8567 // For the case that having pointer as base, we need to remove one
8568 // level of indirection.
8569 if (&Component != &*Components.begin())
8570 ElementType = ElementType->getPointeeOrArrayElementType();
8571 ElementTypeSize =
8572 Context.getTypeSizeInChars(T: ElementType).getQuantity();
8573 CurCounts.push_back(
8574 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: ElementTypeSize));
8575 }
8576 }
8577 // Get dimension value except for the last dimension since we don't need
8578 // it.
8579 if (DimSizes.size() < Components.size() - 1) {
8580 if (CAT)
8581 DimSizes.push_back(
8582 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: CAT->getZExtSize()));
8583 else if (VAT)
8584 DimSizes.push_back(Elt: CGF.Builder.CreateIntCast(
8585 V: CGF.EmitScalarExpr(E: VAT->getSizeExpr()), DestTy: CGF.Int64Ty,
8586 /*IsSigned=*/isSigned: false));
8587 }
8588 }
8589
8590 // Skip the dummy dimension since we have already have its information.
8591 auto *DI = DimSizes.begin() + 1;
8592 // Product of dimension.
8593 llvm::Value *DimProd =
8594 llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: ElementTypeSize);
8595
8596 // Collect info for non-contiguous. Notice that offset, count, and stride
8597 // are only meaningful for array-section, so we insert a null for anything
8598 // other than array-section.
8599 // Also, the size of offset, count, and stride are not the same as
8600 // pointers, base_pointers, sizes, or dims. Instead, the size of offset,
8601 // count, and stride are the same as the number of non-contiguous
8602 // declaration in target update to/from clause.
8603 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8604 Components) {
8605 const Expr *AssocExpr = Component.getAssociatedExpression();
8606
8607 if (const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr)) {
8608 llvm::Value *Offset = CGF.Builder.CreateIntCast(
8609 V: CGF.EmitScalarExpr(E: AE->getIdx()), DestTy: CGF.Int64Ty,
8610 /*isSigned=*/false);
8611 CurOffsets.push_back(Elt: Offset);
8612 CurCounts.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/1));
8613 CurStrides.push_back(Elt: CurStrides.back());
8614 continue;
8615 }
8616
8617 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8618
8619 if (!OASE)
8620 continue;
8621
8622 // Offset
8623 const Expr *OffsetExpr = OASE->getLowerBound();
8624 llvm::Value *Offset = nullptr;
8625 if (!OffsetExpr) {
8626 // If offset is absent, then we just set it to zero.
8627 Offset = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
8628 } else {
8629 Offset = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: OffsetExpr),
8630 DestTy: CGF.Int64Ty,
8631 /*isSigned=*/false);
8632 }
8633
8634 // Count
8635 const Expr *CountExpr = OASE->getLength();
8636 llvm::Value *Count = nullptr;
8637 if (!CountExpr) {
8638 // In Clang, once a high dimension is an array section, we construct all
8639 // the lower dimension as array section, however, for case like
8640 // arr[0:2][2], Clang construct the inner dimension as an array section
8641 // but it actually is not in an array section form according to spec.
8642 if (!OASE->getColonLocFirst().isValid() &&
8643 !OASE->getColonLocSecond().isValid()) {
8644 Count = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 1);
8645 } else {
8646 // OpenMP 5.0, 2.1.5 Array Sections, Description.
8647 // When the length is absent it defaults to ⌈(size −
8648 // lower-bound)/stride⌉, where size is the size of the array
8649 // dimension.
8650 const Expr *StrideExpr = OASE->getStride();
8651 llvm::Value *Stride =
8652 StrideExpr
8653 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8654 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8655 : nullptr;
8656 if (Stride)
8657 Count = CGF.Builder.CreateUDiv(
8658 LHS: CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset), RHS: Stride);
8659 else
8660 Count = CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset);
8661 }
8662 } else {
8663 Count = CGF.EmitScalarExpr(E: CountExpr);
8664 }
8665 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: CGF.Int64Ty, /*isSigned=*/false);
8666 CurCounts.push_back(Elt: Count);
8667
8668 // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size
8669 // Offset_n' = Offset_n * (D_0 * D_1 ... * D_n-1) * Unit size
8670 // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example:
8671 // Offset Count Stride
8672 // D0 0 4 1 (int) <- dummy dimension
8673 // D1 0 2 8 (2 * (1) * 4)
8674 // D2 100 2 20 (1 * (1 * 5) * 4)
8675 // D3 0 2 200 (2 * (1 * 5 * 4) * 4)
8676 const Expr *StrideExpr = OASE->getStride();
8677 llvm::Value *Stride =
8678 StrideExpr
8679 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8680 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8681 : nullptr;
8682 DimProd = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: *(DI - 1));
8683 if (Stride)
8684 CurStrides.push_back(Elt: CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Stride));
8685 else
8686 CurStrides.push_back(Elt: DimProd);
8687
8688 Offset = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Offset);
8689 CurOffsets.push_back(Elt: Offset);
8690
8691 if (DI != DimSizes.end())
8692 ++DI;
8693 }
8694
8695 CombinedInfo.NonContigInfo.Offsets.push_back(Elt: CurOffsets);
8696 CombinedInfo.NonContigInfo.Counts.push_back(Elt: CurCounts);
8697 CombinedInfo.NonContigInfo.Strides.push_back(Elt: CurStrides);
8698 }
8699
8700 /// Return the adjusted map modifiers if the declaration a capture refers to
8701 /// appears in a first-private clause. This is expected to be used only with
8702 /// directives that start with 'target'.
8703 OpenMPOffloadMappingFlags
8704 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
8705 assert(Cap.capturesVariable() && "Expected capture by reference only!");
8706
8707 // A first private variable captured by reference will use only the
8708 // 'private ptr' and 'map to' flag. Return the right flags if the captured
8709 // declaration is known as first-private in this handler.
8710 if (FirstPrivateDecls.count(Val: Cap.getCapturedVar())) {
8711 if (Cap.getCapturedVar()->getType()->isAnyPointerType())
8712 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8713 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8714 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8715 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8716 }
8717 auto I = LambdasMap.find(Val: Cap.getCapturedVar()->getCanonicalDecl());
8718 if (I != LambdasMap.end())
8719 // for map(to: lambda): using user specified map type.
8720 return getMapTypeBits(
8721 MapType: I->getSecond()->getMapType(), MapModifiers: I->getSecond()->getMapTypeModifiers(),
8722 /*MotionModifiers=*/{}, IsImplicit: I->getSecond()->isImplicit(),
8723 /*AddPtrFlag=*/false,
8724 /*AddIsTargetParamFlag=*/false,
8725 /*isNonContiguous=*/IsNonContiguous: false);
8726 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8727 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8728 }
8729
8730 void getPlainLayout(const CXXRecordDecl *RD,
8731 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8732 bool AsBase) const {
8733 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
8734
8735 llvm::StructType *St =
8736 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
8737
8738 unsigned NumElements = St->getNumElements();
8739 llvm::SmallVector<
8740 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8741 RecordLayout(NumElements);
8742
8743 // Fill bases.
8744 for (const auto &I : RD->bases()) {
8745 if (I.isVirtual())
8746 continue;
8747
8748 QualType BaseTy = I.getType();
8749 const auto *Base = BaseTy->getAsCXXRecordDecl();
8750 // Ignore empty bases.
8751 if (isEmptyRecordForLayout(Context: CGF.getContext(), T: BaseTy) ||
8752 CGF.getContext()
8753 .getASTRecordLayout(D: Base)
8754 .getNonVirtualSize()
8755 .isZero())
8756 continue;
8757
8758 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(RD: Base);
8759 RecordLayout[FieldIndex] = Base;
8760 }
8761 // Fill in virtual bases.
8762 for (const auto &I : RD->vbases()) {
8763 QualType BaseTy = I.getType();
8764 // Ignore empty bases.
8765 if (isEmptyRecordForLayout(Context: CGF.getContext(), T: BaseTy))
8766 continue;
8767
8768 const auto *Base = BaseTy->getAsCXXRecordDecl();
8769 unsigned FieldIndex = RL.getVirtualBaseIndex(base: Base);
8770 if (RecordLayout[FieldIndex])
8771 continue;
8772 RecordLayout[FieldIndex] = Base;
8773 }
8774 // Fill in all the fields.
8775 assert(!RD->isUnion() && "Unexpected union.");
8776 for (const auto *Field : RD->fields()) {
8777 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
8778 // will fill in later.)
8779 if (!Field->isBitField() &&
8780 !isEmptyFieldForLayout(Context: CGF.getContext(), FD: Field)) {
8781 unsigned FieldIndex = RL.getLLVMFieldNo(FD: Field);
8782 RecordLayout[FieldIndex] = Field;
8783 }
8784 }
8785 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
8786 &Data : RecordLayout) {
8787 if (Data.isNull())
8788 continue;
8789 if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Val: Data))
8790 getPlainLayout(RD: Base, Layout, /*AsBase=*/true);
8791 else
8792 Layout.push_back(Elt: cast<const FieldDecl *>(Val: Data));
8793 }
8794 }
8795
8796 /// Returns the address corresponding to \p PointerExpr.
8797 static Address getAttachPtrAddr(const Expr *PointerExpr,
8798 CodeGenFunction &CGF) {
8799 assert(PointerExpr && "Cannot get addr from null attach-ptr expr");
8800 Address AttachPtrAddr = Address::invalid();
8801
8802 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: PointerExpr)) {
8803 // If the pointer is a variable, we can use its address directly.
8804 AttachPtrAddr = CGF.EmitLValue(E: DRE).getAddress();
8805 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(Val: PointerExpr)) {
8806 AttachPtrAddr =
8807 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/true).getAddress();
8808 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: PointerExpr)) {
8809 AttachPtrAddr = CGF.EmitLValue(E: ASE).getAddress();
8810 } else if (auto *ME = dyn_cast<MemberExpr>(Val: PointerExpr)) {
8811 AttachPtrAddr = CGF.EmitMemberExpr(E: ME).getAddress();
8812 } else if (auto *UO = dyn_cast<UnaryOperator>(Val: PointerExpr)) {
8813 assert(UO->getOpcode() == UO_Deref &&
8814 "Unexpected unary-operator on attach-ptr-expr");
8815 AttachPtrAddr = CGF.EmitLValue(E: UO).getAddress();
8816 }
8817 assert(AttachPtrAddr.isValid() &&
8818 "Failed to get address for attach pointer expression");
8819 return AttachPtrAddr;
8820 }
8821
8822 /// Get the address of the attach pointer, and a load from it, to get the
8823 /// pointee base address.
8824 /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair
8825 /// contains invalid addresses if \p AttachPtrExpr is null.
8826 static std::pair<Address, Address>
8827 getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr,
8828 CodeGenFunction &CGF) {
8829
8830 if (!AttachPtrExpr)
8831 return {Address::invalid(), Address::invalid()};
8832
8833 Address AttachPtrAddr = getAttachPtrAddr(PointerExpr: AttachPtrExpr, CGF);
8834 assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr");
8835
8836 QualType AttachPtrType =
8837 OMPClauseMappableExprCommon::getComponentExprElementType(Exp: AttachPtrExpr)
8838 .getCanonicalType();
8839
8840 Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer(
8841 Ptr: AttachPtrAddr, PtrTy: AttachPtrType->castAs<PointerType>());
8842 assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr");
8843
8844 return {AttachPtrAddr, AttachPteeBaseAddr};
8845 }
8846
8847 /// Returns whether an attach entry should be emitted for a map on
8848 /// \p MapBaseDecl on the directive \p CurDir.
8849 static bool
8850 shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl,
8851 CodeGenFunction &CGF,
8852 llvm::PointerUnion<const OMPExecutableDirective *,
8853 const OMPDeclareMapperDecl *>
8854 CurDir) {
8855 if (!PointerExpr)
8856 return false;
8857
8858 // Pointer attachment is needed at map-entering time or for declare
8859 // mappers.
8860 return isa<const OMPDeclareMapperDecl *>(Val: CurDir) ||
8861 isOpenMPTargetMapEnteringDirective(
8862 DKind: cast<const OMPExecutableDirective *>(Val&: CurDir)
8863 ->getDirectiveKind());
8864 }
8865
8866 /// Computes the attach-ptr expr for \p Components, and updates various maps
8867 /// with the information.
8868 /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr()
8869 /// with the OpenMPDirectiveKind extracted from \p CurDir.
8870 /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and
8871 /// AttachPtrExprMap.
8872 void collectAttachPtrExprInfo(
8873 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
8874 llvm::PointerUnion<const OMPExecutableDirective *,
8875 const OMPDeclareMapperDecl *>
8876 CurDir) {
8877
8878 OpenMPDirectiveKind CurDirectiveID =
8879 isa<const OMPDeclareMapperDecl *>(Val: CurDir)
8880 ? OMPD_declare_mapper
8881 : cast<const OMPExecutableDirective *>(Val&: CurDir)->getDirectiveKind();
8882
8883 const auto &[AttachPtrExpr, Depth] =
8884 OMPClauseMappableExprCommon::findAttachPtrExpr(Components,
8885 CurDirKind: CurDirectiveID);
8886
8887 AttachPtrComputationOrderMap.try_emplace(
8888 Key: AttachPtrExpr, Args: AttachPtrComputationOrderMap.size());
8889 AttachPtrComponentDepthMap.try_emplace(Key: AttachPtrExpr, Args: Depth);
8890 AttachPtrExprMap.try_emplace(Key: Components, Args: AttachPtrExpr);
8891 }
8892
8893 /// Generate all the base pointers, section pointers, sizes, map types, and
8894 /// mappers for the extracted mappable expressions (all included in \a
8895 /// CombinedInfo). Also, for each item that relates with a device pointer, a
8896 /// pair of the relevant declaration and index where it occurs is appended to
8897 /// the device pointers info array.
8898 void generateAllInfoForClauses(
8899 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
8900 llvm::OpenMPIRBuilder &OMPBuilder,
8901 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
8902 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
8903 // We have to process the component lists that relate with the same
8904 // declaration in a single chunk so that we can generate the map flags
8905 // correctly. Therefore, we organize all lists in a map.
8906 enum MapKind { Present, Allocs, Other, Total };
8907 llvm::MapVector<CanonicalDeclPtr<const Decl>,
8908 SmallVector<SmallVector<MapInfo, 8>, 4>>
8909 Info;
8910
8911 // Helper function to fill the information map for the different supported
8912 // clauses.
8913 auto &&InfoGen =
8914 [&Info, &SkipVarSet](
8915 const ValueDecl *D, MapKind Kind,
8916 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
8917 OpenMPMapClauseKind MapType,
8918 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8919 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
8920 bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,
8921 const Expr *VarRef = nullptr, bool ForDeviceAddr = false) {
8922 if (SkipVarSet.contains(V: D))
8923 return;
8924 auto It = Info.try_emplace(Key: D, Args: Total).first;
8925 It->second[Kind].emplace_back(
8926 Args&: L, Args&: MapType, Args&: MapModifiers, Args&: MotionModifiers, Args&: ReturnDevicePointer,
8927 Args&: IsImplicit, Args&: Mapper, Args&: VarRef, Args&: ForDeviceAddr);
8928 };
8929
8930 for (const auto *Cl : Clauses) {
8931 const auto *C = dyn_cast<OMPMapClause>(Val: Cl);
8932 if (!C)
8933 continue;
8934 MapKind Kind = Other;
8935 if (llvm::is_contained(Range: C->getMapTypeModifiers(),
8936 Element: OMPC_MAP_MODIFIER_present))
8937 Kind = Present;
8938 else if (C->getMapType() == OMPC_MAP_alloc)
8939 Kind = Allocs;
8940 const auto *EI = C->getVarRefs().begin();
8941 for (const auto L : C->component_lists()) {
8942 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
8943 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), C->getMapType(),
8944 C->getMapTypeModifiers(), {},
8945 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
8946 E);
8947 ++EI;
8948 }
8949 }
8950 for (const auto *Cl : Clauses) {
8951 const auto *C = dyn_cast<OMPToClause>(Val: Cl);
8952 if (!C)
8953 continue;
8954 MapKind Kind = Other;
8955 if (llvm::is_contained(Range: C->getMotionModifiers(),
8956 Element: OMPC_MOTION_MODIFIER_present))
8957 Kind = Present;
8958 if (llvm::is_contained(Range: C->getMotionModifiers(),
8959 Element: OMPC_MOTION_MODIFIER_iterator)) {
8960 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
8961 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
8962 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
8963 CGF.EmitVarDecl(D: *VD);
8964 }
8965 }
8966
8967 const auto *EI = C->getVarRefs().begin();
8968 for (const auto L : C->component_lists()) {
8969 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_to, {},
8970 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
8971 C->isImplicit(), std::get<2>(t: L), *EI);
8972 ++EI;
8973 }
8974 }
8975 for (const auto *Cl : Clauses) {
8976 const auto *C = dyn_cast<OMPFromClause>(Val: Cl);
8977 if (!C)
8978 continue;
8979 MapKind Kind = Other;
8980 if (llvm::is_contained(Range: C->getMotionModifiers(),
8981 Element: OMPC_MOTION_MODIFIER_present))
8982 Kind = Present;
8983 if (llvm::is_contained(Range: C->getMotionModifiers(),
8984 Element: OMPC_MOTION_MODIFIER_iterator)) {
8985 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
8986 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
8987 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
8988 CGF.EmitVarDecl(D: *VD);
8989 }
8990 }
8991
8992 const auto *EI = C->getVarRefs().begin();
8993 for (const auto L : C->component_lists()) {
8994 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_from, {},
8995 C->getMotionModifiers(),
8996 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
8997 *EI);
8998 ++EI;
8999 }
9000 }
9001
9002 // Look at the use_device_ptr and use_device_addr clauses information and
9003 // mark the existing map entries as such. If there is no map information for
9004 // an entry in the use_device_ptr and use_device_addr list, we create one
9005 // with map type 'return_param' and zero size section. It is the user's
9006 // fault if that was not mapped before. If there is no map information, then
9007 // we defer the emission of that entry until all the maps for the same VD
9008 // have been handled.
9009 MapCombinedInfoTy UseDeviceDataCombinedInfo;
9010
9011 auto &&UseDeviceDataCombinedInfoGen =
9012 [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,
9013 CodeGenFunction &CGF, bool IsDevAddr,
9014 bool HasUdpFbNullify = false) {
9015 UseDeviceDataCombinedInfo.Exprs.push_back(Elt: VD);
9016 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Args&: Ptr);
9017 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(Args&: VD);
9018 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
9019 Args: IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
9020 // FIXME: For use_device_addr on array-sections, this should
9021 // be the starting address of the section.
9022 // e.g. int *p;
9023 // ... use_device_addr(p[3])
9024 // &p[0], &p[3], /*size=*/0, RETURN_PARAM
9025 UseDeviceDataCombinedInfo.Pointers.push_back(Elt: Ptr);
9026 UseDeviceDataCombinedInfo.Sizes.push_back(
9027 Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
9028 OpenMPOffloadMappingFlags Flags =
9029 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9030 if (HasUdpFbNullify)
9031 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9032 UseDeviceDataCombinedInfo.Types.push_back(Elt: Flags);
9033 UseDeviceDataCombinedInfo.HasAttachPtr.push_back(Elt: false);
9034 UseDeviceDataCombinedInfo.Mappers.push_back(Elt: nullptr);
9035 };
9036
9037 auto &&MapInfoGen =
9038 [&UseDeviceDataCombinedInfoGen](
9039 CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
9040 OMPClauseMappableExprCommon::MappableExprComponentListRef
9041 Components,
9042 bool IsDevAddr, bool IEIsAttachPtrForDevAddr = false,
9043 bool HasUdpFbNullify = false) {
9044 // We didn't find any match in our map information - generate a zero
9045 // size array section.
9046 llvm::Value *Ptr;
9047 if (IsDevAddr && !IEIsAttachPtrForDevAddr) {
9048 if (IE->isGLValue())
9049 Ptr = CGF.EmitLValue(E: IE).getPointer(CGF);
9050 else
9051 Ptr = CGF.EmitScalarExpr(E: IE);
9052 } else {
9053 Ptr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: IE), Loc: IE->getExprLoc());
9054 }
9055 bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr;
9056 // For the purpose of address-translation, treat something like the
9057 // following:
9058 // int *p;
9059 // ... use_device_addr(p[1])
9060 // equivalent to
9061 // ... use_device_ptr(p)
9062 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, /*IsDevAddr=*/IsDevAddr &&
9063 !TreatDevAddrAsDevPtr,
9064 HasUdpFbNullify);
9065 };
9066
9067 auto &&IsMapInfoExist =
9068 [&Info, this](CodeGenFunction &CGF, const ValueDecl *VD, const Expr *IE,
9069 const Expr *DesiredAttachPtrExpr, bool IsDevAddr,
9070 bool HasUdpFbNullify = false) -> bool {
9071 // We potentially have map information for this declaration already.
9072 // Look for the first set of components that refer to it. If found,
9073 // return true.
9074 // If the first component is a member expression, we have to look into
9075 // 'this', which maps to null in the map of map information. Otherwise
9076 // look directly for the information.
9077 auto It = Info.find(Key: isa<MemberExpr>(Val: IE) ? nullptr : VD);
9078 if (It != Info.end()) {
9079 bool Found = false;
9080 for (auto &Data : It->second) {
9081 MapInfo *CI = nullptr;
9082 // We potentially have multiple maps for the same decl. We need to
9083 // only consider those for which the attach-ptr matches the desired
9084 // attach-ptr.
9085 auto *It = llvm::find_if(Range&: Data, P: [&](const MapInfo &MI) {
9086 if (MI.Components.back().getAssociatedDeclaration() != VD)
9087 return false;
9088
9089 const Expr *MapAttachPtr = getAttachPtrExpr(Components: MI.Components);
9090 bool Match = AttachPtrComparator.areEqual(LHS: MapAttachPtr,
9091 RHS: DesiredAttachPtrExpr);
9092 return Match;
9093 });
9094
9095 if (It != Data.end())
9096 CI = &*It;
9097
9098 if (CI) {
9099 if (IsDevAddr) {
9100 CI->ForDeviceAddr = true;
9101 CI->ReturnDevicePointer = true;
9102 CI->HasUdpFbNullify = HasUdpFbNullify;
9103 Found = true;
9104 break;
9105 } else {
9106 auto PrevCI = std::next(x: CI->Components.rbegin());
9107 const auto *VarD = dyn_cast<VarDecl>(Val: VD);
9108 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: CI->Components);
9109 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
9110 isa<MemberExpr>(Val: IE) ||
9111 !VD->getType().getNonReferenceType()->isPointerType() ||
9112 PrevCI == CI->Components.rend() ||
9113 isa<MemberExpr>(Val: PrevCI->getAssociatedExpression()) || !VarD ||
9114 VarD->hasLocalStorage() ||
9115 (isa_and_nonnull<DeclRefExpr>(Val: AttachPtrExpr) &&
9116 VD == cast<DeclRefExpr>(Val: AttachPtrExpr)->getDecl())) {
9117 CI->ForDeviceAddr = IsDevAddr;
9118 CI->ReturnDevicePointer = true;
9119 CI->HasUdpFbNullify = HasUdpFbNullify;
9120 Found = true;
9121 break;
9122 }
9123 }
9124 }
9125 }
9126 return Found;
9127 }
9128 return false;
9129 };
9130
9131 // Look at the use_device_ptr clause information and mark the existing map
9132 // entries as such. If there is no map information for an entry in the
9133 // use_device_ptr list, we create one with map type 'alloc' and zero size
9134 // section. It is the user fault if that was not mapped before. If there is
9135 // no map information and the pointer is a struct member, then we defer the
9136 // emission of that entry until the whole struct has been processed.
9137 for (const auto *Cl : Clauses) {
9138 const auto *C = dyn_cast<OMPUseDevicePtrClause>(Val: Cl);
9139 if (!C)
9140 continue;
9141 bool HasUdpFbNullify =
9142 C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify;
9143 for (const auto L : C->component_lists()) {
9144 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9145 std::get<1>(t: L);
9146 assert(!Components.empty() &&
9147 "Not expecting empty list of components!");
9148 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
9149 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9150 const Expr *IE = Components.back().getAssociatedExpression();
9151 // For use_device_ptr, we match an existing map clause if its attach-ptr
9152 // is same as the use_device_ptr operand. e.g.
9153 // map expr | use_device_ptr expr | current behavior
9154 // ---------|---------------------|-----------------
9155 // p[1] | p | match
9156 // ps->a | ps | match
9157 // p | p | no match
9158 const Expr *UDPOperandExpr =
9159 Components.front().getAssociatedExpression();
9160 if (IsMapInfoExist(CGF, VD, IE,
9161 /*DesiredAttachPtrExpr=*/UDPOperandExpr,
9162 /*IsDevAddr=*/false, HasUdpFbNullify))
9163 continue;
9164 MapInfoGen(CGF, IE, VD, Components, /*IsDevAddr=*/false,
9165 /*IEIsAttachPtrForDevAddr=*/false, HasUdpFbNullify);
9166 }
9167 }
9168
9169 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
9170 for (const auto *Cl : Clauses) {
9171 const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Val: Cl);
9172 if (!C)
9173 continue;
9174 for (const auto L : C->component_lists()) {
9175 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9176 std::get<1>(t: L);
9177 assert(!std::get<1>(L).empty() &&
9178 "Not expecting empty list of components!");
9179 const ValueDecl *VD = std::get<1>(t: L).back().getAssociatedDeclaration();
9180 if (!Processed.insert(V: VD).second)
9181 continue;
9182 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9183 // For use_device_addr, we match an existing map clause if the
9184 // use_device_addr operand's attach-ptr matches the map operand's
9185 // attach-ptr.
9186 // We chould also restrict to only match cases when there is a full
9187 // match between the map/use_device_addr clause exprs, but that may be
9188 // unnecessary.
9189 //
9190 // map expr | use_device_addr expr | current | possible restrictive/
9191 // | | behavior | safer behavior
9192 // ---------|----------------------|-----------|-----------------------
9193 // p | p | match | match
9194 // p[0] | p[0] | match | match
9195 // p[0:1] | p[0] | match | no match
9196 // p[0:1] | p[2:1] | match | no match
9197 // p[1] | p[0] | match | no match
9198 // ps->a | ps->b | match | no match
9199 // p | p[0] | no match | no match
9200 // pp | pp[0][0] | no match | no match
9201 const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components);
9202 const Expr *IE = std::get<1>(t: L).back().getAssociatedExpression();
9203 assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) &&
9204 "use_device_addr operand has an attach-ptr, but does not match "
9205 "last component's expr.");
9206 if (IsMapInfoExist(CGF, VD, IE,
9207 /*DesiredAttachPtrExpr=*/UDAAttachPtrExpr,
9208 /*IsDevAddr=*/true))
9209 continue;
9210 MapInfoGen(CGF, IE, VD, Components,
9211 /*IsDevAddr=*/true,
9212 /*IEIsAttachPtrForDevAddr=*/UDAAttachPtrExpr != nullptr);
9213 }
9214 }
9215
9216 for (const auto &Data : Info) {
9217 MapCombinedInfoTy CurInfo;
9218 const Decl *D = Data.first;
9219 const ValueDecl *VD = cast_or_null<ValueDecl>(Val: D);
9220 // Group component lists by their AttachPtrExpr and process them in order
9221 // of increasing complexity (nullptr first, then simple expressions like
9222 // p, then more complex ones like p[0], etc.)
9223 //
9224 // This is similar to how generateInfoForCaptureFromClauseInfo handles
9225 // grouping for target constructs.
9226 SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs;
9227
9228 // First, collect all MapData entries with their attach-ptr exprs.
9229 for (const auto &M : Data.second) {
9230 for (const MapInfo &L : M) {
9231 assert(!L.Components.empty() &&
9232 "Not expecting declaration with no component lists.");
9233
9234 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: L.Components);
9235 AttachPtrMapInfoPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
9236 }
9237 }
9238
9239 // Next, sort by increasing order of their complexity.
9240 llvm::stable_sort(Range&: AttachPtrMapInfoPairs,
9241 C: [this](const auto &LHS, const auto &RHS) {
9242 return AttachPtrComparator(LHS.first, RHS.first);
9243 });
9244
9245 // And finally, process them all in order, grouping those with
9246 // equivalent attach-ptr exprs together.
9247 auto *It = AttachPtrMapInfoPairs.begin();
9248 while (It != AttachPtrMapInfoPairs.end()) {
9249 const Expr *AttachPtrExpr = It->first;
9250
9251 SmallVector<MapInfo, 8> GroupLists;
9252 while (It != AttachPtrMapInfoPairs.end() &&
9253 (It->first == AttachPtrExpr ||
9254 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
9255 GroupLists.push_back(Elt: It->second);
9256 ++It;
9257 }
9258 assert(!GroupLists.empty() && "GroupLists should not be empty");
9259
9260 StructRangeInfoTy PartialStruct;
9261 AttachInfoTy AttachInfo;
9262 MapCombinedInfoTy GroupCurInfo;
9263 // Current group's struct base information:
9264 MapCombinedInfoTy GroupStructBaseCurInfo;
9265 for (const MapInfo &L : GroupLists) {
9266 // Remember the current base pointer index.
9267 unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size();
9268 unsigned StructBasePointersIdx =
9269 GroupStructBaseCurInfo.BasePointers.size();
9270
9271 GroupCurInfo.NonContigInfo.IsNonContiguous =
9272 L.Components.back().isNonContiguous();
9273 generateInfoForComponentList(
9274 MapType: L.MapType, MapModifiers: L.MapModifiers, MotionModifiers: L.MotionModifiers, Components: L.Components,
9275 CombinedInfo&: GroupCurInfo, StructBaseCombinedInfo&: GroupStructBaseCurInfo, PartialStruct, AttachInfo,
9276 /*IsFirstComponentList=*/false, IsImplicit: L.IsImplicit,
9277 /*GenerateAllInfoForClauses*/ true, Mapper: L.Mapper, ForDeviceAddr: L.ForDeviceAddr, BaseDecl: VD,
9278 MapExpr: L.VarRef, /*OverlappedElements*/ {});
9279
9280 // If this entry relates to a device pointer, set the relevant
9281 // declaration and add the 'return pointer' flag.
9282 if (L.ReturnDevicePointer) {
9283 // Check whether a value was added to either GroupCurInfo or
9284 // GroupStructBaseCurInfo and error if no value was added to either
9285 // of them:
9286 assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() ||
9287 StructBasePointersIdx <
9288 GroupStructBaseCurInfo.BasePointers.size()) &&
9289 "Unexpected number of mapped base pointers.");
9290
9291 // Choose a base pointer index which is always valid:
9292 const ValueDecl *RelevantVD =
9293 L.Components.back().getAssociatedDeclaration();
9294 assert(RelevantVD &&
9295 "No relevant declaration related with device pointer??");
9296
9297 // If GroupStructBaseCurInfo has been updated this iteration then
9298 // work on the first new entry added to it i.e. make sure that when
9299 // multiple values are added to any of the lists, the first value
9300 // added is being modified by the assignments below (not the last
9301 // value added).
9302 auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info,
9303 unsigned Idx) {
9304 Info.DevicePtrDecls[Idx] = RelevantVD;
9305 Info.DevicePointers[Idx] = L.ForDeviceAddr
9306 ? DeviceInfoTy::Address
9307 : DeviceInfoTy::Pointer;
9308 Info.Types[Idx] |=
9309 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9310 if (L.HasUdpFbNullify)
9311 Info.Types[Idx] |=
9312 OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9313 };
9314
9315 if (StructBasePointersIdx <
9316 GroupStructBaseCurInfo.BasePointers.size())
9317 SetDevicePointerInfo(GroupStructBaseCurInfo,
9318 StructBasePointersIdx);
9319 else
9320 SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx);
9321 }
9322 }
9323
9324 // Unify entries in one list making sure the struct mapping precedes the
9325 // individual fields:
9326 MapCombinedInfoTy GroupUnionCurInfo;
9327 GroupUnionCurInfo.append(CurInfo&: GroupStructBaseCurInfo);
9328 GroupUnionCurInfo.append(CurInfo&: GroupCurInfo);
9329
9330 // If there is an entry in PartialStruct it means we have a struct with
9331 // individual members mapped. Emit an extra combined entry.
9332 if (PartialStruct.Base.isValid()) {
9333 // Prepend a synthetic dimension of length 1 to represent the
9334 // aggregated struct object. Using 1 (not 0, as 0 produced an
9335 // incorrect non-contiguous descriptor (DimSize==1), causing the
9336 // non-contiguous motion clause path to be skipped.) is important:
9337 // * It preserves the correct rank so targetDataUpdate() computes
9338 // DimSize == 2 for cases like strided array sections originating
9339 // from user-defined mappers (e.g. test with s.data[0:8:2]).
9340 GroupUnionCurInfo.NonContigInfo.Dims.insert(
9341 I: GroupUnionCurInfo.NonContigInfo.Dims.begin(), Elt: 1);
9342 emitCombinedEntry(
9343 CombinedInfo&: CurInfo, CurTypes&: GroupUnionCurInfo.Types, PartialStruct, AttachInfo,
9344 /*IsMapThis=*/!VD, OMPBuilder, VD,
9345 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size(),
9346 /*NotTargetParams=*/true);
9347 }
9348
9349 // Append this group's results to the overall CurInfo in the correct
9350 // order: combined-entry -> original-field-entries -> attach-entry
9351 CurInfo.append(CurInfo&: GroupUnionCurInfo);
9352 if (AttachInfo.isValid())
9353 emitAttachEntry(CGF, CombinedInfo&: CurInfo, AttachInfo);
9354 }
9355
9356 // We need to append the results of this capture to what we already have.
9357 CombinedInfo.append(CurInfo);
9358 }
9359 // Append data for use_device_ptr/addr clauses.
9360 CombinedInfo.append(CurInfo&: UseDeviceDataCombinedInfo);
9361 }
9362
9363public:
9364 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
9365 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9366 // Extract firstprivate clause information.
9367 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
9368 for (const auto *D : C->varlist())
9369 FirstPrivateDecls.try_emplace(
9370 Key: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D)->getDecl()), Args: C->isImplicit());
9371 // Extract implicit firstprivates from uses_allocators clauses.
9372 for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
9373 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
9374 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
9375 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: D.AllocatorTraits))
9376 FirstPrivateDecls.try_emplace(Key: cast<VarDecl>(Val: DRE->getDecl()),
9377 /*Implicit=*/Args: true);
9378 else if (const auto *VD = dyn_cast<VarDecl>(
9379 Val: cast<DeclRefExpr>(Val: D.Allocator->IgnoreParenImpCasts())
9380 ->getDecl()))
9381 FirstPrivateDecls.try_emplace(Key: VD, /*Implicit=*/Args: true);
9382 }
9383 }
9384 // Extract defaultmap clause information.
9385 for (const auto *C : Dir.getClausesOfKind<OMPDefaultmapClause>())
9386 if (C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate)
9387 DefaultmapFirstprivateKinds.insert(V: C->getDefaultmapKind());
9388 // Extract device pointer clause information.
9389 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9390 for (auto L : C->component_lists())
9391 DevPointersMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9392 // Extract device addr clause information.
9393 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9394 for (auto L : C->component_lists())
9395 HasDevAddrsMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9396 // Extract map information.
9397 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) {
9398 if (C->getMapType() != OMPC_MAP_to)
9399 continue;
9400 for (auto L : C->component_lists()) {
9401 const ValueDecl *VD = std::get<0>(t&: L);
9402 const auto *RD = VD ? VD->getType()
9403 .getCanonicalType()
9404 .getNonReferenceType()
9405 ->getAsCXXRecordDecl()
9406 : nullptr;
9407 if (RD && RD->isLambda())
9408 LambdasMap.try_emplace(Key: std::get<0>(t&: L), Args&: C);
9409 }
9410 }
9411
9412 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9413 for (auto L : C->component_lists()) {
9414 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9415 std::get<1>(L);
9416 if (!Components.empty())
9417 collectAttachPtrExprInfo(Components, CurDir);
9418 }
9419 };
9420
9421 // Populate the AttachPtrExprMap for all component lists from map-related
9422 // clauses.
9423 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>())
9424 CollectAttachPtrExprsForClauseComponents(C);
9425 for (const auto *C : Dir.getClausesOfKind<OMPToClause>())
9426 CollectAttachPtrExprsForClauseComponents(C);
9427 for (const auto *C : Dir.getClausesOfKind<OMPFromClause>())
9428 CollectAttachPtrExprsForClauseComponents(C);
9429 for (const auto *C : Dir.getClausesOfKind<OMPUseDevicePtrClause>())
9430 CollectAttachPtrExprsForClauseComponents(C);
9431 for (const auto *C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>())
9432 CollectAttachPtrExprsForClauseComponents(C);
9433 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9434 CollectAttachPtrExprsForClauseComponents(C);
9435 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9436 CollectAttachPtrExprsForClauseComponents(C);
9437 }
9438
9439 /// Constructor for the declare mapper directive.
9440 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
9441 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9442 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9443 for (auto L : C->component_lists()) {
9444 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9445 std::get<1>(L);
9446 if (!Components.empty())
9447 collectAttachPtrExprInfo(Components, CurDir);
9448 }
9449 };
9450
9451 // Populate the AttachPtrExprMap for all component lists from map-related
9452 // clauses in the declare mapper directive, to enable attach-style mapping
9453 // for mappers.
9454 for (const auto *Cl : Dir.clauses()) {
9455 if (const auto *C = dyn_cast<OMPMapClause>(Val: Cl))
9456 CollectAttachPtrExprsForClauseComponents(C);
9457 else if (const auto *C = dyn_cast<OMPToClause>(Val: Cl))
9458 CollectAttachPtrExprsForClauseComponents(C);
9459 else if (const auto *C = dyn_cast<OMPFromClause>(Val: Cl))
9460 CollectAttachPtrExprsForClauseComponents(C);
9461 }
9462 }
9463
9464 /// Generate code for the combined entry if we have a partially mapped struct
9465 /// and take care of the mapping flags of the arguments corresponding to
9466 /// individual struct members.
9467 /// If a valid \p AttachInfo exists, its pointee addr will be updated to point
9468 /// to the combined-entry's begin address, if emitted.
9469 /// \p PartialStruct contains attach base-pointer information.
9470 /// \returns The index of the combined entry if one was added, std::nullopt
9471 /// otherwise.
9472 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9473 MapFlagsArrayTy &CurTypes,
9474 const StructRangeInfoTy &PartialStruct,
9475 AttachInfoTy &AttachInfo, bool IsMapThis,
9476 llvm::OpenMPIRBuilder &OMPBuilder, const ValueDecl *VD,
9477 unsigned OffsetForMemberOfFlag,
9478 bool NotTargetParams) const {
9479 if (CurTypes.size() == 1 &&
9480 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9481 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9482 !PartialStruct.IsArraySection)
9483 return;
9484 Address LBAddr = PartialStruct.LowestElem.second;
9485 Address HBAddr = PartialStruct.HighestElem.second;
9486 if (PartialStruct.HasCompleteRecord) {
9487 LBAddr = PartialStruct.LB;
9488 HBAddr = PartialStruct.LB;
9489 }
9490 CombinedInfo.Exprs.push_back(Elt: VD);
9491 // Base is the base of the struct
9492 CombinedInfo.BasePointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9493 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9494 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9495 // Pointer is the address of the lowest element
9496 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
9497 const CXXMethodDecl *MD =
9498 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(Val: CGF.CurFuncDecl) : nullptr;
9499 const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
9500 bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false;
9501 // There should not be a mapper for a combined entry.
9502 if (HasBaseClass) {
9503 // OpenMP 5.2 148:21:
9504 // If the target construct is within a class non-static member function,
9505 // and a variable is an accessible data member of the object for which the
9506 // non-static data member function is invoked, the variable is treated as
9507 // if the this[:1] expression had appeared in a map clause with a map-type
9508 // of tofrom.
9509 // Emit this[:1]
9510 CombinedInfo.Pointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9511 QualType Ty = MD->getFunctionObjectParameterType();
9512 llvm::Value *Size =
9513 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty), DestTy: CGF.Int64Ty,
9514 /*isSigned=*/true);
9515 CombinedInfo.Sizes.push_back(Elt: Size);
9516 } else {
9517 CombinedInfo.Pointers.push_back(Elt: LB);
9518 // Size is (addr of {highest+1} element) - (addr of lowest element)
9519 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
9520 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
9521 Ty: HBAddr.getElementType(), Ptr: HB, /*Idx0=*/1);
9522 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(V: LB, DestTy: CGF.VoidPtrTy);
9523 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(V: HAddr, DestTy: CGF.VoidPtrTy);
9524 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(LHS: CHAddr, RHS: CLAddr);
9525 llvm::Value *Size = CGF.Builder.CreateIntCast(V: Diff, DestTy: CGF.Int64Ty,
9526 /*isSigned=*/false);
9527 CombinedInfo.Sizes.push_back(Elt: Size);
9528 }
9529 CombinedInfo.Mappers.push_back(Elt: nullptr);
9530 // Map type is always TARGET_PARAM, if generate info for captures.
9531 CombinedInfo.Types.push_back(
9532 Elt: NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9533 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9534 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9535 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9536 // A combined entry has a base attach-ptr if its constituents do. e.g.:
9537 // map(s2.s1p->x, s2.s1p->y)
9538 // combined entry:
9539 // s2.s1p[0], s2.s1p->x, sizeof(s1p->x..y), ALLOC
9540 // here s2.s1p is the attach-ptr for the combined entry.
9541 // See the inline comments in emitUserDefinedMapper's definition for how
9542 // entries with an attach-ptr are treated.
9543 CombinedInfo.HasAttachPtr.push_back(Elt: AttachInfo.isValid());
9544 // If any element has the present modifier, then make sure the runtime
9545 // doesn't attempt to allocate the struct.
9546 if (CurTypes.end() !=
9547 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9548 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9549 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9550 }))
9551 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9552 // Remove TARGET_PARAM flag from the first element
9553 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9554 // If any element has the ompx_hold modifier, then make sure the runtime
9555 // uses the hold reference count for the struct as a whole so that it won't
9556 // be unmapped by an extra dynamic reference count decrement. Add it to all
9557 // elements as well so the runtime knows which reference count to check
9558 // when determining whether it's time for device-to-host transfers of
9559 // individual elements.
9560 if (CurTypes.end() !=
9561 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9562 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9563 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9564 })) {
9565 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9566 for (auto &M : CurTypes)
9567 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9568 }
9569
9570 // All other current entries will be MEMBER_OF the combined entry
9571 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9572 // 0xFFFF in the MEMBER_OF field, or ATTACH entries since they are expected
9573 // to be handled by themselves, after all other maps).
9574 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9575 Position: OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9576 for (auto &M : CurTypes)
9577 OMPBuilder.setCorrectMemberOfFlag(Flags&: M, MemberOfFlag);
9578
9579 // When we are emitting a combined entry. If there were any pending
9580 // attachments to be done, we do them to the begin address of the combined
9581 // entry. Note that this means only one attachment per combined-entry will
9582 // be done. So, for instance, if we have:
9583 // S *ps;
9584 // ... map(ps->a, ps->b)
9585 // When we are emitting a combined entry. If AttachInfo is valid,
9586 // update the pointee address to point to the begin address of the combined
9587 // entry. This ensures that if we have multiple maps like:
9588 // `map(ps->a, ps->b)`, we still get a single ATTACH entry, like:
9589 //
9590 // &ps[0], &ps->a, sizeof(ps->a to ps->b), ALLOC // combined-entry
9591 // &ps[0], &ps->a, sizeof(ps->a), TO | FROM
9592 // &ps[0], &ps->b, sizeof(ps->b), TO | FROM
9593 // &ps, &ps->a, sizeof(void*), ATTACH // Use combined-entry's LB
9594 if (AttachInfo.isValid())
9595 AttachInfo.AttachPteeAddr = LBAddr;
9596 }
9597
9598 /// Generate all the base pointers, section pointers, sizes, map types, and
9599 /// mappers for the extracted mappable expressions (all included in \a
9600 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9601 /// pair of the relevant declaration and index where it occurs is appended to
9602 /// the device pointers info array.
9603 void generateAllInfo(
9604 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9605 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9606 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9607 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9608 "Expect a executable directive");
9609 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9610 generateAllInfoForClauses(Clauses: CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9611 SkipVarSet);
9612 }
9613
9614 /// Generate all the base pointers, section pointers, sizes, map types, and
9615 /// mappers for the extracted map clauses of user-defined mapper (all included
9616 /// in \a CombinedInfo).
9617 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9618 llvm::OpenMPIRBuilder &OMPBuilder) const {
9619 assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&
9620 "Expect a declare mapper directive");
9621 const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(Val: CurDir);
9622 generateAllInfoForClauses(Clauses: CurMapperDir->clauses(), CombinedInfo,
9623 OMPBuilder);
9624 }
9625
9626 /// Emit capture info for lambdas for variables captured by reference.
9627 void generateInfoForLambdaCaptures(
9628 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9629 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
9630 QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();
9631 const auto *RD = VDType->getAsCXXRecordDecl();
9632 if (!RD || !RD->isLambda())
9633 return;
9634 Address VDAddr(Arg, CGF.ConvertTypeForMem(T: VDType),
9635 CGF.getContext().getDeclAlign(D: VD));
9636 LValue VDLVal = CGF.MakeAddrLValue(Addr: VDAddr, T: VDType);
9637 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9638 FieldDecl *ThisCapture = nullptr;
9639 RD->getCaptureFields(Captures, ThisCapture);
9640 if (ThisCapture) {
9641 LValue ThisLVal =
9642 CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: ThisCapture);
9643 LValue ThisLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: ThisCapture);
9644 LambdaPointers.try_emplace(Key: ThisLVal.getPointer(CGF),
9645 Args: VDLVal.getPointer(CGF));
9646 CombinedInfo.Exprs.push_back(Elt: VD);
9647 CombinedInfo.BasePointers.push_back(Elt: ThisLVal.getPointer(CGF));
9648 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9649 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9650 CombinedInfo.Pointers.push_back(Elt: ThisLValVal.getPointer(CGF));
9651 CombinedInfo.Sizes.push_back(
9652 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy),
9653 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9654 CombinedInfo.Types.push_back(
9655 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9656 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9657 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9658 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9659 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9660 CombinedInfo.Mappers.push_back(Elt: nullptr);
9661 }
9662 for (const LambdaCapture &LC : RD->captures()) {
9663 if (!LC.capturesVariable())
9664 continue;
9665 const VarDecl *VD = cast<VarDecl>(Val: LC.getCapturedVar());
9666 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
9667 continue;
9668 auto It = Captures.find(Val: VD);
9669 assert(It != Captures.end() && "Found lambda capture without field.");
9670 LValue VarLVal = CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: It->second);
9671 if (LC.getCaptureKind() == LCK_ByRef) {
9672 LValue VarLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: It->second);
9673 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9674 Args: VDLVal.getPointer(CGF));
9675 CombinedInfo.Exprs.push_back(Elt: VD);
9676 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9677 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9678 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9679 CombinedInfo.Pointers.push_back(Elt: VarLValVal.getPointer(CGF));
9680 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9681 V: CGF.getTypeSize(
9682 Ty: VD->getType().getCanonicalType().getNonReferenceType()),
9683 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9684 } else {
9685 RValue VarRVal = CGF.EmitLoadOfLValue(V: VarLVal, Loc: RD->getLocation());
9686 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9687 Args: VDLVal.getPointer(CGF));
9688 CombinedInfo.Exprs.push_back(Elt: VD);
9689 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9690 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9691 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9692 CombinedInfo.Pointers.push_back(Elt: VarRVal.getScalarVal());
9693 CombinedInfo.Sizes.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0));
9694 }
9695 CombinedInfo.Types.push_back(
9696 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9697 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9698 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9699 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9700 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9701 CombinedInfo.Mappers.push_back(Elt: nullptr);
9702 }
9703 }
9704
9705 /// Set correct indices for lambdas captures.
9706 void adjustMemberOfForLambdaCaptures(
9707 llvm::OpenMPIRBuilder &OMPBuilder,
9708 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9709 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9710 MapFlagsArrayTy &Types) const {
9711 for (unsigned I = 0, E = Types.size(); I < E; ++I) {
9712 // Set correct member_of idx for all implicit lambda captures.
9713 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9714 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9715 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9716 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9717 continue;
9718 llvm::Value *BasePtr = LambdaPointers.lookup(Val: BasePointers[I]);
9719 assert(BasePtr && "Unable to find base lambda address.");
9720 int TgtIdx = -1;
9721 for (unsigned J = I; J > 0; --J) {
9722 unsigned Idx = J - 1;
9723 if (Pointers[Idx] != BasePtr)
9724 continue;
9725 TgtIdx = Idx;
9726 break;
9727 }
9728 assert(TgtIdx != -1 && "Unable to find parent lambda.");
9729 // All other current entries will be MEMBER_OF the combined entry
9730 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9731 // 0xFFFF in the MEMBER_OF field).
9732 OpenMPOffloadMappingFlags MemberOfFlag =
9733 OMPBuilder.getMemberOfFlag(Position: TgtIdx);
9734 OMPBuilder.setCorrectMemberOfFlag(Flags&: Types[I], MemberOfFlag);
9735 }
9736 }
9737
9738 /// Populate component lists for non-lambda captured variables from map,
9739 /// is_device_ptr and has_device_addr clause info.
9740 void populateComponentListsForNonLambdaCaptureFromClauses(
9741 const ValueDecl *VD, MapDataArrayTy &DeclComponentLists,
9742 SmallVectorImpl<
9743 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9744 &StorageForImplicitlyAddedComponentLists) const {
9745 if (VD && LambdasMap.count(Val: VD))
9746 return;
9747
9748 // For member fields list in is_device_ptr, store it in
9749 // DeclComponentLists for generating components info.
9750 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9751 auto It = DevPointersMap.find(Val: VD);
9752 if (It != DevPointersMap.end())
9753 for (const auto &MCL : It->second)
9754 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_to, Args: Unknown,
9755 /*IsImpicit = */ Args: true, Args: nullptr,
9756 Args: nullptr);
9757 auto I = HasDevAddrsMap.find(Val: VD);
9758 if (I != HasDevAddrsMap.end())
9759 for (const auto &MCL : I->second)
9760 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_tofrom, Args: Unknown,
9761 /*IsImpicit = */ Args: true, Args: nullptr,
9762 Args: nullptr);
9763 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9764 "Expect a executable directive");
9765 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9766 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
9767 const auto *EI = C->getVarRefs().begin();
9768 for (const auto L : C->decl_component_lists(VD)) {
9769 const ValueDecl *VDecl, *Mapper;
9770 // The Expression is not correct if the mapping is implicit
9771 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
9772 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
9773 std::tie(args&: VDecl, args&: Components, args&: Mapper) = L;
9774 assert(VDecl == VD && "We got information for the wrong declaration??");
9775 assert(!Components.empty() &&
9776 "Not expecting declaration with no component lists.");
9777 DeclComponentLists.emplace_back(Args&: Components, Args: C->getMapType(),
9778 Args: C->getMapTypeModifiers(),
9779 Args: C->isImplicit(), Args&: Mapper, Args&: E);
9780 ++EI;
9781 }
9782 }
9783
9784 // For the target construct, if there's a map with a base-pointer that's
9785 // a member of an implicitly captured struct, of the current class,
9786 // we need to emit an implicit map on the pointer.
9787 if (isOpenMPTargetExecutionDirective(DKind: CurExecDir->getDirectiveKind()))
9788 addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
9789 CapturedVD: VD, DeclComponentLists, ComponentVectorStorage&: StorageForImplicitlyAddedComponentLists);
9790
9791 llvm::stable_sort(Range&: DeclComponentLists, C: [](const MapData &LHS,
9792 const MapData &RHS) {
9793 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(t: LHS);
9794 OpenMPMapClauseKind MapType = std::get<1>(t: RHS);
9795 bool HasPresent =
9796 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
9797 bool HasAllocs = MapType == OMPC_MAP_alloc;
9798 MapModifiers = std::get<2>(t: RHS);
9799 MapType = std::get<1>(t: LHS);
9800 bool HasPresentR =
9801 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
9802 bool HasAllocsR = MapType == OMPC_MAP_alloc;
9803 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
9804 });
9805 }
9806
9807 /// On a target construct, if there's an implicit map on a struct, or that of
9808 /// this[:], and an explicit map with a member of that struct/class as the
9809 /// base-pointer, we need to make sure that base-pointer is implicitly mapped,
9810 /// to make sure we don't map the full struct/class. For example:
9811 ///
9812 /// \code
9813 /// struct S {
9814 /// int dummy[10000];
9815 /// int *p;
9816 /// void f1() {
9817 /// #pragma omp target map(p[0:1])
9818 /// (void)this;
9819 /// }
9820 /// }; S s;
9821 ///
9822 /// void f2() {
9823 /// #pragma omp target map(s.p[0:10])
9824 /// (void)s;
9825 /// }
9826 /// \endcode
9827 ///
9828 /// Only `this-p` and `s.p` should be mapped in the two cases above.
9829 //
9830 // OpenMP 6.0: 7.9.6 map clause, pg 285
9831 // If a list item with an implicitly determined data-mapping attribute does
9832 // not have any corresponding storage in the device data environment prior to
9833 // a task encountering the construct associated with the map clause, and one
9834 // or more contiguous parts of the original storage are either list items or
9835 // base pointers to list items that are explicitly mapped on the construct,
9836 // only those parts of the original storage will have corresponding storage in
9837 // the device data environment as a result of the map clauses on the
9838 // construct.
9839 void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
9840 const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists,
9841 SmallVectorImpl<
9842 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9843 &ComponentVectorStorage) const {
9844 bool IsThisCapture = CapturedVD == nullptr;
9845
9846 for (const auto &ComponentsAndAttachPtr : AttachPtrExprMap) {
9847 OMPClauseMappableExprCommon::MappableExprComponentListRef
9848 ComponentsWithAttachPtr = ComponentsAndAttachPtr.first;
9849 const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second;
9850 if (!AttachPtrExpr)
9851 continue;
9852
9853 const auto *ME = dyn_cast<MemberExpr>(Val: AttachPtrExpr);
9854 if (!ME)
9855 continue;
9856
9857 const Expr *Base = ME->getBase()->IgnoreParenImpCasts();
9858
9859 // If we are handling a "this" capture, then we are looking for
9860 // attach-ptrs of form `this->p`, either explicitly or implicitly.
9861 if (IsThisCapture && !ME->isImplicitCXXThis() && !isa<CXXThisExpr>(Val: Base))
9862 continue;
9863
9864 if (!IsThisCapture && (!isa<DeclRefExpr>(Val: Base) ||
9865 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
9866 continue;
9867
9868 // For non-this captures, we are looking for attach-ptrs of form
9869 // `s.p`.
9870 // For non-this captures, we are looking for attach-ptrs like `s.p`.
9871 if (!IsThisCapture && (ME->isArrow() || !isa<DeclRefExpr>(Val: Base) ||
9872 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
9873 continue;
9874
9875 // Check if we have an existing map on either:
9876 // this[:], s, this->p, or s.p, in which case, we don't need to add
9877 // an implicit one for the attach-ptr s.p/this->p.
9878 bool FoundExistingMap = false;
9879 for (const MapData &ExistingL : DeclComponentLists) {
9880 OMPClauseMappableExprCommon::MappableExprComponentListRef
9881 ExistingComponents = std::get<0>(t: ExistingL);
9882
9883 if (ExistingComponents.empty())
9884 continue;
9885
9886 // First check if we have a map like map(this->p) or map(s.p).
9887 const auto &FirstComponent = ExistingComponents.front();
9888 const Expr *FirstExpr = FirstComponent.getAssociatedExpression();
9889
9890 if (!FirstExpr)
9891 continue;
9892
9893 // First check if we have a map like map(this->p) or map(s.p).
9894 if (AttachPtrComparator.areEqual(LHS: FirstExpr, RHS: AttachPtrExpr)) {
9895 FoundExistingMap = true;
9896 break;
9897 }
9898
9899 // Check if we have a map like this[0:1]
9900 if (IsThisCapture) {
9901 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: FirstExpr)) {
9902 if (isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts())) {
9903 FoundExistingMap = true;
9904 break;
9905 }
9906 }
9907 continue;
9908 }
9909
9910 // When the attach-ptr is something like `s.p`, check if
9911 // `s` itself is mapped explicitly.
9912 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: FirstExpr)) {
9913 if (DRE->getDecl() == CapturedVD) {
9914 FoundExistingMap = true;
9915 break;
9916 }
9917 }
9918 }
9919
9920 if (FoundExistingMap)
9921 continue;
9922
9923 // If no base map is found, we need to create an implicit map for the
9924 // attach-pointer expr.
9925
9926 ComponentVectorStorage.emplace_back();
9927 auto &AttachPtrComponents = ComponentVectorStorage.back();
9928
9929 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9930 bool SeenAttachPtrComponent = false;
9931 // For creating a map on the attach-ptr `s.p/this->p`, we copy all
9932 // components from the component-list which has `s.p/this->p`
9933 // as the attach-ptr, starting from the component which matches
9934 // `s.p/this->p`. This way, we'll have component-lists of
9935 // `s.p` -> `s`, and `this->p` -> `this`.
9936 for (size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) {
9937 const auto &Component = ComponentsWithAttachPtr[i];
9938 const Expr *ComponentExpr = Component.getAssociatedExpression();
9939
9940 if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr)
9941 continue;
9942 SeenAttachPtrComponent = true;
9943
9944 AttachPtrComponents.emplace_back(Args: Component.getAssociatedExpression(),
9945 Args: Component.getAssociatedDeclaration(),
9946 Args: Component.isNonContiguous());
9947 }
9948 assert(!AttachPtrComponents.empty() &&
9949 "Could not populate component-lists for mapping attach-ptr");
9950
9951 DeclComponentLists.emplace_back(
9952 Args&: AttachPtrComponents, Args: OMPC_MAP_tofrom, Args: Unknown,
9953 /*IsImplicit=*/Args: true, /*mapper=*/Args: nullptr, Args&: AttachPtrExpr);
9954 }
9955 }
9956
9957 /// For a capture that has an associated clause, generate the base pointers,
9958 /// section pointers, sizes, map types, and mappers (all included in
9959 /// \a CurCaptureVarInfo).
9960 void generateInfoForCaptureFromClauseInfo(
9961 const MapDataArrayTy &DeclComponentListsFromClauses,
9962 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
9963 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9964 unsigned OffsetForMemberOfFlag) const {
9965 assert(!Cap->capturesVariableArrayType() &&
9966 "Not expecting to generate map info for a variable array type!");
9967
9968 // We need to know when we generating information for the first component
9969 const ValueDecl *VD = Cap->capturesThis()
9970 ? nullptr
9971 : Cap->getCapturedVar()->getCanonicalDecl();
9972
9973 // for map(to: lambda): skip here, processing it in
9974 // generateDefaultMapInfo
9975 if (LambdasMap.count(Val: VD))
9976 return;
9977
9978 // If this declaration appears in a is_device_ptr clause we just have to
9979 // pass the pointer by value. If it is a reference to a declaration, we just
9980 // pass its value.
9981 if (VD && (DevPointersMap.count(Val: VD) || HasDevAddrsMap.count(Val: VD))) {
9982 CurCaptureVarInfo.Exprs.push_back(Elt: VD);
9983 CurCaptureVarInfo.BasePointers.emplace_back(Args&: Arg);
9984 CurCaptureVarInfo.DevicePtrDecls.emplace_back(Args&: VD);
9985 CurCaptureVarInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::Pointer);
9986 CurCaptureVarInfo.Pointers.push_back(Elt: Arg);
9987 CurCaptureVarInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9988 V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy), DestTy: CGF.Int64Ty,
9989 /*isSigned=*/true));
9990 CurCaptureVarInfo.Types.push_back(
9991 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9992 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9993 CurCaptureVarInfo.HasAttachPtr.push_back(Elt: false);
9994 CurCaptureVarInfo.Mappers.push_back(Elt: nullptr);
9995 return;
9996 }
9997
9998 auto GenerateInfoForComponentLists =
9999 [&](ArrayRef<MapData> DeclComponentListsFromClauses,
10000 bool IsEligibleForTargetParamFlag) {
10001 MapCombinedInfoTy CurInfoForComponentLists;
10002 StructRangeInfoTy PartialStruct;
10003 AttachInfoTy AttachInfo;
10004
10005 if (DeclComponentListsFromClauses.empty())
10006 return;
10007
10008 generateInfoForCaptureFromComponentLists(
10009 VD, DeclComponentLists: DeclComponentListsFromClauses, CurComponentListInfo&: CurInfoForComponentLists,
10010 PartialStruct, AttachInfo, IsListEligibleForTargetParamFlag: IsEligibleForTargetParamFlag);
10011
10012 // If there is an entry in PartialStruct it means we have a
10013 // struct with individual members mapped. Emit an extra combined
10014 // entry.
10015 if (PartialStruct.Base.isValid()) {
10016 CurCaptureVarInfo.append(CurInfo&: PartialStruct.PreliminaryMapData);
10017 emitCombinedEntry(
10018 CombinedInfo&: CurCaptureVarInfo, CurTypes&: CurInfoForComponentLists.Types,
10019 PartialStruct, AttachInfo, IsMapThis: Cap->capturesThis(), OMPBuilder,
10020 /*VD=*/nullptr, OffsetForMemberOfFlag,
10021 /*NotTargetParams*/ !IsEligibleForTargetParamFlag);
10022 }
10023
10024 // We do the appends to get the entries in the following order:
10025 // combined-entry -> individual-field-entries -> attach-entry,
10026 CurCaptureVarInfo.append(CurInfo&: CurInfoForComponentLists);
10027 if (AttachInfo.isValid())
10028 emitAttachEntry(CGF, CombinedInfo&: CurCaptureVarInfo, AttachInfo);
10029 };
10030
10031 // Group component lists by their AttachPtrExpr and process them in order
10032 // of increasing complexity (nullptr first, then simple expressions like p,
10033 // then more complex ones like p[0], etc.)
10034 //
10035 // This ensure that we:
10036 // * handle maps that can contribute towards setting the kernel argument,
10037 // (e.g. map(ps), or map(ps[0])), before any that cannot (e.g. ps->pt->d).
10038 // * allocate a single contiguous storage for all exprs with the same
10039 // captured var and having the same attach-ptr.
10040 //
10041 // Example: The map clauses below should be handled grouped together based
10042 // on their attachable-base-pointers:
10043 // map-clause | attachable-base-pointer
10044 // --------------------------+------------------------
10045 // map(p, ps) | nullptr
10046 // map(p[0]) | p
10047 // map(p[0]->b, p[0]->c) | p[0]
10048 // map(ps->d, ps->e, ps->pt) | ps
10049 // map(ps->pt->d, ps->pt->e) | ps->pt
10050
10051 // First, collect all MapData entries with their attach-ptr exprs.
10052 SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs;
10053
10054 for (const MapData &L : DeclComponentListsFromClauses) {
10055 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
10056 std::get<0>(t: L);
10057 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
10058 AttachPtrMapDataPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
10059 }
10060
10061 // Next, sort by increasing order of their complexity.
10062 llvm::stable_sort(Range&: AttachPtrMapDataPairs,
10063 C: [this](const auto &LHS, const auto &RHS) {
10064 return AttachPtrComparator(LHS.first, RHS.first);
10065 });
10066
10067 bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty();
10068 bool IsFirstGroup = true;
10069
10070 // And finally, process them all in order, grouping those with
10071 // equivalent attach-ptr exprs together.
10072 auto *It = AttachPtrMapDataPairs.begin();
10073 while (It != AttachPtrMapDataPairs.end()) {
10074 const Expr *AttachPtrExpr = It->first;
10075
10076 MapDataArrayTy GroupLists;
10077 while (It != AttachPtrMapDataPairs.end() &&
10078 (It->first == AttachPtrExpr ||
10079 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
10080 GroupLists.push_back(Elt: It->second);
10081 ++It;
10082 }
10083 assert(!GroupLists.empty() && "GroupLists should not be empty");
10084
10085 // Determine if this group of component-lists is eligible for TARGET_PARAM
10086 // flag. Only the first group processed should be eligible, and only if no
10087 // default mapping was done.
10088 bool IsEligibleForTargetParamFlag =
10089 IsFirstGroup && NoDefaultMappingDoneForVD;
10090
10091 GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag);
10092 IsFirstGroup = false;
10093 }
10094 }
10095
10096 /// Generate the base pointers, section pointers, sizes, map types, and
10097 /// mappers associated to \a DeclComponentLists for a given capture
10098 /// \a VD (all included in \a CurComponentListInfo).
10099 void generateInfoForCaptureFromComponentLists(
10100 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
10101 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
10102 AttachInfoTy &AttachInfo, bool IsListEligibleForTargetParamFlag) const {
10103 // Find overlapping elements (including the offset from the base element).
10104 llvm::SmallDenseMap<
10105 const MapData *,
10106 llvm::SmallVector<
10107 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
10108 4>
10109 OverlappedData;
10110 size_t Count = 0;
10111 for (const MapData &L : DeclComponentLists) {
10112 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10113 OpenMPMapClauseKind MapType;
10114 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10115 bool IsImplicit;
10116 const ValueDecl *Mapper;
10117 const Expr *VarRef;
10118 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10119 L;
10120 ++Count;
10121 for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(N: Count)) {
10122 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
10123 std::tie(args&: Components1, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper,
10124 args&: VarRef) = L1;
10125 auto CI = Components.rbegin();
10126 auto CE = Components.rend();
10127 auto SI = Components1.rbegin();
10128 auto SE = Components1.rend();
10129 for (; CI != CE && SI != SE; ++CI, ++SI) {
10130 if (CI->getAssociatedExpression()->getStmtClass() !=
10131 SI->getAssociatedExpression()->getStmtClass())
10132 break;
10133 // Are we dealing with different variables/fields?
10134 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
10135 break;
10136 }
10137 // Found overlapping if, at least for one component, reached the head
10138 // of the components list.
10139 if (CI == CE || SI == SE) {
10140 // Ignore it if it is the same component.
10141 if (CI == CE && SI == SE)
10142 continue;
10143 const auto It = (SI == SE) ? CI : SI;
10144 // If one component is a pointer and another one is a kind of
10145 // dereference of this pointer (array subscript, section, dereference,
10146 // etc.), it is not an overlapping.
10147 // Same, if one component is a base and another component is a
10148 // dereferenced pointer memberexpr with the same base.
10149 if (!isa<MemberExpr>(Val: It->getAssociatedExpression()) ||
10150 (std::prev(x: It)->getAssociatedDeclaration() &&
10151 std::prev(x: It)
10152 ->getAssociatedDeclaration()
10153 ->getType()
10154 ->isPointerType()) ||
10155 (It->getAssociatedDeclaration() &&
10156 It->getAssociatedDeclaration()->getType()->isPointerType() &&
10157 std::next(x: It) != CE && std::next(x: It) != SE))
10158 continue;
10159 const MapData &BaseData = CI == CE ? L : L1;
10160 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
10161 SI == SE ? Components : Components1;
10162 OverlappedData[&BaseData].push_back(Elt: SubData);
10163 }
10164 }
10165 }
10166 // Sort the overlapped elements for each item.
10167 llvm::SmallVector<const FieldDecl *, 4> Layout;
10168 if (!OverlappedData.empty()) {
10169 const Type *BaseType = VD->getType().getCanonicalType().getTypePtr();
10170 const Type *OrigType = BaseType->getPointeeOrArrayElementType();
10171 while (BaseType != OrigType) {
10172 BaseType = OrigType->getCanonicalTypeInternal().getTypePtr();
10173 OrigType = BaseType->getPointeeOrArrayElementType();
10174 }
10175
10176 if (const auto *CRD = BaseType->getAsCXXRecordDecl())
10177 getPlainLayout(RD: CRD, Layout, /*AsBase=*/false);
10178 else {
10179 const auto *RD = BaseType->getAsRecordDecl();
10180 Layout.append(in_start: RD->field_begin(), in_end: RD->field_end());
10181 }
10182 }
10183 for (auto &Pair : OverlappedData) {
10184 llvm::stable_sort(
10185 Range&: Pair.getSecond(),
10186 C: [&Layout](
10187 OMPClauseMappableExprCommon::MappableExprComponentListRef First,
10188 OMPClauseMappableExprCommon::MappableExprComponentListRef
10189 Second) {
10190 auto CI = First.rbegin();
10191 auto CE = First.rend();
10192 auto SI = Second.rbegin();
10193 auto SE = Second.rend();
10194 for (; CI != CE && SI != SE; ++CI, ++SI) {
10195 if (CI->getAssociatedExpression()->getStmtClass() !=
10196 SI->getAssociatedExpression()->getStmtClass())
10197 break;
10198 // Are we dealing with different variables/fields?
10199 if (CI->getAssociatedDeclaration() !=
10200 SI->getAssociatedDeclaration())
10201 break;
10202 }
10203
10204 // Lists contain the same elements.
10205 if (CI == CE && SI == SE)
10206 return false;
10207
10208 // List with less elements is less than list with more elements.
10209 if (CI == CE || SI == SE)
10210 return CI == CE;
10211
10212 const auto *FD1 = cast<FieldDecl>(Val: CI->getAssociatedDeclaration());
10213 const auto *FD2 = cast<FieldDecl>(Val: SI->getAssociatedDeclaration());
10214 if (FD1->getParent() == FD2->getParent())
10215 return FD1->getFieldIndex() < FD2->getFieldIndex();
10216 const auto *It =
10217 llvm::find_if(Range&: Layout, P: [FD1, FD2](const FieldDecl *FD) {
10218 return FD == FD1 || FD == FD2;
10219 });
10220 return *It == FD1;
10221 });
10222 }
10223
10224 // Associated with a capture, because the mapping flags depend on it.
10225 // Go through all of the elements with the overlapped elements.
10226 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
10227 MapCombinedInfoTy StructBaseCombinedInfo;
10228 for (const auto &Pair : OverlappedData) {
10229 const MapData &L = *Pair.getFirst();
10230 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10231 OpenMPMapClauseKind MapType;
10232 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10233 bool IsImplicit;
10234 const ValueDecl *Mapper;
10235 const Expr *VarRef;
10236 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10237 L;
10238 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
10239 OverlappedComponents = Pair.getSecond();
10240 generateInfoForComponentList(
10241 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10242 StructBaseCombinedInfo, PartialStruct, AttachInfo, IsFirstComponentList: AddTargetParamFlag,
10243 IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
10244 /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef, OverlappedElements: OverlappedComponents);
10245 AddTargetParamFlag = false;
10246 }
10247 // Go through other elements without overlapped elements.
10248 for (const MapData &L : DeclComponentLists) {
10249 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10250 OpenMPMapClauseKind MapType;
10251 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10252 bool IsImplicit;
10253 const ValueDecl *Mapper;
10254 const Expr *VarRef;
10255 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10256 L;
10257 auto It = OverlappedData.find(Val: &L);
10258 if (It == OverlappedData.end())
10259 generateInfoForComponentList(
10260 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10261 StructBaseCombinedInfo, PartialStruct, AttachInfo,
10262 IsFirstComponentList: AddTargetParamFlag, IsImplicit, /*GenerateAllInfoForClauses*/ false,
10263 Mapper, /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef,
10264 /*OverlappedElements*/ {});
10265 AddTargetParamFlag = false;
10266 }
10267 }
10268
10269 /// Check if a variable should be treated as firstprivate due to explicit
10270 /// firstprivate clause or defaultmap(firstprivate:...).
10271 bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const {
10272 // Check explicit firstprivate clauses (not implicit from defaultmap)
10273 auto I = FirstPrivateDecls.find(Val: VD);
10274 if (I != FirstPrivateDecls.end() && !I->getSecond())
10275 return true; // Explicit firstprivate only
10276
10277 // Check defaultmap(firstprivate:scalar) for scalar types
10278 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_scalar)) {
10279 if (Type->isScalarType())
10280 return true;
10281 }
10282
10283 // Check defaultmap(firstprivate:pointer) for pointer types
10284 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_pointer)) {
10285 if (Type->isAnyPointerType())
10286 return true;
10287 }
10288
10289 // Check defaultmap(firstprivate:aggregate) for aggregate types
10290 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_aggregate)) {
10291 if (Type->isAggregateType())
10292 return true;
10293 }
10294
10295 // Check defaultmap(firstprivate:all) for all types
10296 return DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_all);
10297 }
10298
10299 /// Generate the default map information for a given capture \a CI,
10300 /// record field declaration \a RI and captured value \a CV.
10301 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
10302 const FieldDecl &RI, llvm::Value *CV,
10303 MapCombinedInfoTy &CombinedInfo) const {
10304 bool IsImplicit = true;
10305 // Do the default mapping.
10306 if (CI.capturesThis()) {
10307 CombinedInfo.Exprs.push_back(Elt: nullptr);
10308 CombinedInfo.BasePointers.push_back(Elt: CV);
10309 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10310 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10311 CombinedInfo.Pointers.push_back(Elt: CV);
10312 const auto *PtrTy = cast<PointerType>(Val: RI.getType().getTypePtr());
10313 CombinedInfo.Sizes.push_back(
10314 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: PtrTy->getPointeeType()),
10315 DestTy: CGF.Int64Ty, /*isSigned=*/true));
10316 // Default map type.
10317 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TO |
10318 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
10319 } else if (CI.capturesVariableByCopy()) {
10320 const VarDecl *VD = CI.getCapturedVar();
10321 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10322 CombinedInfo.BasePointers.push_back(Elt: CV);
10323 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10324 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10325 CombinedInfo.Pointers.push_back(Elt: CV);
10326 bool IsFirstprivate =
10327 isEffectivelyFirstprivate(VD, Type: RI.getType().getNonReferenceType());
10328
10329 if (!RI.getType()->isAnyPointerType()) {
10330 // We have to signal to the runtime captures passed by value that are
10331 // not pointers.
10332 CombinedInfo.Types.push_back(
10333 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10334 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10335 V: CGF.getTypeSize(Ty: RI.getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10336 } else if (IsFirstprivate) {
10337 // Firstprivate pointers should be passed by value (as literals)
10338 // without performing a present table lookup at runtime.
10339 CombinedInfo.Types.push_back(
10340 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10341 // Use zero size for pointer literals (just passing the pointer value)
10342 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10343 } else {
10344 // Pointers are implicitly mapped with a zero size and no flags
10345 // (other than first map that is added for all implicit maps).
10346 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_NONE);
10347 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10348 }
10349 auto I = FirstPrivateDecls.find(Val: VD);
10350 if (I != FirstPrivateDecls.end())
10351 IsImplicit = I->getSecond();
10352 } else {
10353 assert(CI.capturesVariable() && "Expected captured reference.");
10354 const auto *PtrTy = cast<ReferenceType>(Val: RI.getType().getTypePtr());
10355 QualType ElementType = PtrTy->getPointeeType();
10356 const VarDecl *VD = CI.getCapturedVar();
10357 bool IsFirstprivate = isEffectivelyFirstprivate(VD, Type: ElementType);
10358 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10359 CombinedInfo.BasePointers.push_back(Elt: CV);
10360 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10361 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10362
10363 // For firstprivate pointers, pass by value instead of dereferencing
10364 if (IsFirstprivate && ElementType->isAnyPointerType()) {
10365 // Treat as a literal value (pass the pointer value itself)
10366 CombinedInfo.Pointers.push_back(Elt: CV);
10367 // Use zero size for pointer literals
10368 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10369 CombinedInfo.Types.push_back(
10370 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10371 } else {
10372 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10373 V: CGF.getTypeSize(Ty: ElementType), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10374 // The default map type for a scalar/complex type is 'to' because by
10375 // default the value doesn't have to be retrieved. For an aggregate
10376 // type, the default is 'tofrom'.
10377 CombinedInfo.Types.push_back(Elt: getMapModifiersForPrivateClauses(Cap: CI));
10378 CombinedInfo.Pointers.push_back(Elt: CV);
10379 }
10380 auto I = FirstPrivateDecls.find(Val: VD);
10381 if (I != FirstPrivateDecls.end())
10382 IsImplicit = I->getSecond();
10383 }
10384 // Every default map produces a single argument which is a target parameter.
10385 CombinedInfo.Types.back() |=
10386 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
10387
10388 // Add flag stating this is an implicit map.
10389 if (IsImplicit)
10390 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
10391
10392 CombinedInfo.HasAttachPtr.push_back(Elt: false);
10393 // No user-defined mapper for default mapping.
10394 CombinedInfo.Mappers.push_back(Elt: nullptr);
10395 }
10396};
10397} // anonymous namespace
10398
10399// Try to extract the base declaration from a `this->x` expression if possible.
10400static ValueDecl *getDeclFromThisExpr(const Expr *E) {
10401 if (!E)
10402 return nullptr;
10403
10404 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenCasts()))
10405 if (const MemberExpr *ME =
10406 dyn_cast<MemberExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))
10407 return ME->getMemberDecl();
10408 return nullptr;
10409}
10410
10411/// Emit a string constant containing the names of the values mapped to the
10412/// offloading runtime library.
10413static llvm::Constant *
10414emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
10415 MappableExprsHandler::MappingExprInfo &MapExprs) {
10416
10417 uint32_t SrcLocStrSize;
10418 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
10419 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10420
10421 SourceLocation Loc;
10422 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
10423 if (const ValueDecl *VD = getDeclFromThisExpr(E: MapExprs.getMapExpr()))
10424 Loc = VD->getLocation();
10425 else
10426 Loc = MapExprs.getMapExpr()->getExprLoc();
10427 } else {
10428 Loc = MapExprs.getMapDecl()->getLocation();
10429 }
10430
10431 std::string ExprName;
10432 if (MapExprs.getMapExpr()) {
10433 PrintingPolicy P(CGF.getContext().getLangOpts());
10434 llvm::raw_string_ostream OS(ExprName);
10435 MapExprs.getMapExpr()->printPretty(OS, Helper: nullptr, Policy: P);
10436 } else {
10437 ExprName = MapExprs.getMapDecl()->getNameAsString();
10438 }
10439
10440 std::string FileName;
10441 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
10442 if (auto *DbgInfo = CGF.getDebugInfo())
10443 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
10444 else
10445 FileName = PLoc.getFilename();
10446 return OMPBuilder.getOrCreateSrcLocStr(FunctionName: FileName, FileName: ExprName, Line: PLoc.getLine(),
10447 Column: PLoc.getColumn(), SrcLocStrSize);
10448}
10449/// Emit the arrays used to pass the captures and map information to the
10450/// offloading runtime library. If there is no map or capture information,
10451/// return nullptr by reference.
10452static void emitOffloadingArraysAndArgs(
10453 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10454 CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder,
10455 bool IsNonContiguous = false, bool ForEndCall = false) {
10456 CodeGenModule &CGM = CGF.CGM;
10457
10458 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10459 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
10460 CGF.AllocaInsertPt->getIterator());
10461 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
10462 CGF.Builder.GetInsertPoint());
10463
10464 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
10465 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10466 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
10467 }
10468 };
10469
10470 auto CustomMapperCB = [&](unsigned int I) {
10471 llvm::Function *MFunc = nullptr;
10472 if (CombinedInfo.Mappers[I]) {
10473 Info.HasMapper = true;
10474 MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
10475 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10476 }
10477 return MFunc;
10478 };
10479 cantFail(Err: OMPBuilder.emitOffloadingArraysAndArgs(
10480 AllocaIP, CodeGenIP, Info, RTArgs&: Info.RTArgs, CombinedInfo, CustomMapperCB,
10481 IsNonContiguous, ForEndCall, DeviceAddrCB));
10482}
10483
10484/// Check for inner distribute directive.
10485static const OMPExecutableDirective *
10486getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
10487 const auto *CS = D.getInnermostCapturedStmt();
10488 const auto *Body =
10489 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
10490 const Stmt *ChildStmt =
10491 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10492
10493 if (const auto *NestedDir =
10494 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10495 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
10496 switch (D.getDirectiveKind()) {
10497 case OMPD_target:
10498 // For now, treat 'target' with nested 'teams loop' as if it's
10499 // distributed (target teams distribute).
10500 if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop)
10501 return NestedDir;
10502 if (DKind == OMPD_teams) {
10503 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
10504 /*IgnoreCaptured=*/true);
10505 if (!Body)
10506 return nullptr;
10507 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10508 if (const auto *NND =
10509 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10510 DKind = NND->getDirectiveKind();
10511 if (isOpenMPDistributeDirective(DKind))
10512 return NND;
10513 }
10514 }
10515 return nullptr;
10516 case OMPD_target_teams:
10517 if (isOpenMPDistributeDirective(DKind))
10518 return NestedDir;
10519 return nullptr;
10520 case OMPD_target_parallel:
10521 case OMPD_target_simd:
10522 case OMPD_target_parallel_for:
10523 case OMPD_target_parallel_for_simd:
10524 return nullptr;
10525 case OMPD_target_teams_distribute:
10526 case OMPD_target_teams_distribute_simd:
10527 case OMPD_target_teams_distribute_parallel_for:
10528 case OMPD_target_teams_distribute_parallel_for_simd:
10529 case OMPD_parallel:
10530 case OMPD_for:
10531 case OMPD_parallel_for:
10532 case OMPD_parallel_master:
10533 case OMPD_parallel_sections:
10534 case OMPD_for_simd:
10535 case OMPD_parallel_for_simd:
10536 case OMPD_cancel:
10537 case OMPD_cancellation_point:
10538 case OMPD_ordered_standalone:
10539 case OMPD_ordered_blockassoc:
10540 case OMPD_threadprivate:
10541 case OMPD_allocate:
10542 case OMPD_task:
10543 case OMPD_simd:
10544 case OMPD_tile:
10545 case OMPD_unroll:
10546 case OMPD_sections:
10547 case OMPD_section:
10548 case OMPD_single:
10549 case OMPD_master:
10550 case OMPD_critical:
10551 case OMPD_taskyield:
10552 case OMPD_barrier:
10553 case OMPD_taskwait:
10554 case OMPD_taskgroup:
10555 case OMPD_atomic:
10556 case OMPD_flush:
10557 case OMPD_depobj:
10558 case OMPD_scan:
10559 case OMPD_teams:
10560 case OMPD_target_data:
10561 case OMPD_target_exit_data:
10562 case OMPD_target_enter_data:
10563 case OMPD_distribute:
10564 case OMPD_distribute_simd:
10565 case OMPD_distribute_parallel_for:
10566 case OMPD_distribute_parallel_for_simd:
10567 case OMPD_teams_distribute:
10568 case OMPD_teams_distribute_simd:
10569 case OMPD_teams_distribute_parallel_for:
10570 case OMPD_teams_distribute_parallel_for_simd:
10571 case OMPD_target_update:
10572 case OMPD_declare_simd:
10573 case OMPD_declare_variant:
10574 case OMPD_begin_declare_variant:
10575 case OMPD_end_declare_variant:
10576 case OMPD_declare_target:
10577 case OMPD_end_declare_target:
10578 case OMPD_declare_reduction:
10579 case OMPD_declare_mapper:
10580 case OMPD_taskloop:
10581 case OMPD_taskloop_simd:
10582 case OMPD_master_taskloop:
10583 case OMPD_master_taskloop_simd:
10584 case OMPD_parallel_master_taskloop:
10585 case OMPD_parallel_master_taskloop_simd:
10586 case OMPD_requires:
10587 case OMPD_metadirective:
10588 case OMPD_unknown:
10589 default:
10590 llvm_unreachable("Unexpected directive.");
10591 }
10592 }
10593
10594 return nullptr;
10595}
10596
10597/// Emit the user-defined mapper function. The code generation follows the
10598/// pattern in the example below.
10599/// \code
10600/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
10601/// void *base, void *begin,
10602/// int64_t size, int64_t type,
10603/// void *name = nullptr) {
10604/// // Allocate space for an array section first.
10605/// if ((size > 1 || (base != begin)) && !maptype.IsDelete)
10606/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10607/// size*sizeof(Ty), clearToFromMember(type));
10608/// // Map members.
10609/// for (unsigned i = 0; i < size; i++) {
10610/// N = __tgt_mapper_num_components(rt_mapper_handle);
10611/// // For each component specified by this mapper:
10612/// for (auto c : begin[i]->all_components) {
10613/// // MEMBER_OF grouping: tie this component to the current array element
10614/// // (component N) by adding N<<48. Exceptions:
10615/// // - ATTACH entries are not members of any struct storage range.
10616/// // - Pointee entries (reached via a pointer member) occupy separate
10617/// // storage; their inner MEMBER_OF bits are shifted by N instead.
10618/// if (c.isAttach() || c.isPointee())
10619/// member_type = c.arg_type + (c.hasInnerMemberOf() ? N<<48 : 0);
10620/// else
10621/// member_type = c.arg_type + N<<48;
10622/// // Map-type-modifying bits (ALWAYS, DELETE, CLOSE) from the outer map
10623/// // clause are propagated to each component, except ATTACH entries
10624/// // (ATTACH|ALWAYS is reserved for attach(always), and other modifier
10625/// // bits have no meaning for ATTACH). PRESENT is additionally
10626/// // propagated to components with HasAttachPtr (the pointee data) at
10627/// // OpenMP >= 6.0.
10628/// present_bit = (v60 && c.hasAttachPtr()) ? PRESENT : 0;
10629/// imported_modifier_bits =
10630/// type & (ALWAYS | DELETE | CLOSE | present_bit);
10631/// effective_type = c.isAttach() ? member_type
10632/// : member_type | imported_modifier_bits;
10633/// if (c.hasMapper())
10634/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
10635/// effective_type, c.arg_name);
10636/// else
10637/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
10638/// c.arg_begin, c.arg_size, effective_type,
10639/// c.arg_name);
10640/// }
10641/// }
10642/// // Delete the array section.
10643/// if (size > 1 && maptype.IsDelete)
10644/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10645/// size*sizeof(Ty), clearToFromMember(type));
10646/// }
10647/// \endcode
10648void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
10649 CodeGenFunction *CGF) {
10650 if (UDMMap.count(Val: D) > 0)
10651 return;
10652 ASTContext &C = CGM.getContext();
10653 QualType Ty = D->getType();
10654 auto *MapperVarDecl =
10655 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getMapperVarRef())->getDecl());
10656 CharUnits ElementSize = C.getTypeSizeInChars(T: Ty);
10657 llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
10658
10659 CodeGenFunction MapperCGF(CGM);
10660 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10661 auto PrivatizeAndGenMapInfoCB =
10662 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10663 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10664 MapperCGF.Builder.restoreIP(IP: CodeGenIP);
10665
10666 // Privatize the declared variable of mapper to be the current array
10667 // element.
10668 Address PtrCurrent(
10669 PtrPHI, ElemTy,
10670 Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign())
10671 .getAlignment()
10672 .alignmentOfArrayElement(elementSize: ElementSize));
10673 CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
10674 Scope.addPrivate(LocalVD: MapperVarDecl, Addr: PtrCurrent);
10675 (void)Scope.Privatize();
10676
10677 // Get map clause information.
10678 MappableExprsHandler MEHandler(*D, MapperCGF);
10679 MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder);
10680
10681 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10682 return emitMappingInformation(CGF&: MapperCGF, OMPBuilder, MapExprs&: MapExpr);
10683 };
10684 if (CGM.getCodeGenOpts().getDebugInfo() !=
10685 llvm::codegenoptions::NoDebugInfo) {
10686 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10687 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10688 F: FillInfoMap);
10689 }
10690
10691 return CombinedInfo;
10692 };
10693
10694 auto CustomMapperCB = [&](unsigned I) {
10695 llvm::Function *MapperFunc = nullptr;
10696 if (CombinedInfo.Mappers[I]) {
10697 // Call the corresponding mapper function.
10698 MapperFunc = getOrCreateUserDefinedMapperFunc(
10699 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10700 assert(MapperFunc && "Expect a valid mapper function is available.");
10701 }
10702 return MapperFunc;
10703 };
10704
10705 SmallString<64> TyStr;
10706 llvm::raw_svector_ostream Out(TyStr);
10707 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out);
10708 std::string Name = getName(Parts: {"omp_mapper", TyStr, D->getName()});
10709
10710 // Propagate the PRESENT modifier to the pointee entries (those with
10711 // HasAttachPtr) only for OpenMP >= 6.0; before 6.0 the present modifier does
10712 // not apply to the pointee (see the OpenMP 6.0 erratum on the present motion
10713 // vs. map-type modifier divergence).
10714 bool PropagatePresentToPointee = CGM.getLangOpts().OpenMP >= 60;
10715 llvm::Function *NewFn = cantFail(ValOrErr: OMPBuilder.emitUserDefinedMapper(
10716 PrivAndGenMapInfoCB: PrivatizeAndGenMapInfoCB, ElemTy, FuncName: Name, CustomMapperCB,
10717 /*PreserveMemberOfFlags=*/false, PropagatePresentToPointee));
10718 UDMMap.try_emplace(Key: D, Args&: NewFn);
10719 if (CGF)
10720 FunctionUDMMap[CGF->CurFn].push_back(Elt: D);
10721}
10722
10723llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc(
10724 const OMPDeclareMapperDecl *D) {
10725 auto I = UDMMap.find(Val: D);
10726 if (I != UDMMap.end())
10727 return I->second;
10728 emitUserDefinedMapper(D);
10729 return UDMMap.lookup(Val: D);
10730}
10731
10732llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall(
10733 CodeGenFunction &CGF, const OMPExecutableDirective &D,
10734 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10735 const OMPLoopDirective &D)>
10736 SizeEmitter) {
10737 OpenMPDirectiveKind Kind = D.getDirectiveKind();
10738 const OMPExecutableDirective *TD = &D;
10739 // Get nested teams distribute kind directive, if any. For now, treat
10740 // 'target_teams_loop' as if it's really a target_teams_distribute.
10741 if ((!isOpenMPDistributeDirective(DKind: Kind) || !isOpenMPTeamsDirective(DKind: Kind)) &&
10742 Kind != OMPD_target_teams_loop)
10743 TD = getNestedDistributeDirective(Ctx&: CGM.getContext(), D);
10744 if (!TD)
10745 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10746
10747 const auto *LD = cast<OMPLoopDirective>(Val: TD);
10748 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
10749 return NumIterations;
10750 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10751}
10752
10753static void
10754emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10755 const OMPExecutableDirective &D,
10756 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10757 bool RequiresOuterTask, const CapturedStmt &CS,
10758 bool OffloadingMandatory, CodeGenFunction &CGF) {
10759 if (OffloadingMandatory) {
10760 CGF.Builder.CreateUnreachable();
10761 } else {
10762 if (RequiresOuterTask) {
10763 CapturedVars.clear();
10764 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
10765 }
10766 llvm::SmallVector<llvm::Value *, 16> Args(CapturedVars.begin(),
10767 CapturedVars.end());
10768 Args.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy()));
10769 OMPRuntime->emitOutlinedFunctionCall(CGF, Loc: D.getBeginLoc(), OutlinedFn,
10770 Args);
10771 }
10772}
10773
10774static llvm::Value *emitDeviceID(
10775 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
10776 CodeGenFunction &CGF) {
10777 // Emit device ID if any.
10778 llvm::Value *DeviceID;
10779 if (Device.getPointer()) {
10780 assert((Device.getInt() == OMPC_DEVICE_unknown ||
10781 Device.getInt() == OMPC_DEVICE_device_num) &&
10782 "Expected device_num modifier.");
10783 llvm::Value *DevVal = CGF.EmitScalarExpr(E: Device.getPointer());
10784 DeviceID =
10785 CGF.Builder.CreateIntCast(V: DevVal, DestTy: CGF.Int64Ty, /*isSigned=*/true);
10786 } else {
10787 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
10788 }
10789 return DeviceID;
10790}
10791
10792static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>
10793emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) {
10794 llvm::Value *DynGP = CGF.Builder.getInt32(C: 0);
10795 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
10796
10797 if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) {
10798 CodeGenFunction::RunCleanupsScope DynGPScope(CGF);
10799 llvm::Value *DynGPVal =
10800 CGF.EmitScalarExpr(E: DynGPClause->getSize(), /*IgnoreResultAssign=*/true);
10801 DynGP = CGF.Builder.CreateIntCast(V: DynGPVal, DestTy: CGF.Int32Ty,
10802 /*isSigned=*/false);
10803 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();
10804 switch (FallbackModifier) {
10805 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:
10806 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
10807 break;
10808 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:
10809 DynGPFallback = OMPDynGroupprivateFallbackType::Null;
10810 break;
10811 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:
10812 case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown:
10813 // This is the default for dyn_groupprivate.
10814 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;
10815 break;
10816 default:
10817 llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate");
10818 }
10819 } else if (auto *OMPXDynCGClause =
10820 D.getSingleClause<OMPXDynCGroupMemClause>()) {
10821 CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF);
10822 llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(E: OMPXDynCGClause->getSize(),
10823 /*IgnoreResultAssign=*/true);
10824 DynGP = CGF.Builder.CreateIntCast(V: DynCGMemVal, DestTy: CGF.Int32Ty,
10825 /*isSigned=*/false);
10826 }
10827 return {DynGP, DynGPFallback};
10828}
10829
10830static void genMapInfoForCaptures(
10831 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
10832 const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10833 llvm::OpenMPIRBuilder &OMPBuilder,
10834 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet,
10835 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10836
10837 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
10838 auto RI = CS.getCapturedRecordDecl()->field_begin();
10839 auto *CV = CapturedVars.begin();
10840 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
10841 CE = CS.capture_end();
10842 CI != CE; ++CI, ++RI, ++CV) {
10843 MappableExprsHandler::MapCombinedInfoTy CurInfo;
10844
10845 // VLA sizes are passed to the outlined region by copy and do not have map
10846 // information associated.
10847 if (CI->capturesVariableArrayType()) {
10848 CurInfo.Exprs.push_back(Elt: nullptr);
10849 CurInfo.BasePointers.push_back(Elt: *CV);
10850 CurInfo.DevicePtrDecls.push_back(Elt: nullptr);
10851 CurInfo.DevicePointers.push_back(
10852 Elt: MappableExprsHandler::DeviceInfoTy::None);
10853 CurInfo.Pointers.push_back(Elt: *CV);
10854 CurInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10855 V: CGF.getTypeSize(Ty: RI->getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10856 // Copy to the device as an argument. No need to retrieve it.
10857 CurInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10858 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
10859 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
10860 CurInfo.HasAttachPtr.push_back(Elt: false);
10861 CurInfo.Mappers.push_back(Elt: nullptr);
10862 } else {
10863 const ValueDecl *CapturedVD =
10864 CI->capturesThis() ? nullptr
10865 : CI->getCapturedVar()->getCanonicalDecl();
10866 bool HasEntryWithCVAsAttachPtr = false;
10867 if (CapturedVD)
10868 HasEntryWithCVAsAttachPtr =
10869 MEHandler.hasAttachEntryForCapturedVar(VD: CapturedVD);
10870
10871 // Populate component lists for the captured variable from clauses.
10872 MappableExprsHandler::MapDataArrayTy DeclComponentLists;
10873 SmallVector<
10874 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>, 4>
10875 StorageForImplicitlyAddedComponentLists;
10876 MEHandler.populateComponentListsForNonLambdaCaptureFromClauses(
10877 VD: CapturedVD, DeclComponentLists,
10878 StorageForImplicitlyAddedComponentLists);
10879
10880 // OpenMP 6.0, 15.8, target construct, restrictions:
10881 // * A list item in a map clause that is specified on a target construct
10882 // must have a base variable or base pointer.
10883 //
10884 // Map clauses on a target construct must either have a base pointer, or a
10885 // base-variable. So, if we don't have a base-pointer, that means that it
10886 // must have a base-variable, i.e. we have a map like `map(s)`, `map(s.x)`
10887 // etc. In such cases, we do not need to handle default map generation
10888 // for `s`.
10889 bool HasEntryWithoutAttachPtr =
10890 llvm::any_of(Range&: DeclComponentLists, P: [&](const auto &MapData) {
10891 OMPClauseMappableExprCommon::MappableExprComponentListRef
10892 Components = std::get<0>(MapData);
10893 return !MEHandler.getAttachPtrExpr(Components);
10894 });
10895
10896 // Generate default map info first if there's no direct map with CV as
10897 // the base-variable, or attach pointer.
10898 if (DeclComponentLists.empty() ||
10899 (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr))
10900 MEHandler.generateDefaultMapInfo(CI: *CI, RI: **RI, CV: *CV, CombinedInfo&: CurInfo);
10901
10902 // If we have any information in the map clause, we use it, otherwise we
10903 // just do a default mapping.
10904 MEHandler.generateInfoForCaptureFromClauseInfo(
10905 DeclComponentListsFromClauses: DeclComponentLists, Cap: CI, Arg: *CV, CurCaptureVarInfo&: CurInfo, OMPBuilder,
10906 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());
10907
10908 if (!CI->capturesThis())
10909 MappedVarSet.insert(V: CI->getCapturedVar());
10910 else
10911 MappedVarSet.insert(V: nullptr);
10912
10913 // Generate correct mapping for variables captured by reference in
10914 // lambdas.
10915 if (CI->capturesVariable())
10916 MEHandler.generateInfoForLambdaCaptures(VD: CI->getCapturedVar(), Arg: *CV,
10917 CombinedInfo&: CurInfo, LambdaPointers);
10918 }
10919 // We expect to have at least an element of information for this capture.
10920 assert(!CurInfo.BasePointers.empty() &&
10921 "Non-existing map pointer for capture!");
10922 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
10923 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
10924 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
10925 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
10926 "Inconsistent map information sizes!");
10927
10928 // We need to append the results of this capture to what we already have.
10929 CombinedInfo.append(CurInfo);
10930 }
10931 // Adjust MEMBER_OF flags for the lambdas captures.
10932 MEHandler.adjustMemberOfForLambdaCaptures(
10933 OMPBuilder, LambdaPointers, BasePointers&: CombinedInfo.BasePointers,
10934 Pointers&: CombinedInfo.Pointers, Types&: CombinedInfo.Types);
10935}
10936static void
10937genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
10938 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10939 llvm::OpenMPIRBuilder &OMPBuilder,
10940 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet =
10941 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) {
10942
10943 CodeGenModule &CGM = CGF.CGM;
10944 // Map any list items in a map clause that were not captures because they
10945 // weren't referenced within the construct.
10946 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkipVarSet: SkippedVarSet);
10947
10948 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10949 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
10950 };
10951 if (CGM.getCodeGenOpts().getDebugInfo() !=
10952 llvm::codegenoptions::NoDebugInfo) {
10953 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10954 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10955 F: FillInfoMap);
10956 }
10957}
10958
10959static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
10960 const CapturedStmt &CS,
10961 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10962 llvm::OpenMPIRBuilder &OMPBuilder,
10963 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10964 // Get mappable expression information.
10965 MappableExprsHandler MEHandler(D, CGF);
10966 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
10967
10968 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,
10969 MappedVarSet, CombinedInfo);
10970 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, SkippedVarSet: MappedVarSet);
10971}
10972
10973template <typename ClauseTy>
10974static void
10975emitClauseForBareTargetDirective(CodeGenFunction &CGF,
10976 const OMPExecutableDirective &D,
10977 llvm::SmallVectorImpl<llvm::Value *> &Values) {
10978 const auto *C = D.getSingleClause<ClauseTy>();
10979 assert(!C->varlist_empty() &&
10980 "ompx_bare requires explicit num_teams and thread_limit");
10981 CodeGenFunction::RunCleanupsScope Scope(CGF);
10982 for (auto *E : C->varlist()) {
10983 llvm::Value *V = CGF.EmitScalarExpr(E);
10984 Values.push_back(
10985 Elt: CGF.Builder.CreateIntCast(V, DestTy: CGF.Int32Ty, /*isSigned=*/true));
10986 }
10987}
10988
10989static void emitTargetCallKernelLaunch(
10990 CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10991 const OMPExecutableDirective &D,
10992 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask,
10993 const CapturedStmt &CS, bool OffloadingMandatory,
10994 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
10995 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
10996 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
10997 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10998 const OMPLoopDirective &D)>
10999 SizeEmitter,
11000 CodeGenFunction &CGF, CodeGenModule &CGM) {
11001 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();
11002
11003 // Fill up the arrays with all the captured variables.
11004 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11005 CGOpenMPRuntime::TargetDataInfo Info;
11006 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
11007
11008 // Append a null entry for the implicit dyn_ptr argument.
11009 using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
11010 auto *NullPtr = llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy());
11011 CombinedInfo.BasePointers.push_back(Elt: NullPtr);
11012 CombinedInfo.Pointers.push_back(Elt: NullPtr);
11013 CombinedInfo.DevicePointers.push_back(
11014 Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
11015 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.getInt64(C: 0));
11016 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11017 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
11018 CombinedInfo.HasAttachPtr.push_back(Elt: false);
11019 if (!CombinedInfo.Names.empty())
11020 CombinedInfo.Names.push_back(Elt: NullPtr);
11021 CombinedInfo.Exprs.push_back(Elt: nullptr);
11022 CombinedInfo.Mappers.push_back(Elt: nullptr);
11023 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
11024
11025 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11026 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11027
11028 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11029 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11030 CGF.VoidPtrTy, CGM.getPointerAlign());
11031 InputInfo.PointersArray =
11032 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11033 InputInfo.SizesArray =
11034 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11035 InputInfo.MappersArray =
11036 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11037 MapTypesArray = Info.RTArgs.MapTypesArray;
11038 MapNamesArray = Info.RTArgs.MapNamesArray;
11039
11040 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
11041 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11042 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
11043 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
11044 bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
11045
11046 if (IsReverseOffloading) {
11047 // Reverse offloading is not supported, so just execute on the host.
11048 // FIXME: This fallback solution is incorrect since it ignores the
11049 // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to
11050 // assert here and ensure SEMA emits an error.
11051 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11052 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11053 return;
11054 }
11055
11056 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
11057 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
11058
11059 llvm::Value *BasePointersArray =
11060 InputInfo.BasePointersArray.emitRawPointer(CGF);
11061 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
11062 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
11063 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
11064
11065 auto &&EmitTargetCallFallbackCB =
11066 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11067 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
11068 -> llvm::OpenMPIRBuilder::InsertPointTy {
11069 CGF.Builder.restoreIP(IP);
11070 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11071 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11072 return CGF.Builder.saveIP();
11073 };
11074
11075 bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
11076 SmallVector<llvm::Value *, 3> NumTeams;
11077 SmallVector<llvm::Value *, 3> NumThreads;
11078 if (IsBare) {
11079 emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, Values&: NumTeams);
11080 emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
11081 Values&: NumThreads);
11082 } else {
11083 NumTeams.push_back(Elt: OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
11084 NumThreads.push_back(
11085 Elt: OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
11086 }
11087
11088 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
11089 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11090 llvm::Value *NumIterations =
11091 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
11092 auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF);
11093 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
11094 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
11095
11096 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
11097 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
11098 nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray);
11099
11100 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
11101 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
11102 DynCGroupMem, HasNoWait, /*StrictBlocks=*/IsBare,
11103 /*StrictThreads=*/IsBare, DynCGroupMemFallback);
11104
11105 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11106 cantFail(ValOrErr: OMPRuntime->getOMPBuilder().emitKernelLaunch(
11107 Loc: CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
11108 RTLoc, AllocaIP));
11109 CGF.Builder.restoreIP(IP: AfterIP);
11110 };
11111
11112 if (RequiresOuterTask)
11113 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11114 else
11115 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11116}
11117
11118static void
11119emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11120 const OMPExecutableDirective &D,
11121 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11122 bool RequiresOuterTask, const CapturedStmt &CS,
11123 bool OffloadingMandatory, CodeGenFunction &CGF) {
11124
11125 // Notify that the host version must be executed.
11126 auto &&ElseGen =
11127 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11128 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11129 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11130 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11131 };
11132
11133 if (RequiresOuterTask) {
11134 CodeGenFunction::OMPTargetDataInfo InputInfo;
11135 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ElseGen, InputInfo);
11136 } else {
11137 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ElseGen);
11138 }
11139}
11140
11141void CGOpenMPRuntime::emitTargetCall(
11142 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11143 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11144 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11145 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11146 const OMPLoopDirective &D)>
11147 SizeEmitter) {
11148 if (!CGF.HaveInsertPoint())
11149 return;
11150
11151 const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice &&
11152 CGM.getLangOpts().OpenMPOffloadMandatory;
11153
11154 assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
11155
11156 const bool RequiresOuterTask =
11157 D.hasClausesOfKind<OMPDependClause>() ||
11158 D.hasClausesOfKind<OMPNowaitClause>() ||
11159 D.hasClausesOfKind<OMPInReductionClause>() ||
11160 (CGM.getLangOpts().OpenMP >= 51 &&
11161 needsTaskBasedThreadLimit(DKind: D.getDirectiveKind()) &&
11162 D.hasClausesOfKind<OMPThreadLimitClause>());
11163 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
11164 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
11165 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
11166 PrePostActionTy &) {
11167 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
11168 };
11169 emitInlinedDirective(CGF, InnerKind: OMPD_unknown, CodeGen: ArgsCodegen);
11170
11171 CodeGenFunction::OMPTargetDataInfo InputInfo;
11172 llvm::Value *MapTypesArray = nullptr;
11173 llvm::Value *MapNamesArray = nullptr;
11174
11175 auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,
11176 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11177 OutlinedFnID, &InputInfo, &MapTypesArray,
11178 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
11179 PrePostActionTy &) {
11180 emitTargetCallKernelLaunch(OMPRuntime: this, OutlinedFn, D, CapturedVars,
11181 RequiresOuterTask, CS, OffloadingMandatory,
11182 Device, OutlinedFnID, InputInfo, MapTypesArray,
11183 MapNamesArray, SizeEmitter, CGF, CGM);
11184 };
11185
11186 auto &&TargetElseGen =
11187 [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11188 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11189 emitTargetCallElse(OMPRuntime: this, OutlinedFn, D, CapturedVars, RequiresOuterTask,
11190 CS, OffloadingMandatory, CGF);
11191 };
11192
11193 // If we have a target function ID it means that we need to support
11194 // offloading, otherwise, just execute on the host. We need to execute on host
11195 // regardless of the conditional in the if clause if, e.g., the user do not
11196 // specify target triples.
11197 if (OutlinedFnID) {
11198 if (IfCond) {
11199 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen, ElseGen: TargetElseGen);
11200 } else {
11201 RegionCodeGenTy ThenRCG(TargetThenGen);
11202 ThenRCG(CGF);
11203 }
11204 } else {
11205 RegionCodeGenTy ElseRCG(TargetElseGen);
11206 ElseRCG(CGF);
11207 }
11208}
11209
11210void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
11211 StringRef ParentName) {
11212 if (!S)
11213 return;
11214
11215 // Register vtable from device for target data and target directives.
11216 // Add this block here since scanForTargetRegionsFunctions ignores
11217 // target data by checking if S is a executable directive (target).
11218 if (auto *E = dyn_cast<OMPExecutableDirective>(Val: S);
11219 E && isOpenMPTargetDataManagementDirective(DKind: E->getDirectiveKind())) {
11220 // Don't need to check if it's device compile
11221 // since scanForTargetRegionsFunctions currently only called
11222 // in device compilation.
11223 registerVTable(D: *E);
11224 }
11225
11226 // Codegen OMP target directives that offload compute to the device.
11227 bool RequiresDeviceCodegen =
11228 isa<OMPExecutableDirective>(Val: S) &&
11229 isOpenMPTargetExecutionDirective(
11230 DKind: cast<OMPExecutableDirective>(Val: S)->getDirectiveKind());
11231
11232 if (RequiresDeviceCodegen) {
11233 const auto &E = *cast<OMPExecutableDirective>(Val: S);
11234
11235 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
11236 CGM, OMPBuilder, BeginLoc: E.getBeginLoc(), ParentName);
11237
11238 // Is this a target region that should not be emitted as an entry point? If
11239 // so just signal we are done with this target region.
11240 if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
11241 return;
11242
11243 switch (E.getDirectiveKind()) {
11244 case OMPD_target:
11245 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
11246 S: cast<OMPTargetDirective>(Val: E));
11247 break;
11248 case OMPD_target_parallel:
11249 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
11250 CGM, ParentName, S: cast<OMPTargetParallelDirective>(Val: E));
11251 break;
11252 case OMPD_target_teams:
11253 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
11254 CGM, ParentName, S: cast<OMPTargetTeamsDirective>(Val: E));
11255 break;
11256 case OMPD_target_teams_distribute:
11257 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
11258 CGM, ParentName, S: cast<OMPTargetTeamsDistributeDirective>(Val: E));
11259 break;
11260 case OMPD_target_teams_distribute_simd:
11261 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
11262 CGM, ParentName, S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: E));
11263 break;
11264 case OMPD_target_parallel_for:
11265 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
11266 CGM, ParentName, S: cast<OMPTargetParallelForDirective>(Val: E));
11267 break;
11268 case OMPD_target_parallel_for_simd:
11269 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
11270 CGM, ParentName, S: cast<OMPTargetParallelForSimdDirective>(Val: E));
11271 break;
11272 case OMPD_target_simd:
11273 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
11274 CGM, ParentName, S: cast<OMPTargetSimdDirective>(Val: E));
11275 break;
11276 case OMPD_target_teams_distribute_parallel_for:
11277 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
11278 CGM, ParentName,
11279 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: E));
11280 break;
11281 case OMPD_target_teams_distribute_parallel_for_simd:
11282 CodeGenFunction::
11283 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
11284 CGM, ParentName,
11285 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: E));
11286 break;
11287 case OMPD_target_teams_loop:
11288 CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
11289 CGM, ParentName, S: cast<OMPTargetTeamsGenericLoopDirective>(Val: E));
11290 break;
11291 case OMPD_target_parallel_loop:
11292 CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
11293 CGM, ParentName, S: cast<OMPTargetParallelGenericLoopDirective>(Val: E));
11294 break;
11295 case OMPD_parallel:
11296 case OMPD_for:
11297 case OMPD_parallel_for:
11298 case OMPD_parallel_master:
11299 case OMPD_parallel_sections:
11300 case OMPD_for_simd:
11301 case OMPD_parallel_for_simd:
11302 case OMPD_cancel:
11303 case OMPD_cancellation_point:
11304 case OMPD_ordered_standalone:
11305 case OMPD_ordered_blockassoc:
11306 case OMPD_threadprivate:
11307 case OMPD_allocate:
11308 case OMPD_task:
11309 case OMPD_simd:
11310 case OMPD_tile:
11311 case OMPD_unroll:
11312 case OMPD_sections:
11313 case OMPD_section:
11314 case OMPD_single:
11315 case OMPD_master:
11316 case OMPD_critical:
11317 case OMPD_taskyield:
11318 case OMPD_barrier:
11319 case OMPD_taskwait:
11320 case OMPD_taskgroup:
11321 case OMPD_atomic:
11322 case OMPD_flush:
11323 case OMPD_depobj:
11324 case OMPD_scan:
11325 case OMPD_teams:
11326 case OMPD_target_data:
11327 case OMPD_target_exit_data:
11328 case OMPD_target_enter_data:
11329 case OMPD_distribute:
11330 case OMPD_distribute_simd:
11331 case OMPD_distribute_parallel_for:
11332 case OMPD_distribute_parallel_for_simd:
11333 case OMPD_teams_distribute:
11334 case OMPD_teams_distribute_simd:
11335 case OMPD_teams_distribute_parallel_for:
11336 case OMPD_teams_distribute_parallel_for_simd:
11337 case OMPD_target_update:
11338 case OMPD_declare_simd:
11339 case OMPD_declare_variant:
11340 case OMPD_begin_declare_variant:
11341 case OMPD_end_declare_variant:
11342 case OMPD_declare_target:
11343 case OMPD_end_declare_target:
11344 case OMPD_declare_reduction:
11345 case OMPD_declare_mapper:
11346 case OMPD_taskloop:
11347 case OMPD_taskloop_simd:
11348 case OMPD_master_taskloop:
11349 case OMPD_master_taskloop_simd:
11350 case OMPD_parallel_master_taskloop:
11351 case OMPD_parallel_master_taskloop_simd:
11352 case OMPD_requires:
11353 case OMPD_metadirective:
11354 case OMPD_unknown:
11355 default:
11356 llvm_unreachable("Unknown target directive for OpenMP device codegen.");
11357 }
11358 return;
11359 }
11360
11361 if (const auto *E = dyn_cast<OMPExecutableDirective>(Val: S)) {
11362 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
11363 return;
11364
11365 scanForTargetRegionsFunctions(S: E->getRawStmt(), ParentName);
11366 return;
11367 }
11368
11369 // If this is a lambda function, look into its body.
11370 if (const auto *L = dyn_cast<LambdaExpr>(Val: S))
11371 S = L->getBody();
11372
11373 // Keep looking for target regions recursively.
11374 for (const Stmt *II : S->children())
11375 scanForTargetRegionsFunctions(S: II, ParentName);
11376}
11377
11378static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) {
11379 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
11380 OMPDeclareTargetDeclAttr::getDeviceType(VD);
11381 if (!DevTy)
11382 return false;
11383 // Do not emit device_type(nohost) functions for the host.
11384 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
11385 return true;
11386 // Do not emit device_type(host) functions for the device.
11387 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
11388 return true;
11389 return false;
11390}
11391
11392bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
11393 // If emitting code for the host, we do not process FD here. Instead we do
11394 // the normal code generation.
11395 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
11396 if (const auto *FD = dyn_cast<FunctionDecl>(Val: GD.getDecl()))
11397 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11398 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11399 return true;
11400 return false;
11401 }
11402
11403 const ValueDecl *VD = cast<ValueDecl>(Val: GD.getDecl());
11404 // Try to detect target regions in the function.
11405 if (const auto *FD = dyn_cast<FunctionDecl>(Val: VD)) {
11406 StringRef Name = CGM.getMangledName(GD);
11407 scanForTargetRegionsFunctions(S: FD->getBody(), ParentName: Name);
11408 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11409 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11410 return true;
11411 }
11412
11413 // Do not emit function if it is not marked as declare target.
11414 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
11415 AlreadyEmittedTargetDecls.count(V: VD) == 0;
11416}
11417
11418bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
11419 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: GD.getDecl()),
11420 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11421 return true;
11422
11423 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
11424 return false;
11425
11426 // Check if there are Ctors/Dtors in this declaration and look for target
11427 // regions in it. We use the complete variant to produce the kernel name
11428 // mangling.
11429 QualType RDTy = cast<VarDecl>(Val: GD.getDecl())->getType();
11430 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
11431 for (const CXXConstructorDecl *Ctor : RD->ctors()) {
11432 StringRef ParentName =
11433 CGM.getMangledName(GD: GlobalDecl(Ctor, Ctor_Complete));
11434 scanForTargetRegionsFunctions(S: Ctor->getBody(), ParentName);
11435 }
11436 if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
11437 StringRef ParentName =
11438 CGM.getMangledName(GD: GlobalDecl(Dtor, Dtor_Complete));
11439 scanForTargetRegionsFunctions(S: Dtor->getBody(), ParentName);
11440 }
11441 }
11442
11443 // Do not emit variable if it is not marked as declare target.
11444 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11445 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
11446 VD: cast<VarDecl>(Val: GD.getDecl()));
11447 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
11448 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11449 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11450 HasRequiresUnifiedSharedMemory)) {
11451 DeferredGlobalVariables.insert(V: cast<VarDecl>(Val: GD.getDecl()));
11452 return true;
11453 }
11454 return false;
11455}
11456
11457void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
11458 llvm::Constant *Addr) {
11459 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
11460 !CGM.getLangOpts().OpenMPIsTargetDevice)
11461 return;
11462
11463 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11464 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11465
11466 // If this is an 'extern' declaration we defer to the canonical definition and
11467 // do not emit an offloading entry.
11468 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
11469 VD->hasExternalStorage())
11470 return;
11471
11472 // MT_Local variables use direct access with no host-device mapping.
11473 // No offload entry needed — the device global keeps its own initializer.
11474 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local)
11475 return;
11476
11477 if (!Res) {
11478 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11479 // Register non-target variables being emitted in device code (debug info
11480 // may cause this).
11481 StringRef VarName = CGM.getMangledName(GD: VD);
11482 EmittedNonTargetVariables.try_emplace(Key: VarName, Args&: Addr);
11483 }
11484 return;
11485 }
11486
11487 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
11488 auto LinkageForVariable = [&VD, this]() {
11489 return CGM.getLLVMLinkageVarDefinition(VD);
11490 };
11491
11492 std::vector<llvm::GlobalVariable *> GeneratedRefs;
11493 OMPBuilder.registerTargetGlobalVariable(
11494 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
11495 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
11496 IsExternallyVisible: VD->isExternallyVisible(),
11497 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
11498 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
11499 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
11500 TargetTriple: CGM.getLangOpts().OMPTargetTriples, GlobalInitializer: AddrOfGlobal, VariableLinkage: LinkageForVariable,
11501 LlvmPtrTy: CGM.getTypes().ConvertTypeForMem(
11502 T: CGM.getContext().getPointerType(T: VD->getType())),
11503 Addr);
11504
11505 for (auto *ref : GeneratedRefs)
11506 CGM.addCompilerUsedGlobal(GV: ref);
11507}
11508
11509bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
11510 if (isa<FunctionDecl>(Val: GD.getDecl()) ||
11511 isa<OMPDeclareReductionDecl>(Val: GD.getDecl()))
11512 return emitTargetFunctions(GD);
11513
11514 return emitTargetGlobalVariable(GD);
11515}
11516
11517void CGOpenMPRuntime::emitDeferredTargetDecls() const {
11518 for (const VarDecl *VD : DeferredGlobalVariables) {
11519 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11520 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11521 if (!Res)
11522 continue;
11523 // MT_Local and MT_To/MT_Enter without USM are always emitted.
11524 if (*Res == OMPDeclareTargetDeclAttr::MT_Local ||
11525 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11526 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11527 !HasRequiresUnifiedSharedMemory)) {
11528 CGM.EmitGlobal(D: VD);
11529 } else {
11530 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
11531 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11532 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
11533 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
11534 HasRequiresUnifiedSharedMemory)) &&
11535 "Expected link clause or to clause with unified memory.");
11536 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
11537 }
11538 }
11539}
11540
11541void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
11542 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
11543 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
11544 " Expected target-based directive.");
11545}
11546
11547void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) {
11548 for (const OMPClause *Clause : D->clauselists()) {
11549 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
11550 HasRequiresUnifiedSharedMemory = true;
11551 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
11552 } else if (const auto *AC =
11553 dyn_cast<OMPAtomicDefaultMemOrderClause>(Val: Clause)) {
11554 switch (AC->getAtomicDefaultMemOrderKind()) {
11555 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
11556 RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;
11557 break;
11558 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
11559 RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
11560 break;
11561 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
11562 RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
11563 break;
11564 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown:
11565 break;
11566 }
11567 }
11568 }
11569}
11570
11571llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {
11572 return RequiresAtomicOrdering;
11573}
11574
11575bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
11576 LangAS &AS) {
11577 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
11578 return false;
11579 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
11580 switch(A->getAllocatorType()) {
11581 case OMPAllocateDeclAttr::OMPNullMemAlloc:
11582 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
11583 // Not supported, fallback to the default mem space.
11584 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
11585 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
11586 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
11587 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
11588 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
11589 case OMPAllocateDeclAttr::OMPConstMemAlloc:
11590 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
11591 AS = LangAS::Default;
11592 return true;
11593 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
11594 llvm_unreachable("Expected predefined allocator for the variables with the "
11595 "static storage.");
11596 }
11597 return false;
11598}
11599
11600bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {
11601 return HasRequiresUnifiedSharedMemory;
11602}
11603
11604CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
11605 CodeGenModule &CGM)
11606 : CGM(CGM) {
11607 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11608 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
11609 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
11610 }
11611}
11612
11613CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
11614 if (CGM.getLangOpts().OpenMPIsTargetDevice)
11615 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
11616}
11617
11618bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
11619 if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal)
11620 return true;
11621
11622 const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
11623 // Do not emit function if it is marked as declare target as it was already
11624 // emitted.
11625 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: D)) {
11626 if (D->hasBody() && AlreadyEmittedTargetDecls.count(V: D) == 0) {
11627 if (auto *F = dyn_cast_or_null<llvm::Function>(
11628 Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD))))
11629 return !F->isDeclaration();
11630 return false;
11631 }
11632 return true;
11633 }
11634
11635 return !AlreadyEmittedTargetDecls.insert(V: D).second;
11636}
11637
11638void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
11639 const OMPExecutableDirective &D,
11640 SourceLocation Loc,
11641 llvm::Function *OutlinedFn,
11642 ArrayRef<llvm::Value *> CapturedVars) {
11643 if (!CGF.HaveInsertPoint())
11644 return;
11645
11646 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11647 CodeGenFunction::RunCleanupsScope Scope(CGF);
11648
11649 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
11650 llvm::Value *Args[] = {
11651 RTLoc,
11652 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
11653 OutlinedFn};
11654 llvm::SmallVector<llvm::Value *, 16> RealArgs;
11655 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
11656 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
11657
11658 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
11659 M&: CGM.getModule(), FnID: OMPRTL___kmpc_fork_teams);
11660 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
11661}
11662
11663void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11664 const Expr *NumTeams,
11665 const Expr *ThreadLimit,
11666 SourceLocation Loc) {
11667 if (!CGF.HaveInsertPoint())
11668 return;
11669
11670 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11671
11672 llvm::Value *NumTeamsVal =
11673 NumTeams
11674 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: NumTeams),
11675 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11676 : CGF.Builder.getInt32(C: 0);
11677
11678 llvm::Value *ThreadLimitVal =
11679 ThreadLimit
11680 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11681 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11682 : CGF.Builder.getInt32(C: 0);
11683
11684 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
11685 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
11686 ThreadLimitVal};
11687 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11688 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_num_teams),
11689 args: PushNumTeamsArgs);
11690}
11691
11692void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
11693 const Expr *ThreadLimit,
11694 SourceLocation Loc) {
11695 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11696 llvm::Value *ThreadLimitVal =
11697 ThreadLimit
11698 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11699 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11700 : CGF.Builder.getInt32(C: 0);
11701
11702 // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit)
11703 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
11704 ThreadLimitVal};
11705 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11706 M&: CGM.getModule(), FnID: OMPRTL___kmpc_set_thread_limit),
11707 args: ThreadLimitArgs);
11708}
11709
11710void CGOpenMPRuntime::emitTargetDataCalls(
11711 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11712 const Expr *Device, const RegionCodeGenTy &CodeGen,
11713 CGOpenMPRuntime::TargetDataInfo &Info) {
11714 if (!CGF.HaveInsertPoint())
11715 return;
11716
11717 // Action used to replace the default codegen action and turn privatization
11718 // off.
11719 PrePostActionTy NoPrivAction;
11720
11721 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
11722
11723 llvm::Value *IfCondVal = nullptr;
11724 if (IfCond)
11725 IfCondVal = CGF.EvaluateExprAsBool(E: IfCond);
11726
11727 // Emit device ID if any.
11728 llvm::Value *DeviceID = nullptr;
11729 if (Device) {
11730 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
11731 DestTy: CGF.Int64Ty, /*isSigned=*/true);
11732 } else {
11733 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11734 }
11735
11736 // Fill up the arrays with all the mapped variables.
11737 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11738 auto GenMapInfoCB =
11739 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
11740 CGF.Builder.restoreIP(IP: CodeGenIP);
11741 // Get map clause information.
11742 MappableExprsHandler MEHandler(D, CGF);
11743 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);
11744
11745 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11746 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
11747 };
11748 if (CGM.getCodeGenOpts().getDebugInfo() !=
11749 llvm::codegenoptions::NoDebugInfo) {
11750 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
11751 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
11752 F: FillInfoMap);
11753 }
11754
11755 return CombinedInfo;
11756 };
11757 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
11758 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
11759 CGF.Builder.restoreIP(IP: CodeGenIP);
11760 switch (BodyGenType) {
11761 case BodyGenTy::Priv:
11762 if (!Info.CaptureDeviceAddrMap.empty())
11763 CodeGen(CGF);
11764 break;
11765 case BodyGenTy::DupNoPriv:
11766 if (!Info.CaptureDeviceAddrMap.empty()) {
11767 CodeGen.setAction(NoPrivAction);
11768 CodeGen(CGF);
11769 }
11770 break;
11771 case BodyGenTy::NoPriv:
11772 if (Info.CaptureDeviceAddrMap.empty()) {
11773 CodeGen.setAction(NoPrivAction);
11774 CodeGen(CGF);
11775 }
11776 break;
11777 }
11778 return InsertPointTy(CGF.Builder.GetInsertBlock(),
11779 CGF.Builder.GetInsertPoint());
11780 };
11781
11782 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
11783 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
11784 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
11785 }
11786 };
11787
11788 auto CustomMapperCB = [&](unsigned int I) {
11789 llvm::Function *MFunc = nullptr;
11790 if (CombinedInfo.Mappers[I]) {
11791 Info.HasMapper = true;
11792 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
11793 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
11794 }
11795 return MFunc;
11796 };
11797
11798 // Source location for the ident struct
11799 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11800
11801 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
11802 CGF.AllocaInsertPt->getIterator());
11803 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
11804 CGF.Builder.GetInsertPoint());
11805 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);
11806 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11807 cantFail(ValOrErr: OMPBuilder.createTargetData(
11808 Loc: OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
11809 IfCond: IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
11810 /*MapperFunc=*/nullptr, BodyGenCB: BodyCB, DeviceAddrCB, SrcLocInfo: RTLoc));
11811 CGF.Builder.restoreIP(IP: AfterIP);
11812}
11813
11814void CGOpenMPRuntime::emitTargetDataStandAloneCall(
11815 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11816 const Expr *Device) {
11817 if (!CGF.HaveInsertPoint())
11818 return;
11819
11820 assert((isa<OMPTargetEnterDataDirective>(D) ||
11821 isa<OMPTargetExitDataDirective>(D) ||
11822 isa<OMPTargetUpdateDirective>(D)) &&
11823 "Expecting either target enter, exit data, or update directives.");
11824
11825 CodeGenFunction::OMPTargetDataInfo InputInfo;
11826 llvm::Value *MapTypesArray = nullptr;
11827 llvm::Value *MapNamesArray = nullptr;
11828 // Generate the code for the opening of the data environment.
11829 auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,
11830 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
11831 // Emit device ID if any.
11832 llvm::Value *DeviceID = nullptr;
11833 if (Device) {
11834 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
11835 DestTy: CGF.Int64Ty, /*isSigned=*/true);
11836 } else {
11837 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11838 }
11839
11840 // Emit the number of elements in the offloading arrays.
11841 llvm::Constant *PointerNum =
11842 CGF.Builder.getInt32(C: InputInfo.NumberOfTargetItems);
11843
11844 // Source location for the ident struct
11845 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11846
11847 SmallVector<llvm::Value *, 13> OffloadingArgs(
11848 {RTLoc, DeviceID, PointerNum,
11849 InputInfo.BasePointersArray.emitRawPointer(CGF),
11850 InputInfo.PointersArray.emitRawPointer(CGF),
11851 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
11852 InputInfo.MappersArray.emitRawPointer(CGF)});
11853
11854 // Select the right runtime function call for each standalone
11855 // directive.
11856 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
11857 RuntimeFunction RTLFn;
11858 switch (D.getDirectiveKind()) {
11859 case OMPD_target_enter_data:
11860 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
11861 : OMPRTL___tgt_target_data_begin_mapper;
11862 break;
11863 case OMPD_target_exit_data:
11864 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
11865 : OMPRTL___tgt_target_data_end_mapper;
11866 break;
11867 case OMPD_target_update:
11868 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
11869 : OMPRTL___tgt_target_data_update_mapper;
11870 break;
11871 case OMPD_parallel:
11872 case OMPD_for:
11873 case OMPD_parallel_for:
11874 case OMPD_parallel_master:
11875 case OMPD_parallel_sections:
11876 case OMPD_for_simd:
11877 case OMPD_parallel_for_simd:
11878 case OMPD_cancel:
11879 case OMPD_cancellation_point:
11880 case OMPD_ordered_standalone:
11881 case OMPD_ordered_blockassoc:
11882 case OMPD_threadprivate:
11883 case OMPD_allocate:
11884 case OMPD_task:
11885 case OMPD_simd:
11886 case OMPD_tile:
11887 case OMPD_unroll:
11888 case OMPD_sections:
11889 case OMPD_section:
11890 case OMPD_single:
11891 case OMPD_master:
11892 case OMPD_critical:
11893 case OMPD_taskyield:
11894 case OMPD_barrier:
11895 case OMPD_taskwait:
11896 case OMPD_taskgroup:
11897 case OMPD_atomic:
11898 case OMPD_flush:
11899 case OMPD_depobj:
11900 case OMPD_scan:
11901 case OMPD_teams:
11902 case OMPD_target_data:
11903 case OMPD_distribute:
11904 case OMPD_distribute_simd:
11905 case OMPD_distribute_parallel_for:
11906 case OMPD_distribute_parallel_for_simd:
11907 case OMPD_teams_distribute:
11908 case OMPD_teams_distribute_simd:
11909 case OMPD_teams_distribute_parallel_for:
11910 case OMPD_teams_distribute_parallel_for_simd:
11911 case OMPD_declare_simd:
11912 case OMPD_declare_variant:
11913 case OMPD_begin_declare_variant:
11914 case OMPD_end_declare_variant:
11915 case OMPD_declare_target:
11916 case OMPD_end_declare_target:
11917 case OMPD_declare_reduction:
11918 case OMPD_declare_mapper:
11919 case OMPD_taskloop:
11920 case OMPD_taskloop_simd:
11921 case OMPD_master_taskloop:
11922 case OMPD_master_taskloop_simd:
11923 case OMPD_parallel_master_taskloop:
11924 case OMPD_parallel_master_taskloop_simd:
11925 case OMPD_target:
11926 case OMPD_target_simd:
11927 case OMPD_target_teams_distribute:
11928 case OMPD_target_teams_distribute_simd:
11929 case OMPD_target_teams_distribute_parallel_for:
11930 case OMPD_target_teams_distribute_parallel_for_simd:
11931 case OMPD_target_teams:
11932 case OMPD_target_parallel:
11933 case OMPD_target_parallel_for:
11934 case OMPD_target_parallel_for_simd:
11935 case OMPD_requires:
11936 case OMPD_metadirective:
11937 case OMPD_unknown:
11938 default:
11939 llvm_unreachable("Unexpected standalone target data directive.");
11940 break;
11941 }
11942 if (HasNowait) {
11943 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
11944 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
11945 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
11946 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
11947 }
11948 CGF.EmitRuntimeCall(
11949 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID: RTLFn),
11950 args: OffloadingArgs);
11951 };
11952
11953 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
11954 &MapNamesArray](CodeGenFunction &CGF,
11955 PrePostActionTy &) {
11956 // Fill up the arrays with all the mapped variables.
11957 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11958 CGOpenMPRuntime::TargetDataInfo Info;
11959 MappableExprsHandler MEHandler(D, CGF);
11960 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
11961 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11962 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11963
11964 bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
11965 D.hasClausesOfKind<OMPNowaitClause>();
11966
11967 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11968 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11969 CGF.VoidPtrTy, CGM.getPointerAlign());
11970 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
11971 CGM.getPointerAlign());
11972 InputInfo.SizesArray =
11973 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11974 InputInfo.MappersArray =
11975 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11976 MapTypesArray = Info.RTArgs.MapTypesArray;
11977 MapNamesArray = Info.RTArgs.MapNamesArray;
11978 if (RequiresOuterTask)
11979 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11980 else
11981 emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11982 };
11983
11984 if (IfCond) {
11985 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen,
11986 ElseGen: [](CodeGenFunction &CGF, PrePostActionTy &) {});
11987 } else {
11988 RegionCodeGenTy ThenRCG(TargetThenGen);
11989 ThenRCG(CGF);
11990 }
11991}
11992
11993static unsigned
11994evaluateCDTSize(const FunctionDecl *FD,
11995 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
11996 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
11997 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
11998 // of that clause. The VLEN value must be power of 2.
11999 // In other case the notion of the function`s "characteristic data type" (CDT)
12000 // is used to compute the vector length.
12001 // CDT is defined in the following order:
12002 // a) For non-void function, the CDT is the return type.
12003 // b) If the function has any non-uniform, non-linear parameters, then the
12004 // CDT is the type of the first such parameter.
12005 // c) If the CDT determined by a) or b) above is struct, union, or class
12006 // type which is pass-by-value (except for the type that maps to the
12007 // built-in complex data type), the characteristic data type is int.
12008 // d) If none of the above three cases is applicable, the CDT is int.
12009 // The VLEN is then determined based on the CDT and the size of vector
12010 // register of that ISA for which current vector version is generated. The
12011 // VLEN is computed using the formula below:
12012 // VLEN = sizeof(vector_register) / sizeof(CDT),
12013 // where vector register size specified in section 3.2.1 Registers and the
12014 // Stack Frame of original AMD64 ABI document.
12015 QualType RetType = FD->getReturnType();
12016 if (RetType.isNull())
12017 return 0;
12018 ASTContext &C = FD->getASTContext();
12019 QualType CDT;
12020 if (!RetType.isNull() && !RetType->isVoidType()) {
12021 CDT = RetType;
12022 } else {
12023 unsigned Offset = 0;
12024 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD)) {
12025 if (ParamAttrs[Offset].Kind ==
12026 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector)
12027 CDT = C.getPointerType(T: C.getCanonicalTagType(TD: MD->getParent()));
12028 ++Offset;
12029 }
12030 if (CDT.isNull()) {
12031 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12032 if (ParamAttrs[I + Offset].Kind ==
12033 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) {
12034 CDT = FD->getParamDecl(i: I)->getType();
12035 break;
12036 }
12037 }
12038 }
12039 }
12040 if (CDT.isNull())
12041 CDT = C.IntTy;
12042 CDT = CDT->getCanonicalTypeUnqualified();
12043 if (CDT->isRecordType() || CDT->isUnionType())
12044 CDT = C.IntTy;
12045 return C.getTypeSize(T: CDT);
12046}
12047
12048// This are the Functions that are needed to mangle the name of the
12049// vector functions generated by the compiler, according to the rules
12050// defined in the "Vector Function ABI specifications for AArch64",
12051// available at
12052// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
12053
12054/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
12055static bool getAArch64MTV(QualType QT,
12056 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) {
12057 QT = QT.getCanonicalType();
12058
12059 if (QT->isVoidType())
12060 return false;
12061
12062 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform)
12063 return false;
12064
12065 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12066 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)
12067 return false;
12068
12069 if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12070 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) &&
12071 !QT->isReferenceType())
12072 return false;
12073
12074 return true;
12075}
12076
12077/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
12078static bool getAArch64PBV(QualType QT, ASTContext &C) {
12079 QT = QT.getCanonicalType();
12080 unsigned Size = C.getTypeSize(T: QT);
12081
12082 // Only scalars and complex within 16 bytes wide set PVB to true.
12083 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
12084 return false;
12085
12086 if (QT->isFloatingType())
12087 return true;
12088
12089 if (QT->isIntegerType())
12090 return true;
12091
12092 if (QT->isPointerType())
12093 return true;
12094
12095 // TODO: Add support for complex types (section 3.1.2, item 2).
12096
12097 return false;
12098}
12099
12100/// Computes the lane size (LS) of a return type or of an input parameter,
12101/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
12102/// TODO: Add support for references, section 3.2.1, item 1.
12103static unsigned getAArch64LS(QualType QT,
12104 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind,
12105 ASTContext &C) {
12106 if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
12107 QualType PTy = QT.getCanonicalType()->getPointeeType();
12108 if (getAArch64PBV(QT: PTy, C))
12109 return C.getTypeSize(T: PTy);
12110 }
12111 if (getAArch64PBV(QT, C))
12112 return C.getTypeSize(T: QT);
12113
12114 return C.getTypeSize(T: C.getUIntPtrType());
12115}
12116
12117// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
12118// signature of the scalar function, as defined in 3.2.2 of the
12119// AAVFABI.
12120static std::tuple<unsigned, unsigned, bool>
12121getNDSWDS(const FunctionDecl *FD,
12122 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12123 QualType RetType = FD->getReturnType().getCanonicalType();
12124
12125 ASTContext &C = FD->getASTContext();
12126
12127 bool OutputBecomesInput = false;
12128
12129 llvm::SmallVector<unsigned, 8> Sizes;
12130 if (!RetType->isVoidType()) {
12131 Sizes.push_back(Elt: getAArch64LS(
12132 QT: RetType, Kind: llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C));
12133 if (!getAArch64PBV(QT: RetType, C) && getAArch64MTV(QT: RetType, Kind: {}))
12134 OutputBecomesInput = true;
12135 }
12136 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12137 QualType QT = FD->getParamDecl(i: I)->getType().getCanonicalType();
12138 Sizes.push_back(Elt: getAArch64LS(QT, Kind: ParamAttrs[I].Kind, C));
12139 }
12140
12141 assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
12142 // The LS of a function parameter / return value can only be a power
12143 // of 2, starting from 8 bits, up to 128.
12144 assert(llvm::all_of(Sizes,
12145 [](unsigned Size) {
12146 return Size == 8 || Size == 16 || Size == 32 ||
12147 Size == 64 || Size == 128;
12148 }) &&
12149 "Invalid size");
12150
12151 return std::make_tuple(args&: *llvm::min_element(Range&: Sizes), args&: *llvm::max_element(Range&: Sizes),
12152 args&: OutputBecomesInput);
12153}
12154
12155static llvm::OpenMPIRBuilder::DeclareSimdBranch
12156convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) {
12157 switch (State) {
12158 case OMPDeclareSimdDeclAttr::BS_Undefined:
12159 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined;
12160 case OMPDeclareSimdDeclAttr::BS_Inbranch:
12161 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch;
12162 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
12163 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch;
12164 }
12165 llvm_unreachable("unexpected declare simd branch state");
12166}
12167
12168// Check the values provided via `simdlen` by the user.
12169static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc,
12170 unsigned UserVLEN, unsigned WDS, char ISA) {
12171 // 1. A `simdlen(1)` doesn't produce vector signatures.
12172 if (UserVLEN == 1) {
12173 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_1_no_effect);
12174 return false;
12175 }
12176
12177 // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced
12178 // SIMD.
12179 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(Value: UserVLEN)) {
12180 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_requires_power_of_2);
12181 return false;
12182 }
12183
12184 // 3. Section 3.4.1: SVE fixed length must obey the architectural limits.
12185 if (ISA == 's' && UserVLEN != 0 &&
12186 ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) {
12187 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_must_fit_lanes) << WDS;
12188 return false;
12189 }
12190
12191 return true;
12192}
12193
12194void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
12195 llvm::Function *Fn) {
12196 ASTContext &C = CGM.getContext();
12197 FD = FD->getMostRecentDecl();
12198 while (FD) {
12199 // Map params to their positions in function decl.
12200 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
12201 if (isa<CXXMethodDecl>(Val: FD))
12202 ParamPositions.try_emplace(Key: FD, Args: 0);
12203 unsigned ParamPos = ParamPositions.size();
12204 for (const ParmVarDecl *P : FD->parameters()) {
12205 ParamPositions.try_emplace(Key: P->getCanonicalDecl(), Args&: ParamPos);
12206 ++ParamPos;
12207 }
12208 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
12209 llvm::SmallVector<llvm::OpenMPIRBuilder::DeclareSimdAttrTy, 8> ParamAttrs(
12210 ParamPositions.size());
12211 // Mark uniform parameters.
12212 for (const Expr *E : Attr->uniforms()) {
12213 E = E->IgnoreParenImpCasts();
12214 unsigned Pos;
12215 if (isa<CXXThisExpr>(Val: E)) {
12216 Pos = ParamPositions[FD];
12217 } else {
12218 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12219 ->getCanonicalDecl();
12220 auto It = ParamPositions.find(Val: PVD);
12221 assert(It != ParamPositions.end() && "Function parameter not found");
12222 Pos = It->second;
12223 }
12224 ParamAttrs[Pos].Kind =
12225 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform;
12226 }
12227 // Get alignment info.
12228 auto *NI = Attr->alignments_begin();
12229 for (const Expr *E : Attr->aligneds()) {
12230 E = E->IgnoreParenImpCasts();
12231 unsigned Pos;
12232 QualType ParmTy;
12233 if (isa<CXXThisExpr>(Val: E)) {
12234 Pos = ParamPositions[FD];
12235 ParmTy = E->getType();
12236 } else {
12237 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12238 ->getCanonicalDecl();
12239 auto It = ParamPositions.find(Val: PVD);
12240 assert(It != ParamPositions.end() && "Function parameter not found");
12241 Pos = It->second;
12242 ParmTy = PVD->getType();
12243 }
12244 ParamAttrs[Pos].Alignment =
12245 (*NI)
12246 ? (*NI)->EvaluateKnownConstInt(Ctx: C)
12247 : llvm::APSInt::getUnsigned(
12248 X: C.toCharUnitsFromBits(BitSize: C.getOpenMPDefaultSimdAlign(T: ParmTy))
12249 .getQuantity());
12250 ++NI;
12251 }
12252 // Mark linear parameters.
12253 auto *SI = Attr->steps_begin();
12254 auto *MI = Attr->modifiers_begin();
12255 for (const Expr *E : Attr->linears()) {
12256 E = E->IgnoreParenImpCasts();
12257 unsigned Pos;
12258 bool IsReferenceType = false;
12259 // Rescaling factor needed to compute the linear parameter
12260 // value in the mangled name.
12261 unsigned PtrRescalingFactor = 1;
12262 if (isa<CXXThisExpr>(Val: E)) {
12263 Pos = ParamPositions[FD];
12264 auto *P = cast<PointerType>(Val: E->getType());
12265 PtrRescalingFactor = CGM.getContext()
12266 .getTypeSizeInChars(T: P->getPointeeType())
12267 .getQuantity();
12268 } else {
12269 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12270 ->getCanonicalDecl();
12271 auto It = ParamPositions.find(Val: PVD);
12272 assert(It != ParamPositions.end() && "Function parameter not found");
12273 Pos = It->second;
12274 if (auto *P = dyn_cast<PointerType>(Val: PVD->getType()))
12275 PtrRescalingFactor = CGM.getContext()
12276 .getTypeSizeInChars(T: P->getPointeeType())
12277 .getQuantity();
12278 else if (PVD->getType()->isReferenceType()) {
12279 IsReferenceType = true;
12280 PtrRescalingFactor =
12281 CGM.getContext()
12282 .getTypeSizeInChars(T: PVD->getType().getNonReferenceType())
12283 .getQuantity();
12284 }
12285 }
12286 llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos];
12287 if (*MI == OMPC_LINEAR_ref)
12288 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef;
12289 else if (*MI == OMPC_LINEAR_uval)
12290 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal;
12291 else if (IsReferenceType)
12292 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal;
12293 else
12294 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear;
12295 // Assuming a stride of 1, for `linear` without modifiers.
12296 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: 1);
12297 if (*SI) {
12298 Expr::EvalResult Result;
12299 if (!(*SI)->EvaluateAsInt(Result, Ctx: C, AllowSideEffects: Expr::SE_AllowSideEffects)) {
12300 if (const auto *DRE =
12301 cast<DeclRefExpr>(Val: (*SI)->IgnoreParenImpCasts())) {
12302 if (const auto *StridePVD =
12303 dyn_cast<ParmVarDecl>(Val: DRE->getDecl())) {
12304 ParamAttr.HasVarStride = true;
12305 auto It = ParamPositions.find(Val: StridePVD->getCanonicalDecl());
12306 assert(It != ParamPositions.end() &&
12307 "Function parameter not found");
12308 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: It->second);
12309 }
12310 }
12311 } else {
12312 ParamAttr.StrideOrArg = Result.Val.getInt();
12313 }
12314 }
12315 // If we are using a linear clause on a pointer, we need to
12316 // rescale the value of linear_step with the byte size of the
12317 // pointee type.
12318 if (!ParamAttr.HasVarStride &&
12319 (ParamAttr.Kind ==
12320 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12321 ParamAttr.Kind ==
12322 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef))
12323 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
12324 ++SI;
12325 ++MI;
12326 }
12327 llvm::APSInt VLENVal;
12328 SourceLocation ExprLoc;
12329 const Expr *VLENExpr = Attr->getSimdlen();
12330 if (VLENExpr) {
12331 VLENVal = VLENExpr->EvaluateKnownConstInt(Ctx: C);
12332 ExprLoc = VLENExpr->getExprLoc();
12333 }
12334 llvm::OpenMPIRBuilder::DeclareSimdBranch State =
12335 convertDeclareSimdBranch(State: Attr->getBranchState());
12336 if (CGM.getTriple().isX86()) {
12337 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
12338 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12339 OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElements: NumElts, VLENVal, ParamAttrs,
12340 Branch: State);
12341 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
12342 unsigned VLEN = VLENVal.getExtValue();
12343 // Get basic data for building the vector signature.
12344 const auto Data = getNDSWDS(FD, ParamAttrs);
12345 const unsigned NDS = std::get<0>(t: Data);
12346 const unsigned WDS = std::get<1>(t: Data);
12347 const bool OutputBecomesInput = std::get<2>(t: Data);
12348 if (CGM.getTarget().hasFeature(Feature: "sve")) {
12349 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 's'))
12350 OMPBuilder.emitAArch64DeclareSimdFunction(
12351 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 's', NarrowestDataSize: NDS, OutputBecomesInput);
12352 } else if (CGM.getTarget().hasFeature(Feature: "neon")) {
12353 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 'n'))
12354 OMPBuilder.emitAArch64DeclareSimdFunction(
12355 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 'n', NarrowestDataSize: NDS, OutputBecomesInput);
12356 }
12357 }
12358 }
12359 FD = FD->getPreviousDecl();
12360 }
12361}
12362
12363namespace {
12364/// Cleanup action for doacross support.
12365class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
12366public:
12367 static const int DoacrossFinArgs = 2;
12368
12369private:
12370 llvm::FunctionCallee RTLFn;
12371 llvm::Value *Args[DoacrossFinArgs];
12372
12373public:
12374 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
12375 ArrayRef<llvm::Value *> CallArgs)
12376 : RTLFn(RTLFn) {
12377 assert(CallArgs.size() == DoacrossFinArgs);
12378 std::copy(first: CallArgs.begin(), last: CallArgs.end(), result: std::begin(arr&: Args));
12379 }
12380 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12381 if (!CGF.HaveInsertPoint())
12382 return;
12383 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12384 }
12385};
12386} // namespace
12387
12388void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
12389 const OMPLoopDirective &D,
12390 ArrayRef<Expr *> NumIterations) {
12391 if (!CGF.HaveInsertPoint())
12392 return;
12393
12394 ASTContext &C = CGM.getContext();
12395 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
12396 RecordDecl *RD;
12397 if (KmpDimTy.isNull()) {
12398 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
12399 // kmp_int64 lo; // lower
12400 // kmp_int64 up; // upper
12401 // kmp_int64 st; // stride
12402 // };
12403 RD = C.buildImplicitRecord(Name: "kmp_dim");
12404 RD->startDefinition();
12405 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12406 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12407 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12408 RD->completeDefinition();
12409 KmpDimTy = C.getCanonicalTagType(TD: RD);
12410 } else {
12411 RD = KmpDimTy->castAsRecordDecl();
12412 }
12413 llvm::APInt Size(/*numBits=*/32, NumIterations.size());
12414 QualType ArrayTy = C.getConstantArrayType(EltTy: KmpDimTy, ArySize: Size, SizeExpr: nullptr,
12415 ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12416
12417 Address DimsAddr = CGF.CreateMemTemp(T: ArrayTy, Name: "dims");
12418 CGF.EmitNullInitialization(DestPtr: DimsAddr, Ty: ArrayTy);
12419 enum { LowerFD = 0, UpperFD, StrideFD };
12420 // Fill dims with data.
12421 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
12422 LValue DimsLVal = CGF.MakeAddrLValue(
12423 Addr: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: I), T: KmpDimTy);
12424 // dims.upper = num_iterations;
12425 LValue UpperLVal = CGF.EmitLValueForField(
12426 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: UpperFD));
12427 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
12428 Src: CGF.EmitScalarExpr(E: NumIterations[I]), SrcTy: NumIterations[I]->getType(),
12429 DstTy: Int64Ty, Loc: NumIterations[I]->getExprLoc());
12430 CGF.EmitStoreOfScalar(value: NumIterVal, lvalue: UpperLVal);
12431 // dims.stride = 1;
12432 LValue StrideLVal = CGF.EmitLValueForField(
12433 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: StrideFD));
12434 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::getSigned(Ty: CGM.Int64Ty, /*V=*/1),
12435 lvalue: StrideLVal);
12436 }
12437
12438 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
12439 // kmp_int32 num_dims, struct kmp_dim * dims);
12440 llvm::Value *Args[] = {
12441 emitUpdateLocation(CGF, Loc: D.getBeginLoc()),
12442 getThreadID(CGF, Loc: D.getBeginLoc()),
12443 llvm::ConstantInt::getSigned(Ty: CGM.Int32Ty, V: NumIterations.size()),
12444 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12445 V: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: 0).emitRawPointer(CGF),
12446 DestTy: CGM.VoidPtrTy)};
12447
12448 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12449 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_init);
12450 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12451 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
12452 emitUpdateLocation(CGF, Loc: D.getEndLoc()), getThreadID(CGF, Loc: D.getEndLoc())};
12453 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12454 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_fini);
12455 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(Kind: NormalAndEHCleanup, A: FiniRTLFn,
12456 A: llvm::ArrayRef(FiniArgs));
12457}
12458
12459template <typename T>
12460static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
12461 const T *C, llvm::Value *ULoc,
12462 llvm::Value *ThreadID) {
12463 QualType Int64Ty =
12464 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
12465 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
12466 QualType ArrayTy = CGM.getContext().getConstantArrayType(
12467 EltTy: Int64Ty, ArySize: Size, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12468 Address CntAddr = CGF.CreateMemTemp(T: ArrayTy, Name: ".cnt.addr");
12469 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
12470 const Expr *CounterVal = C->getLoopData(I);
12471 assert(CounterVal);
12472 llvm::Value *CntVal = CGF.EmitScalarConversion(
12473 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
12474 Loc: CounterVal->getExprLoc());
12475 CGF.EmitStoreOfScalar(Value: CntVal, Addr: CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: I),
12476 /*Volatile=*/false, Ty: Int64Ty);
12477 }
12478 llvm::Value *Args[] = {
12479 ULoc, ThreadID,
12480 CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: 0).emitRawPointer(CGF)};
12481 llvm::FunctionCallee RTLFn;
12482 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
12483 OMPDoacrossKind<T> ODK;
12484 if (ODK.isSource(C)) {
12485 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12486 FnID: OMPRTL___kmpc_doacross_post);
12487 } else {
12488 assert(ODK.isSink(C) && "Expect sink modifier.");
12489 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12490 FnID: OMPRTL___kmpc_doacross_wait);
12491 }
12492 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12493}
12494
12495void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12496 const OMPDependClause *C) {
12497 return EmitDoacrossOrdered<OMPDependClause>(
12498 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12499 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12500}
12501
12502void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12503 const OMPDoacrossClause *C) {
12504 return EmitDoacrossOrdered<OMPDoacrossClause>(
12505 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12506 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12507}
12508
12509void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
12510 llvm::FunctionCallee Callee,
12511 ArrayRef<llvm::Value *> Args) const {
12512 assert(Loc.isValid() && "Outlined function call location must be valid.");
12513 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
12514
12515 if (auto *Fn = dyn_cast<llvm::Function>(Val: Callee.getCallee())) {
12516 if (Fn->doesNotThrow()) {
12517 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
12518 return;
12519 }
12520 }
12521 CGF.EmitRuntimeCall(callee: Callee, args: Args);
12522}
12523
12524void CGOpenMPRuntime::emitOutlinedFunctionCall(
12525 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
12526 ArrayRef<llvm::Value *> Args) const {
12527 emitCall(CGF, Loc, Callee: OutlinedFn, Args);
12528}
12529
12530void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
12531 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D))
12532 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: FD))
12533 HasEmittedDeclareTargetRegion = true;
12534}
12535
12536Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
12537 const VarDecl *NativeParam,
12538 const VarDecl *TargetParam) const {
12539 return CGF.GetAddrOfLocalVar(VD: NativeParam);
12540}
12541
12542/// Return allocator value from expression, or return a null allocator (default
12543/// when no allocator specified).
12544static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
12545 const Expr *Allocator) {
12546 llvm::Value *AllocVal;
12547 if (Allocator) {
12548 AllocVal = CGF.EmitScalarExpr(E: Allocator);
12549 // According to the standard, the original allocator type is a enum
12550 // (integer). Convert to pointer type, if required.
12551 AllocVal = CGF.EmitScalarConversion(Src: AllocVal, SrcTy: Allocator->getType(),
12552 DstTy: CGF.getContext().VoidPtrTy,
12553 Loc: Allocator->getExprLoc());
12554 } else {
12555 // If no allocator specified, it defaults to the null allocator.
12556 AllocVal = llvm::Constant::getNullValue(
12557 Ty: CGF.CGM.getTypes().ConvertType(T: CGF.getContext().VoidPtrTy));
12558 }
12559 return AllocVal;
12560}
12561
12562/// Return the alignment from an allocate directive if present.
12563static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {
12564 std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);
12565
12566 if (!AllocateAlignment)
12567 return nullptr;
12568
12569 return llvm::ConstantInt::get(Ty: CGM.SizeTy, V: AllocateAlignment->getQuantity());
12570}
12571
12572Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
12573 const VarDecl *VD) {
12574 if (!VD)
12575 return Address::invalid();
12576 Address UntiedAddr = Address::invalid();
12577 Address UntiedRealAddr = Address::invalid();
12578 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12579 if (It != FunctionToUntiedTaskStackMap.end()) {
12580 const UntiedLocalVarsAddressesMap &UntiedData =
12581 UntiedLocalVarsStack[It->second];
12582 auto I = UntiedData.find(Key: VD);
12583 if (I != UntiedData.end()) {
12584 UntiedAddr = I->second.first;
12585 UntiedRealAddr = I->second.second;
12586 }
12587 }
12588 const VarDecl *CVD = VD->getCanonicalDecl();
12589 if (CVD->hasAttr<OMPAllocateDeclAttr>()) {
12590 // Use the default allocation.
12591 if (!isAllocatableDecl(VD))
12592 return UntiedAddr;
12593 llvm::Value *Size;
12594 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
12595 if (CVD->getType()->isVariablyModifiedType()) {
12596 Size = CGF.getTypeSize(Ty: CVD->getType());
12597 // Align the size: ((size + align - 1) / align) * align
12598 Size = CGF.Builder.CreateNUWAdd(
12599 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
12600 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
12601 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
12602 } else {
12603 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
12604 Size = CGM.getSize(numChars: Sz.alignTo(Align));
12605 }
12606 llvm::Value *ThreadID = getThreadID(CGF, Loc: CVD->getBeginLoc());
12607 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
12608 const Expr *Allocator = AA->getAllocator();
12609 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
12610 llvm::Value *Alignment = getAlignmentValue(CGM, VD: CVD);
12611 SmallVector<llvm::Value *, 4> Args;
12612 Args.push_back(Elt: ThreadID);
12613 if (Alignment)
12614 Args.push_back(Elt: Alignment);
12615 Args.push_back(Elt: Size);
12616 Args.push_back(Elt: AllocVal);
12617 llvm::omp::RuntimeFunction FnID =
12618 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
12619 llvm::Value *Addr = CGF.EmitRuntimeCall(
12620 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args,
12621 name: getName(Parts: {CVD->getName(), ".void.addr"}));
12622 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12623 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free);
12624 QualType Ty = CGM.getContext().getPointerType(T: CVD->getType());
12625 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12626 V: Addr, DestTy: CGF.ConvertTypeForMem(T: Ty), Name: getName(Parts: {CVD->getName(), ".addr"}));
12627 if (UntiedAddr.isValid())
12628 CGF.EmitStoreOfScalar(Value: Addr, Addr: UntiedAddr, /*Volatile=*/false, Ty);
12629
12630 // Cleanup action for allocate support.
12631 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
12632 llvm::FunctionCallee RTLFn;
12633 SourceLocation::UIntTy LocEncoding;
12634 Address Addr;
12635 const Expr *AllocExpr;
12636
12637 public:
12638 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
12639 SourceLocation::UIntTy LocEncoding, Address Addr,
12640 const Expr *AllocExpr)
12641 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
12642 AllocExpr(AllocExpr) {}
12643 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12644 if (!CGF.HaveInsertPoint())
12645 return;
12646 llvm::Value *Args[3];
12647 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
12648 CGF, Loc: SourceLocation::getFromRawEncoding(Encoding: LocEncoding));
12649 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12650 V: Addr.emitRawPointer(CGF), DestTy: CGF.VoidPtrTy);
12651 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator: AllocExpr);
12652 Args[2] = AllocVal;
12653 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12654 }
12655 };
12656 Address VDAddr =
12657 UntiedRealAddr.isValid()
12658 ? UntiedRealAddr
12659 : Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
12660 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
12661 Kind: NormalAndEHCleanup, A: FiniRTLFn, A: CVD->getLocation().getRawEncoding(),
12662 A: VDAddr, A: Allocator);
12663 if (UntiedRealAddr.isValid())
12664 if (auto *Region =
12665 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
12666 Region->emitUntiedSwitch(CGF);
12667 return VDAddr;
12668 }
12669 return UntiedAddr;
12670}
12671
12672bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,
12673 const VarDecl *VD) const {
12674 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12675 if (It == FunctionToUntiedTaskStackMap.end())
12676 return false;
12677 return UntiedLocalVarsStack[It->second].count(Key: VD) > 0;
12678}
12679
12680CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII(
12681 CodeGenModule &CGM, const OMPLoopDirective &S)
12682 : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {
12683 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12684 if (!NeedToPush)
12685 return;
12686 NontemporalDeclsSet &DS =
12687 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12688 for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) {
12689 for (const Stmt *Ref : C->private_refs()) {
12690 const auto *SimpleRefExpr = cast<Expr>(Val: Ref)->IgnoreParenImpCasts();
12691 const ValueDecl *VD;
12692 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: SimpleRefExpr)) {
12693 VD = DRE->getDecl();
12694 } else {
12695 const auto *ME = cast<MemberExpr>(Val: SimpleRefExpr);
12696 assert((ME->isImplicitCXXThis() ||
12697 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
12698 "Expected member of current class.");
12699 VD = ME->getMemberDecl();
12700 }
12701 DS.insert(V: VD);
12702 }
12703 }
12704}
12705
12706CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() {
12707 if (!NeedToPush)
12708 return;
12709 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
12710}
12711
12712CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII(
12713 CodeGenFunction &CGF,
12714 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
12715 std::pair<Address, Address>> &LocalVars)
12716 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
12717 if (!NeedToPush)
12718 return;
12719 CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace(
12720 Key: CGF.CurFn, Args: CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
12721 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(Elt: LocalVars);
12722}
12723
12724CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() {
12725 if (!NeedToPush)
12726 return;
12727 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
12728}
12729
12730bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const {
12731 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12732
12733 return llvm::any_of(
12734 Range&: CGM.getOpenMPRuntime().NontemporalDeclsStack,
12735 P: [VD](const NontemporalDeclsSet &Set) { return Set.contains(V: VD); });
12736}
12737
12738void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
12739 const OMPExecutableDirective &S,
12740 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled)
12741 const {
12742 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
12743 // Vars in target/task regions must be excluded completely.
12744 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()) ||
12745 isOpenMPTaskingDirective(Kind: S.getDirectiveKind())) {
12746 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
12747 getOpenMPCaptureRegions(CaptureRegions, DKind: S.getDirectiveKind());
12748 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CaptureRegions.front());
12749 for (const CapturedStmt::Capture &Cap : CS->captures()) {
12750 if (Cap.capturesVariable() || Cap.capturesVariableByCopy())
12751 NeedToCheckForLPCs.insert(V: Cap.getCapturedVar());
12752 }
12753 }
12754 // Exclude vars in private clauses.
12755 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
12756 for (const Expr *Ref : C->varlist()) {
12757 if (!Ref->getType()->isScalarType())
12758 continue;
12759 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12760 if (!DRE)
12761 continue;
12762 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12763 }
12764 }
12765 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
12766 for (const Expr *Ref : C->varlist()) {
12767 if (!Ref->getType()->isScalarType())
12768 continue;
12769 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12770 if (!DRE)
12771 continue;
12772 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12773 }
12774 }
12775 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
12776 for (const Expr *Ref : C->varlist()) {
12777 if (!Ref->getType()->isScalarType())
12778 continue;
12779 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12780 if (!DRE)
12781 continue;
12782 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12783 }
12784 }
12785 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
12786 for (const Expr *Ref : C->varlist()) {
12787 if (!Ref->getType()->isScalarType())
12788 continue;
12789 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12790 if (!DRE)
12791 continue;
12792 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12793 }
12794 }
12795 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
12796 for (const Expr *Ref : C->varlist()) {
12797 if (!Ref->getType()->isScalarType())
12798 continue;
12799 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12800 if (!DRE)
12801 continue;
12802 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12803 }
12804 }
12805 for (const Decl *VD : NeedToCheckForLPCs) {
12806 for (const LastprivateConditionalData &Data :
12807 llvm::reverse(C&: CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
12808 if (Data.DeclToUniqueName.count(Key: VD) > 0) {
12809 if (!Data.Disabled)
12810 NeedToAddForLPCsAsDisabled.insert(V: VD);
12811 break;
12812 }
12813 }
12814 }
12815}
12816
12817CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12818 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
12819 : CGM(CGF.CGM),
12820 Action((CGM.getLangOpts().OpenMP >= 50 &&
12821 llvm::any_of(Range: S.getClausesOfKind<OMPLastprivateClause>(),
12822 P: [](const OMPLastprivateClause *C) {
12823 return C->getKind() ==
12824 OMPC_LASTPRIVATE_conditional;
12825 }))
12826 ? ActionToDo::PushAsLastprivateConditional
12827 : ActionToDo::DoNotPush) {
12828 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12829 if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
12830 return;
12831 assert(Action == ActionToDo::PushAsLastprivateConditional &&
12832 "Expected a push action.");
12833 LastprivateConditionalData &Data =
12834 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
12835 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
12836 if (C->getKind() != OMPC_LASTPRIVATE_conditional)
12837 continue;
12838
12839 for (const Expr *Ref : C->varlist()) {
12840 Data.DeclToUniqueName.insert(KV: std::make_pair(
12841 x: cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts())->getDecl(),
12842 y: SmallString<16>(generateUniqueName(CGM, Prefix: "pl_cond", Ref))));
12843 }
12844 }
12845 Data.IVLVal = IVLVal;
12846 Data.Fn = CGF.CurFn;
12847}
12848
12849CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12850 CodeGenFunction &CGF, const OMPExecutableDirective &S)
12851 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
12852 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12853 if (CGM.getLangOpts().OpenMP < 50)
12854 return;
12855 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
12856 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
12857 if (!NeedToAddForLPCsAsDisabled.empty()) {
12858 Action = ActionToDo::DisableLastprivateConditional;
12859 LastprivateConditionalData &Data =
12860 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
12861 for (const Decl *VD : NeedToAddForLPCsAsDisabled)
12862 Data.DeclToUniqueName.try_emplace(Key: VD);
12863 Data.Fn = CGF.CurFn;
12864 Data.Disabled = true;
12865 }
12866}
12867
12868CGOpenMPRuntime::LastprivateConditionalRAII
12869CGOpenMPRuntime::LastprivateConditionalRAII::disable(
12870 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
12871 return LastprivateConditionalRAII(CGF, S);
12872}
12873
12874CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() {
12875 if (CGM.getLangOpts().OpenMP < 50)
12876 return;
12877 if (Action == ActionToDo::DisableLastprivateConditional) {
12878 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12879 "Expected list of disabled private vars.");
12880 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12881 }
12882 if (Action == ActionToDo::PushAsLastprivateConditional) {
12883 assert(
12884 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12885 "Expected list of lastprivate conditional vars.");
12886 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12887 }
12888}
12889
12890Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
12891 const VarDecl *VD) {
12892 ASTContext &C = CGM.getContext();
12893 auto I = LastprivateConditionalToTypes.try_emplace(Key: CGF.CurFn).first;
12894 QualType NewType;
12895 const FieldDecl *VDField;
12896 const FieldDecl *FiredField;
12897 LValue BaseLVal;
12898 auto VI = I->getSecond().find(Val: VD);
12899 if (VI == I->getSecond().end()) {
12900 RecordDecl *RD = C.buildImplicitRecord(Name: "lasprivate.conditional");
12901 RD->startDefinition();
12902 VDField = addFieldToRecordDecl(C, DC: RD, FieldTy: VD->getType().getNonReferenceType());
12903 FiredField = addFieldToRecordDecl(C, DC: RD, FieldTy: C.CharTy);
12904 RD->completeDefinition();
12905 NewType = C.getCanonicalTagType(TD: RD);
12906 Address Addr = CGF.CreateMemTemp(T: NewType, Align: C.getDeclAlign(D: VD), Name: VD->getName());
12907 BaseLVal = CGF.MakeAddrLValue(Addr, T: NewType, Source: AlignmentSource::Decl);
12908 I->getSecond().try_emplace(Key: VD, Args&: NewType, Args&: VDField, Args&: FiredField, Args&: BaseLVal);
12909 } else {
12910 NewType = std::get<0>(t&: VI->getSecond());
12911 VDField = std::get<1>(t&: VI->getSecond());
12912 FiredField = std::get<2>(t&: VI->getSecond());
12913 BaseLVal = std::get<3>(t&: VI->getSecond());
12914 }
12915 LValue FiredLVal =
12916 CGF.EmitLValueForField(Base: BaseLVal, Field: FiredField);
12917 CGF.EmitStoreOfScalar(
12918 value: llvm::ConstantInt::getNullValue(Ty: CGF.ConvertTypeForMem(T: C.CharTy)),
12919 lvalue: FiredLVal);
12920 return CGF.EmitLValueForField(Base: BaseLVal, Field: VDField).getAddress();
12921}
12922
12923namespace {
12924/// Checks if the lastprivate conditional variable is referenced in LHS.
12925class LastprivateConditionalRefChecker final
12926 : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {
12927 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM;
12928 const Expr *FoundE = nullptr;
12929 const Decl *FoundD = nullptr;
12930 StringRef UniqueDeclName;
12931 LValue IVLVal;
12932 llvm::Function *FoundFn = nullptr;
12933 SourceLocation Loc;
12934
12935public:
12936 bool VisitDeclRefExpr(const DeclRefExpr *E) {
12937 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
12938 llvm::reverse(C&: LPM)) {
12939 auto It = D.DeclToUniqueName.find(Key: E->getDecl());
12940 if (It == D.DeclToUniqueName.end())
12941 continue;
12942 if (D.Disabled)
12943 return false;
12944 FoundE = E;
12945 FoundD = E->getDecl()->getCanonicalDecl();
12946 UniqueDeclName = It->second;
12947 IVLVal = D.IVLVal;
12948 FoundFn = D.Fn;
12949 break;
12950 }
12951 return FoundE == E;
12952 }
12953 bool VisitMemberExpr(const MemberExpr *E) {
12954 if (!CodeGenFunction::IsWrappedCXXThis(E: E->getBase()))
12955 return false;
12956 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
12957 llvm::reverse(C&: LPM)) {
12958 auto It = D.DeclToUniqueName.find(Key: E->getMemberDecl());
12959 if (It == D.DeclToUniqueName.end())
12960 continue;
12961 if (D.Disabled)
12962 return false;
12963 FoundE = E;
12964 FoundD = E->getMemberDecl()->getCanonicalDecl();
12965 UniqueDeclName = It->second;
12966 IVLVal = D.IVLVal;
12967 FoundFn = D.Fn;
12968 break;
12969 }
12970 return FoundE == E;
12971 }
12972 bool VisitStmt(const Stmt *S) {
12973 for (const Stmt *Child : S->children()) {
12974 if (!Child)
12975 continue;
12976 if (const auto *E = dyn_cast<Expr>(Val: Child))
12977 if (!E->isGLValue())
12978 continue;
12979 if (Visit(S: Child))
12980 return true;
12981 }
12982 return false;
12983 }
12984 explicit LastprivateConditionalRefChecker(
12985 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
12986 : LPM(LPM) {}
12987 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
12988 getFoundData() const {
12989 return std::make_tuple(args: FoundE, args: FoundD, args: UniqueDeclName, args: IVLVal, args: FoundFn);
12990 }
12991};
12992} // namespace
12993
12994void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
12995 LValue IVLVal,
12996 StringRef UniqueDeclName,
12997 LValue LVal,
12998 SourceLocation Loc) {
12999 // Last updated loop counter for the lastprivate conditional var.
13000 // int<xx> last_iv = 0;
13001 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(T: IVLVal.getType());
13002 llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable(
13003 Ty: LLIVTy, Name: getName(Parts: {UniqueDeclName, "iv"}));
13004 cast<llvm::GlobalVariable>(Val: LastIV)->setAlignment(
13005 IVLVal.getAlignment().getAsAlign());
13006 LValue LastIVLVal =
13007 CGF.MakeNaturalAlignRawAddrLValue(V: LastIV, T: IVLVal.getType());
13008
13009 // Last value of the lastprivate conditional.
13010 // decltype(priv_a) last_a;
13011 llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
13012 Ty: CGF.ConvertTypeForMem(T: LVal.getType()), Name: UniqueDeclName);
13013 cast<llvm::GlobalVariable>(Val: Last)->setAlignment(
13014 LVal.getAlignment().getAsAlign());
13015 LValue LastLVal =
13016 CGF.MakeRawAddrLValue(V: Last, T: LVal.getType(), Alignment: LVal.getAlignment());
13017
13018 // Global loop counter. Required to handle inner parallel-for regions.
13019 // iv
13020 llvm::Value *IVVal = CGF.EmitLoadOfScalar(lvalue: IVLVal, Loc);
13021
13022 // #pragma omp critical(a)
13023 // if (last_iv <= iv) {
13024 // last_iv = iv;
13025 // last_a = priv_a;
13026 // }
13027 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
13028 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
13029 Action.Enter(CGF);
13030 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(lvalue: LastIVLVal, Loc);
13031 // (last_iv <= iv) ? Check if the variable is updated and store new
13032 // value in global var.
13033 llvm::Value *CmpRes;
13034 if (IVLVal.getType()->isSignedIntegerType()) {
13035 CmpRes = CGF.Builder.CreateICmpSLE(LHS: LastIVVal, RHS: IVVal);
13036 } else {
13037 assert(IVLVal.getType()->isUnsignedIntegerType() &&
13038 "Loop iteration variable must be integer.");
13039 CmpRes = CGF.Builder.CreateICmpULE(LHS: LastIVVal, RHS: IVVal);
13040 }
13041 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lp_cond_then");
13042 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "lp_cond_exit");
13043 CGF.Builder.CreateCondBr(Cond: CmpRes, True: ThenBB, False: ExitBB);
13044 // {
13045 CGF.EmitBlock(BB: ThenBB);
13046
13047 // last_iv = iv;
13048 CGF.EmitStoreOfScalar(value: IVVal, lvalue: LastIVLVal);
13049
13050 // last_a = priv_a;
13051 switch (CGF.getEvaluationKind(T: LVal.getType())) {
13052 case TEK_Scalar: {
13053 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
13054 CGF.EmitStoreOfScalar(value: PrivVal, lvalue: LastLVal);
13055 break;
13056 }
13057 case TEK_Complex: {
13058 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(src: LVal, loc: Loc);
13059 CGF.EmitStoreOfComplex(V: PrivVal, dest: LastLVal, /*isInit=*/false);
13060 break;
13061 }
13062 case TEK_Aggregate:
13063 llvm_unreachable(
13064 "Aggregates are not supported in lastprivate conditional.");
13065 }
13066 // }
13067 CGF.EmitBranch(Block: ExitBB);
13068 // There is no need to emit line number for unconditional branch.
13069 (void)ApplyDebugLocation::CreateEmpty(CGF);
13070 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
13071 };
13072
13073 if (CGM.getLangOpts().OpenMPSimd) {
13074 // Do not emit as a critical region as no parallel region could be emitted.
13075 RegionCodeGenTy ThenRCG(CodeGen);
13076 ThenRCG(CGF);
13077 } else {
13078 emitCriticalRegion(CGF, CriticalName: UniqueDeclName, CriticalOpGen: CodeGen, Loc);
13079 }
13080}
13081
13082void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
13083 const Expr *LHS) {
13084 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13085 return;
13086 LastprivateConditionalRefChecker Checker(LastprivateConditionalStack);
13087 if (!Checker.Visit(S: LHS))
13088 return;
13089 const Expr *FoundE;
13090 const Decl *FoundD;
13091 StringRef UniqueDeclName;
13092 LValue IVLVal;
13093 llvm::Function *FoundFn;
13094 std::tie(args&: FoundE, args&: FoundD, args&: UniqueDeclName, args&: IVLVal, args&: FoundFn) =
13095 Checker.getFoundData();
13096 if (FoundFn != CGF.CurFn) {
13097 // Special codegen for inner parallel regions.
13098 // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1;
13099 auto It = LastprivateConditionalToTypes[FoundFn].find(Val: FoundD);
13100 assert(It != LastprivateConditionalToTypes[FoundFn].end() &&
13101 "Lastprivate conditional is not found in outer region.");
13102 QualType StructTy = std::get<0>(t&: It->getSecond());
13103 const FieldDecl* FiredDecl = std::get<2>(t&: It->getSecond());
13104 LValue PrivLVal = CGF.EmitLValue(E: FoundE);
13105 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
13106 Addr: PrivLVal.getAddress(),
13107 Ty: CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: StructTy)),
13108 ElementTy: CGF.ConvertTypeForMem(T: StructTy));
13109 LValue BaseLVal =
13110 CGF.MakeAddrLValue(Addr: StructAddr, T: StructTy, Source: AlignmentSource::Decl);
13111 LValue FiredLVal = CGF.EmitLValueForField(Base: BaseLVal, Field: FiredDecl);
13112 CGF.EmitAtomicStore(rvalue: RValue::get(V: llvm::ConstantInt::get(
13113 Ty: CGF.ConvertTypeForMem(T: FiredDecl->getType()), V: 1)),
13114 lvalue: FiredLVal, AO: llvm::AtomicOrdering::Unordered,
13115 /*IsVolatile=*/true, /*isInit=*/false);
13116 return;
13117 }
13118
13119 // Private address of the lastprivate conditional in the current context.
13120 // priv_a
13121 LValue LVal = CGF.EmitLValue(E: FoundE);
13122 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
13123 Loc: FoundE->getExprLoc());
13124}
13125
13126void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional(
13127 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13128 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) {
13129 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13130 return;
13131 auto Range = llvm::reverse(C&: LastprivateConditionalStack);
13132 auto It = llvm::find_if(
13133 Range, P: [](const LastprivateConditionalData &D) { return !D.Disabled; });
13134 if (It == Range.end() || It->Fn != CGF.CurFn)
13135 return;
13136 auto LPCI = LastprivateConditionalToTypes.find(Val: It->Fn);
13137 assert(LPCI != LastprivateConditionalToTypes.end() &&
13138 "Lastprivates must be registered already.");
13139 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
13140 getOpenMPCaptureRegions(CaptureRegions, DKind: D.getDirectiveKind());
13141 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: CaptureRegions.back());
13142 for (const auto &Pair : It->DeclToUniqueName) {
13143 const auto *VD = cast<VarDecl>(Val: Pair.first->getCanonicalDecl());
13144 if (!CS->capturesVariable(Var: VD) || IgnoredDecls.contains(V: VD))
13145 continue;
13146 auto I = LPCI->getSecond().find(Val: Pair.first);
13147 assert(I != LPCI->getSecond().end() &&
13148 "Lastprivate must be rehistered already.");
13149 // bool Cmp = priv_a.Fired != 0;
13150 LValue BaseLVal = std::get<3>(t&: I->getSecond());
13151 LValue FiredLVal =
13152 CGF.EmitLValueForField(Base: BaseLVal, Field: std::get<2>(t&: I->getSecond()));
13153 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: FiredLVal, Loc: D.getBeginLoc());
13154 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Res);
13155 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lpc.then");
13156 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "lpc.done");
13157 // if (Cmp) {
13158 CGF.Builder.CreateCondBr(Cond: Cmp, True: ThenBB, False: DoneBB);
13159 CGF.EmitBlock(BB: ThenBB);
13160 Address Addr = CGF.GetAddrOfLocalVar(VD);
13161 LValue LVal;
13162 if (VD->getType()->isReferenceType())
13163 LVal = CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
13164 Source: AlignmentSource::Decl);
13165 else
13166 LVal = CGF.MakeAddrLValue(Addr, T: VD->getType().getNonReferenceType(),
13167 Source: AlignmentSource::Decl);
13168 emitLastprivateConditionalUpdate(CGF, IVLVal: It->IVLVal, UniqueDeclName: Pair.second, LVal,
13169 Loc: D.getBeginLoc());
13170 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
13171 CGF.EmitBlock(BB: DoneBB, /*IsFinal=*/IsFinished: true);
13172 // }
13173 }
13174}
13175
13176void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
13177 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
13178 SourceLocation Loc) {
13179 if (CGF.getLangOpts().OpenMP < 50)
13180 return;
13181 auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(Key: VD);
13182 assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() &&
13183 "Unknown lastprivate conditional variable.");
13184 StringRef UniqueName = It->second;
13185 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: UniqueName);
13186 // The variable was not updated in the region - exit.
13187 if (!GV)
13188 return;
13189 LValue LPLVal = CGF.MakeRawAddrLValue(
13190 V: GV, T: PrivLVal.getType().getNonReferenceType(), Alignment: PrivLVal.getAlignment());
13191 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: LPLVal, Loc);
13192 CGF.EmitStoreOfScalar(value: Res, lvalue: PrivLVal);
13193}
13194
13195llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
13196 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13197 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13198 const RegionCodeGenTy &CodeGen) {
13199 llvm_unreachable("Not supported in SIMD-only mode");
13200}
13201
13202llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
13203 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13204 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13205 const RegionCodeGenTy &CodeGen) {
13206 llvm_unreachable("Not supported in SIMD-only mode");
13207}
13208
13209llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
13210 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
13211 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
13212 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
13213 bool Tied, unsigned &NumberOfParts) {
13214 llvm_unreachable("Not supported in SIMD-only mode");
13215}
13216
13217void CGOpenMPSIMDRuntime::emitParallelCall(
13218 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
13219 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
13220 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
13221 OpenMPSeverityClauseKind Severity, const Expr *Message) {
13222 llvm_unreachable("Not supported in SIMD-only mode");
13223}
13224
13225void CGOpenMPSIMDRuntime::emitCriticalRegion(
13226 CodeGenFunction &CGF, StringRef CriticalName,
13227 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
13228 const Expr *Hint) {
13229 llvm_unreachable("Not supported in SIMD-only mode");
13230}
13231
13232void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
13233 const RegionCodeGenTy &MasterOpGen,
13234 SourceLocation Loc) {
13235 llvm_unreachable("Not supported in SIMD-only mode");
13236}
13237
13238void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,
13239 const RegionCodeGenTy &MasterOpGen,
13240 SourceLocation Loc,
13241 const Expr *Filter) {
13242 llvm_unreachable("Not supported in SIMD-only mode");
13243}
13244
13245void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
13246 SourceLocation Loc) {
13247 llvm_unreachable("Not supported in SIMD-only mode");
13248}
13249
13250void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
13251 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
13252 SourceLocation Loc) {
13253 llvm_unreachable("Not supported in SIMD-only mode");
13254}
13255
13256void CGOpenMPSIMDRuntime::emitSingleRegion(
13257 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
13258 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
13259 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
13260 ArrayRef<const Expr *> AssignmentOps) {
13261 llvm_unreachable("Not supported in SIMD-only mode");
13262}
13263
13264void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
13265 const RegionCodeGenTy &OrderedOpGen,
13266 SourceLocation Loc,
13267 bool IsThreads) {
13268 llvm_unreachable("Not supported in SIMD-only mode");
13269}
13270
13271void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
13272 SourceLocation Loc,
13273 OpenMPDirectiveKind Kind,
13274 bool EmitChecks,
13275 bool ForceSimpleCall) {
13276 llvm_unreachable("Not supported in SIMD-only mode");
13277}
13278
13279void CGOpenMPSIMDRuntime::emitForDispatchInit(
13280 CodeGenFunction &CGF, SourceLocation Loc,
13281 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
13282 bool Ordered, const DispatchRTInput &DispatchValues) {
13283 llvm_unreachable("Not supported in SIMD-only mode");
13284}
13285
13286void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
13287 SourceLocation Loc) {
13288 llvm_unreachable("Not supported in SIMD-only mode");
13289}
13290
13291void CGOpenMPSIMDRuntime::emitForStaticInit(
13292 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
13293 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
13294 llvm_unreachable("Not supported in SIMD-only mode");
13295}
13296
13297void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
13298 CodeGenFunction &CGF, SourceLocation Loc,
13299 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
13300 llvm_unreachable("Not supported in SIMD-only mode");
13301}
13302
13303void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
13304 SourceLocation Loc,
13305 unsigned IVSize,
13306 bool IVSigned) {
13307 llvm_unreachable("Not supported in SIMD-only mode");
13308}
13309
13310void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
13311 SourceLocation Loc,
13312 OpenMPDirectiveKind DKind) {
13313 llvm_unreachable("Not supported in SIMD-only mode");
13314}
13315
13316llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
13317 SourceLocation Loc,
13318 unsigned IVSize, bool IVSigned,
13319 Address IL, Address LB,
13320 Address UB, Address ST) {
13321 llvm_unreachable("Not supported in SIMD-only mode");
13322}
13323
13324void CGOpenMPSIMDRuntime::emitNumThreadsClause(
13325 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
13326 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
13327 SourceLocation SeverityLoc, const Expr *Message,
13328 SourceLocation MessageLoc) {
13329 llvm_unreachable("Not supported in SIMD-only mode");
13330}
13331
13332void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
13333 ProcBindKind ProcBind,
13334 SourceLocation Loc) {
13335 llvm_unreachable("Not supported in SIMD-only mode");
13336}
13337
13338Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
13339 const VarDecl *VD,
13340 Address VDAddr,
13341 SourceLocation Loc) {
13342 llvm_unreachable("Not supported in SIMD-only mode");
13343}
13344
13345llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
13346 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
13347 CodeGenFunction *CGF) {
13348 llvm_unreachable("Not supported in SIMD-only mode");
13349}
13350
13351Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
13352 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
13353 llvm_unreachable("Not supported in SIMD-only mode");
13354}
13355
13356void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
13357 ArrayRef<const Expr *> Vars,
13358 SourceLocation Loc,
13359 llvm::AtomicOrdering AO) {
13360 llvm_unreachable("Not supported in SIMD-only mode");
13361}
13362
13363void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
13364 const OMPExecutableDirective &D,
13365 llvm::Function *TaskFunction,
13366 QualType SharedsTy, Address Shareds,
13367 const Expr *IfCond,
13368 const OMPTaskDataTy &Data) {
13369 llvm_unreachable("Not supported in SIMD-only mode");
13370}
13371
13372void CGOpenMPSIMDRuntime::emitTaskLoopCall(
13373 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
13374 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
13375 const Expr *IfCond, const OMPTaskDataTy &Data) {
13376 llvm_unreachable("Not supported in SIMD-only mode");
13377}
13378
13379void CGOpenMPSIMDRuntime::emitReduction(
13380 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
13381 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
13382 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
13383 assert(Options.SimpleReduction && "Only simple reduction is expected.");
13384 CGOpenMPRuntime::emitReduction(CGF, Loc, OrgPrivates: Privates, OrgLHSExprs: LHSExprs, OrgRHSExprs: RHSExprs,
13385 OrgReductionOps: ReductionOps, Options);
13386}
13387
13388llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
13389 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
13390 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
13391 llvm_unreachable("Not supported in SIMD-only mode");
13392}
13393
13394void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
13395 SourceLocation Loc,
13396 bool IsWorksharingReduction) {
13397 llvm_unreachable("Not supported in SIMD-only mode");
13398}
13399
13400void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
13401 SourceLocation Loc,
13402 ReductionCodeGen &RCG,
13403 unsigned N) {
13404 llvm_unreachable("Not supported in SIMD-only mode");
13405}
13406
13407Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
13408 SourceLocation Loc,
13409 llvm::Value *ReductionsPtr,
13410 LValue SharedLVal) {
13411 llvm_unreachable("Not supported in SIMD-only mode");
13412}
13413
13414void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
13415 SourceLocation Loc,
13416 const OMPTaskDataTy &Data) {
13417 llvm_unreachable("Not supported in SIMD-only mode");
13418}
13419
13420void CGOpenMPSIMDRuntime::emitCancellationPointCall(
13421 CodeGenFunction &CGF, SourceLocation Loc,
13422 OpenMPDirectiveKind CancelRegion) {
13423 llvm_unreachable("Not supported in SIMD-only mode");
13424}
13425
13426void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
13427 SourceLocation Loc, const Expr *IfCond,
13428 OpenMPDirectiveKind CancelRegion) {
13429 llvm_unreachable("Not supported in SIMD-only mode");
13430}
13431
13432void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
13433 const OMPExecutableDirective &D, StringRef ParentName,
13434 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
13435 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
13436 llvm_unreachable("Not supported in SIMD-only mode");
13437}
13438
13439void CGOpenMPSIMDRuntime::emitTargetCall(
13440 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13441 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
13442 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
13443 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
13444 const OMPLoopDirective &D)>
13445 SizeEmitter) {
13446 llvm_unreachable("Not supported in SIMD-only mode");
13447}
13448
13449bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
13450 llvm_unreachable("Not supported in SIMD-only mode");
13451}
13452
13453bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
13454 llvm_unreachable("Not supported in SIMD-only mode");
13455}
13456
13457bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
13458 return false;
13459}
13460
13461void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
13462 const OMPExecutableDirective &D,
13463 SourceLocation Loc,
13464 llvm::Function *OutlinedFn,
13465 ArrayRef<llvm::Value *> CapturedVars) {
13466 llvm_unreachable("Not supported in SIMD-only mode");
13467}
13468
13469void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
13470 const Expr *NumTeams,
13471 const Expr *ThreadLimit,
13472 SourceLocation Loc) {
13473 llvm_unreachable("Not supported in SIMD-only mode");
13474}
13475
13476void CGOpenMPSIMDRuntime::emitTargetDataCalls(
13477 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13478 const Expr *Device, const RegionCodeGenTy &CodeGen,
13479 CGOpenMPRuntime::TargetDataInfo &Info) {
13480 llvm_unreachable("Not supported in SIMD-only mode");
13481}
13482
13483void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
13484 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13485 const Expr *Device) {
13486 llvm_unreachable("Not supported in SIMD-only mode");
13487}
13488
13489void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
13490 const OMPLoopDirective &D,
13491 ArrayRef<Expr *> NumIterations) {
13492 llvm_unreachable("Not supported in SIMD-only mode");
13493}
13494
13495void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13496 const OMPDependClause *C) {
13497 llvm_unreachable("Not supported in SIMD-only mode");
13498}
13499
13500void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13501 const OMPDoacrossClause *C) {
13502 llvm_unreachable("Not supported in SIMD-only mode");
13503}
13504
13505const VarDecl *
13506CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
13507 const VarDecl *NativeParam) const {
13508 llvm_unreachable("Not supported in SIMD-only mode");
13509}
13510
13511Address
13512CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
13513 const VarDecl *NativeParam,
13514 const VarDecl *TargetParam) const {
13515 llvm_unreachable("Not supported in SIMD-only mode");
13516}
13517