1//===------ CGOpenMPRuntimeGPU.h - Interface to OpenMP GPU Runtimes ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a generalized class for OpenMP runtime code generation
10// specialized by GPU targets NVPTX and AMDGCN.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H
15#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H
16
17#include "CGOpenMPRuntime.h"
18#include "CodeGenFunction.h"
19#include "clang/AST/StmtOpenMP.h"
20
21namespace clang {
22namespace CodeGen {
23
24class CGOpenMPRuntimeGPU : public CGOpenMPRuntime {
25public:
26 /// Defines the execution mode.
27 enum ExecutionMode {
28 /// SPMD execution mode (all threads are worker threads).
29 EM_SPMD,
30 /// Non-SPMD execution mode (1 master thread, others are workers).
31 EM_NonSPMD,
32 /// Unknown execution mode (orphaned directive).
33 EM_Unknown,
34 };
35
36 /// Target codegen is specialized based on two data-sharing modes: CUDA, in
37 /// which the local variables are actually global threadlocal, and Generic, in
38 /// which the local variables are placed in global memory if they may escape
39 /// their declaration context.
40 enum DataSharingMode {
41 /// CUDA data sharing mode.
42 DS_CUDA,
43 /// Generic data-sharing mode.
44 DS_Generic,
45 };
46
47private:
48 /// Parallel outlined function work for workers to execute.
49 llvm::SmallVector<llvm::Function *, 16> Work;
50
51 struct EntryFunctionState {
52 SourceLocation Loc;
53 };
54
55 ExecutionMode getExecutionMode() const;
56
57 DataSharingMode getDataSharingMode() const;
58
59 /// Get barrier to synchronize all threads in a block.
60 void syncCTAThreads(CodeGenFunction &CGF);
61
62 /// Helper to emit the kernel environment global for an `ompx_bare` kernel,
63 /// which does not go through emitKernelInit/emitKernelDeinit.
64 void emitBareKernelEnvironment(const OMPExecutableDirective &D,
65 CodeGenFunction &CGF);
66
67 /// Helper for target directive initialization.
68 void emitKernelInit(const OMPExecutableDirective &D, CodeGenFunction &CGF,
69 EntryFunctionState &EST, bool IsSPMD);
70
71 /// Helper for target directive finalization.
72 void emitKernelDeinit(CodeGenFunction &CGF, EntryFunctionState &EST,
73 bool IsSPMD);
74
75 /// Helper for generic variables globalization prolog.
76 void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc);
77
78 /// Helper for generic variables globalization epilog.
79 void emitGenericVarsEpilog(CodeGenFunction &CGF);
80
81 //
82 // Base class overrides.
83 //
84
85 /// Emit outlined function specialized for the Fork-Join
86 /// programming model for applicable target directives on the NVPTX device.
87 /// \param D Directive to emit.
88 /// \param ParentName Name of the function that encloses the target region.
89 /// \param OutlinedFn Outlined function value to be defined by this call.
90 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
91 /// \param IsOffloadEntry True if the outlined function is an offload entry.
92 /// An outlined function may not be an entry if, e.g. the if clause always
93 /// evaluates to false.
94 void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
95 llvm::Function *&OutlinedFn,
96 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
97 const RegionCodeGenTy &CodeGen);
98
99 /// Emit outlined function specialized for the Single Program
100 /// Multiple Data programming model for applicable target directives on the
101 /// NVPTX device.
102 /// \param D Directive to emit.
103 /// \param ParentName Name of the function that encloses the target region.
104 /// \param OutlinedFn Outlined function value to be defined by this call.
105 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
106 /// \param IsOffloadEntry True if the outlined function is an offload entry.
107 /// \param CodeGen Object containing the target statements.
108 /// An outlined function may not be an entry if, e.g. the if clause always
109 /// evaluates to false.
110 void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName,
111 llvm::Function *&OutlinedFn,
112 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry,
113 const RegionCodeGenTy &CodeGen);
114
115 /// Emit outlined function for 'target' directive on the NVPTX
116 /// device.
117 /// \param D Directive to emit.
118 /// \param ParentName Name of the function that encloses the target region.
119 /// \param OutlinedFn Outlined function value to be defined by this call.
120 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
121 /// \param IsOffloadEntry True if the outlined function is an offload entry.
122 /// An outlined function may not be an entry if, e.g. the if clause always
123 /// evaluates to false.
124 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
125 StringRef ParentName,
126 llvm::Function *&OutlinedFn,
127 llvm::Constant *&OutlinedFnID,
128 bool IsOffloadEntry,
129 const RegionCodeGenTy &CodeGen) override;
130
131protected:
132 /// Check if the default location must be constant.
133 /// Constant for NVPTX for better optimization.
134 bool isDefaultLocationConstant() const override { return true; }
135
136public:
137 explicit CGOpenMPRuntimeGPU(CodeGenModule &CGM);
138
139 bool isGPU() const override { return true; };
140
141 /// Declare generalized virtual functions which need to be defined
142 /// by all specializations of OpenMPGPURuntime Targets like AMDGCN
143 /// and NVPTX.
144
145 /// Check if the variable length declaration is delayed:
146 bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
147 const VarDecl *VD) const override;
148
149 /// Get call to __kmpc_alloc_shared
150 std::pair<llvm::Value *, llvm::Value *>
151 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) override;
152
153 /// Get call to __kmpc_free_shared
154 void getKmpcFreeShared(
155 CodeGenFunction &CGF,
156 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) override;
157
158 /// Get the id of the current thread on the GPU.
159 llvm::Value *getGPUThreadID(CodeGenFunction &CGF);
160
161 /// Get the maximum number of threads in a block of the GPU.
162 llvm::Value *getGPUNumThreads(CodeGenFunction &CGF);
163
164 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
165 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
166 void emitProcBindClause(CodeGenFunction &CGF,
167 llvm::omp::ProcBindKind ProcBind,
168 SourceLocation Loc) override;
169
170 // Currently unsupported on the device.
171 using CGOpenMPRuntime::emitMessageClause;
172 llvm::Value *emitMessageClause(CodeGenFunction &CGF, const Expr *Message,
173 SourceLocation Loc) override;
174
175 // Currently unsupported on the device.
176 using CGOpenMPRuntime::emitSeverityClause;
177 llvm::Value *emitSeverityClause(OpenMPSeverityClauseKind Severity,
178 SourceLocation Loc) override;
179
180 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
181 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
182 /// clause.
183 void emitNumThreadsClause(
184 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
185 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,
186 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
187 SourceLocation SeverityLoc = SourceLocation(),
188 const Expr *Message = nullptr,
189 SourceLocation MessageLoc = SourceLocation()) override;
190
191 /// This function ought to emit, in the general case, a call to
192 // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed
193 // as these numbers are obtained through the PTX grid and block configuration.
194 /// \param NumTeams An integer expression of teams.
195 /// \param ThreadLimit An integer expression of threads.
196 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
197 const Expr *ThreadLimit, SourceLocation Loc) override;
198
199 /// Emits inlined function for the specified OpenMP parallel
200 // directive.
201 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
202 /// kmp_int32 BoundID, struct context_vars*).
203 /// \param CGF Reference to current CodeGenFunction.
204 /// \param D OpenMP directive.
205 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
206 /// \param InnermostKind Kind of innermost directive (for simple directives it
207 /// is a directive itself, for combined - its innermost directive).
208 /// \param CodeGen Code generation sequence for the \a D directive.
209 llvm::Function *emitParallelOutlinedFunction(
210 CodeGenFunction &CGF, const OMPExecutableDirective &D,
211 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
212 const RegionCodeGenTy &CodeGen) override;
213
214 /// Emits inlined function for the specified OpenMP teams
215 // directive.
216 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
217 /// kmp_int32 BoundID, struct context_vars*).
218 /// \param CGF Reference to current CodeGenFunction.
219 /// \param D OpenMP directive.
220 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
221 /// \param InnermostKind Kind of innermost directive (for simple directives it
222 /// is a directive itself, for combined - its innermost directive).
223 /// \param CodeGen Code generation sequence for the \a D directive.
224 llvm::Function *emitTeamsOutlinedFunction(
225 CodeGenFunction &CGF, const OMPExecutableDirective &D,
226 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
227 const RegionCodeGenTy &CodeGen) override;
228
229 /// Emits code for teams call of the \a OutlinedFn with
230 /// variables captured in a record which address is stored in \a
231 /// CapturedStruct.
232 /// \param OutlinedFn Outlined function to be run by team masters. Type of
233 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
234 /// \param CapturedVars A pointer to the record with the references to
235 /// variables used in \a OutlinedFn function.
236 ///
237 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
238 SourceLocation Loc, llvm::Function *OutlinedFn,
239 ArrayRef<llvm::Value *> CapturedVars) override;
240
241 /// Emits code for parallel or serial call of the \a OutlinedFn with
242 /// variables captured in a record which address is stored in \a
243 /// CapturedStruct.
244 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
245 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
246 /// \param CapturedVars A pointer to the record with the references to
247 /// variables used in \a OutlinedFn function.
248 /// \param IfCond Condition in the associated 'if' clause, if it was
249 /// specified, nullptr otherwise.
250 /// \param NumThreads The value corresponding to the num_threads clause, if
251 /// any, or nullptr.
252 /// \param NumThreadsModifier The modifier of the num_threads clause, if
253 /// any, ignored otherwise. Currently unused on the device.
254 /// \param Severity The severity corresponding to the num_threads clause, if
255 /// any, ignored otherwise. Currently unused on the device.
256 /// \param Message The message string corresponding to the num_threads clause,
257 /// if any, or nullptr. Currently unused on the device.
258 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
259 llvm::Function *OutlinedFn,
260 ArrayRef<llvm::Value *> CapturedVars,
261 const Expr *IfCond, llvm::Value *NumThreads,
262 OpenMPNumThreadsClauseModifier NumThreadsModifier =
263 OMPC_NUMTHREADS_unknown,
264 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
265 const Expr *Message = nullptr) override;
266
267 /// Emit an implicit/explicit barrier for OpenMP threads.
268 /// \param Kind Directive for which this implicit barrier call must be
269 /// generated. Must be OMPD_barrier for explicit barrier generation.
270 /// \param EmitChecks true if need to emit checks for cancellation barriers.
271 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
272 /// runtime class decides which one to emit (simple or with cancellation
273 /// checks).
274 ///
275 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
276 OpenMPDirectiveKind Kind, bool EmitChecks = true,
277 bool ForceSimpleCall = false) override;
278
279 /// Emits a critical region.
280 /// \param CriticalName Name of the critical region.
281 /// \param CriticalOpGen Generator for the statement associated with the given
282 /// critical region.
283 /// \param Hint Value of the 'hint' clause (optional).
284 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
285 const RegionCodeGenTy &CriticalOpGen,
286 SourceLocation Loc,
287 const Expr *Hint = nullptr) override;
288
289 /// Emit a code for reduction clause.
290 ///
291 /// \param Privates List of private copies for original reduction arguments.
292 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
293 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
294 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
295 /// or 'operator binop(LHS, RHS)'.
296 /// \param Options List of options for reduction codegen:
297 /// WithNowait true if parent directive has also nowait clause, false
298 /// otherwise.
299 /// SimpleReduction Emit reduction operation only. Used for omp simd
300 /// directive on the host.
301 /// ReductionKind The kind of reduction to perform.
302 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
303 ArrayRef<const Expr *> Privates,
304 ArrayRef<const Expr *> LHSExprs,
305 ArrayRef<const Expr *> RHSExprs,
306 ArrayRef<const Expr *> ReductionOps,
307 ReductionOptionsTy Options) override;
308
309 /// Translates the native parameter of outlined function if this is required
310 /// for target.
311 /// \param FD Field decl from captured record for the parameter.
312 /// \param NativeParam Parameter itself.
313 const VarDecl *translateParameter(const FieldDecl *FD,
314 const VarDecl *NativeParam) const override;
315
316 /// Gets the address of the native argument basing on the address of the
317 /// target-specific parameter.
318 /// \param NativeParam Parameter itself.
319 /// \param TargetParam Corresponding target-specific parameter.
320 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
321 const VarDecl *TargetParam) const override;
322
323 /// Emits call of the outlined function with the provided arguments,
324 /// translating these arguments to correct target-specific arguments.
325 void
326 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
327 llvm::FunctionCallee OutlinedFn,
328 ArrayRef<llvm::Value *> Args = {}) const override;
329
330 /// Emits OpenMP-specific function prolog.
331 /// Required for device constructs.
332 void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override;
333
334 /// Gets the OpenMP-specific address of the local variable.
335 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
336 const VarDecl *VD) override;
337
338 /// Cleans up references to the objects in finished function.
339 ///
340 void functionFinished(CodeGenFunction &CGF) override;
341
342 /// Choose a default value for the dist_schedule clause.
343 void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
344 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
345 llvm::Value *&Chunk) const override;
346
347 /// Choose a default value for the schedule clause.
348 void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
349 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
350 const Expr *&ChunkExpr) const override;
351
352 /// Adjust some parameters for the target-based directives, like addresses of
353 /// the variables captured by reference in lambdas.
354 void adjustTargetSpecificDataForLambdas(
355 CodeGenFunction &CGF, const OMPExecutableDirective &D) const override;
356
357 /// Perform check on requires decl to ensure that target architecture
358 /// supports unified addressing
359 void processRequiresDirective(const OMPRequiresDecl *D) override;
360
361 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
362 /// the predefined allocator and translates it into the corresponding address
363 /// space.
364 bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override;
365
366private:
367 /// Track the execution mode when codegening directives within a target
368 /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the
369 /// target region and used by containing directives such as 'parallel'
370 /// to emit optimized code.
371 ExecutionMode CurrentExecutionMode = EM_Unknown;
372
373 /// Track the data sharing mode when codegening directives within a target
374 /// region.
375 DataSharingMode CurrentDataSharingMode = DataSharingMode::DS_Generic;
376
377 /// true if currently emitting code for target/teams/distribute region, false
378 /// - otherwise.
379 bool IsInTTDRegion = false;
380
381 /// Map between an outlined function and its wrapper.
382 llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap;
383
384 /// Emit function which wraps the outline parallel region
385 /// and controls the parameters which are passed to this function.
386 /// The wrapper ensures that the outlined function is called
387 /// with the correct arguments when data is shared.
388 llvm::Function *createParallelDataSharingWrapper(
389 llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D);
390
391 /// The data for the single globalized variable.
392 struct MappedVarData {
393 /// Corresponding field in the global record.
394 llvm::Value *GlobalizedVal = nullptr;
395 /// Corresponding address.
396 Address PrivateAddr = Address::invalid();
397 };
398 /// The map of local variables to their addresses in the global memory.
399 using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;
400 /// Set of the parameters passed by value escaping OpenMP context.
401 using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>;
402 struct FunctionData {
403 DeclToAddrMapTy LocalVarData;
404 EscapedParamsTy EscapedParameters;
405 llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;
406 llvm::SmallVector<const ValueDecl *, 4> DelayedVariableLengthDecls;
407 llvm::SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4>
408 EscapedVariableLengthDeclsAddrs;
409 std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;
410 };
411 /// Maps the function to the list of the globalized variables with their
412 /// addresses.
413 llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;
414 /// List of the records with the list of fields for the reductions across the
415 /// teams. Used to build the intermediate buffer for the fast teams
416 /// reductions.
417 /// All the records are gathered into a union `union.type` is created.
418 llvm::SmallVector<const RecordDecl *, 4> TeamsReductions;
419 /// Pair of the Non-SPMD team and all reductions variables in this team
420 /// region.
421 std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>>
422 TeamAndReductions;
423};
424
425} // CodeGen namespace.
426} // clang namespace.
427
428#endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H
429