1//===-- IPO/OpenMPOpt.cpp - Collection of OpenMP specific optimizations ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// OpenMP specific optimizations:
10//
11// - Deduplication of runtime calls, e.g., omp_get_thread_num.
12// - Replacing globalized device memory with stack memory.
13// - Replacing globalized device memory with shared memory.
14// - Parallel region merging.
15// - Transforming generic-mode device kernels to SPMD mode.
16// - Specializing the state machine for generic-mode device kernels.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/IPO/OpenMPOpt.h"
21
22#include "llvm/ADT/DenseSet.h"
23#include "llvm/ADT/EnumeratedArray.h"
24#include "llvm/ADT/PostOrderIterator.h"
25#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/Statistic.h"
29#include "llvm/ADT/StringExtras.h"
30#include "llvm/ADT/StringRef.h"
31#include "llvm/Analysis/CallGraph.h"
32#include "llvm/Analysis/MemoryBuiltins.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/Analysis/OptimizationRemarkEmitter.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/Frontend/OpenMP/OMPConstants.h"
37#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
38#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
39#include "llvm/IR/Assumptions.h"
40#include "llvm/IR/BasicBlock.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DiagnosticInfo.h"
43#include "llvm/IR/Dominators.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/GlobalValue.h"
46#include "llvm/IR/GlobalVariable.h"
47#include "llvm/IR/InstrTypes.h"
48#include "llvm/IR/Instruction.h"
49#include "llvm/IR/Instructions.h"
50#include "llvm/IR/IntrinsicInst.h"
51#include "llvm/IR/IntrinsicsAMDGPU.h"
52#include "llvm/IR/IntrinsicsNVPTX.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/MDBuilder.h"
55#include "llvm/Support/Casting.h"
56#include "llvm/Support/CommandLine.h"
57#include "llvm/Support/Debug.h"
58#include "llvm/Transforms/IPO/Attributor.h"
59#include "llvm/Transforms/Utils/BasicBlockUtils.h"
60#include "llvm/Transforms/Utils/CallGraphUpdater.h"
61
62#include <algorithm>
63#include <optional>
64#include <string>
65
66using namespace llvm;
67using namespace omp;
68
69#define DEBUG_TYPE "openmp-opt"
70
71static cl::opt<bool> DisableOpenMPOptimizations(
72 "openmp-opt-disable", cl::desc("Disable OpenMP specific optimizations."),
73 cl::Hidden, cl::init(Val: false));
74
75static cl::opt<bool> EnableParallelRegionMerging(
76 "openmp-opt-enable-merging",
77 cl::desc("Enable the OpenMP region merging optimization."), cl::Hidden,
78 cl::init(Val: false));
79
80static cl::opt<bool>
81 DisableInternalization("openmp-opt-disable-internalization",
82 cl::desc("Disable function internalization."),
83 cl::Hidden, cl::init(Val: false));
84
85static cl::opt<bool> DeduceICVValues("openmp-deduce-icv-values",
86 cl::init(Val: false), cl::Hidden);
87static cl::opt<bool> PrintICVValues("openmp-print-icv-values", cl::init(Val: false),
88 cl::Hidden);
89static cl::opt<bool> PrintOpenMPKernels("openmp-print-gpu-kernels",
90 cl::init(Val: false), cl::Hidden);
91
92static cl::opt<bool> HideMemoryTransferLatency(
93 "openmp-hide-memory-transfer-latency",
94 cl::desc("[WIP] Tries to hide the latency of host to device memory"
95 " transfers"),
96 cl::Hidden, cl::init(Val: false));
97
98static cl::opt<bool> DisableOpenMPOptDeglobalization(
99 "openmp-opt-disable-deglobalization",
100 cl::desc("Disable OpenMP optimizations involving deglobalization."),
101 cl::Hidden, cl::init(Val: false));
102
103static cl::opt<bool> DisableOpenMPOptSPMDization(
104 "openmp-opt-disable-spmdization",
105 cl::desc("Disable OpenMP optimizations involving SPMD-ization."),
106 cl::Hidden, cl::init(Val: false));
107
108static cl::opt<bool> DisableOpenMPOptFolding(
109 "openmp-opt-disable-folding",
110 cl::desc("Disable OpenMP optimizations involving folding."), cl::Hidden,
111 cl::init(Val: false));
112
113static cl::opt<bool> DisableOpenMPOptStateMachineRewrite(
114 "openmp-opt-disable-state-machine-rewrite",
115 cl::desc("Disable OpenMP optimizations that replace the state machine."),
116 cl::Hidden, cl::init(Val: false));
117
118static cl::opt<bool> DisableOpenMPOptBarrierElimination(
119 "openmp-opt-disable-barrier-elimination",
120 cl::desc("Disable OpenMP optimizations that eliminate barriers."),
121 cl::Hidden, cl::init(Val: false));
122
123static cl::opt<bool> PrintModuleAfterOptimizations(
124 "openmp-opt-print-module-after",
125 cl::desc("Print the current module after OpenMP optimizations."),
126 cl::Hidden, cl::init(Val: false));
127
128static cl::opt<bool> PrintModuleBeforeOptimizations(
129 "openmp-opt-print-module-before",
130 cl::desc("Print the current module before OpenMP optimizations."),
131 cl::Hidden, cl::init(Val: false));
132
133static cl::opt<bool> AlwaysInlineDeviceFunctions(
134 "openmp-opt-inline-device",
135 cl::desc("Inline all applicable functions on the device."), cl::Hidden,
136 cl::init(Val: false));
137
138static cl::opt<bool>
139 EnableVerboseRemarks("openmp-opt-verbose-remarks",
140 cl::desc("Enables more verbose remarks."), cl::Hidden,
141 cl::init(Val: false));
142
143static cl::opt<unsigned>
144 SetFixpointIterations("openmp-opt-max-iterations", cl::Hidden,
145 cl::desc("Maximal number of attributor iterations."),
146 cl::init(Val: 256));
147
148static cl::opt<unsigned>
149 SharedMemoryLimit("openmp-opt-shared-limit", cl::Hidden,
150 cl::desc("Maximum amount of shared memory to use."),
151 cl::init(Val: std::numeric_limits<unsigned>::max()));
152
153static cl::opt<unsigned> MaxCalleesForSpecialization(
154 "openmp-opt-max-callees-for-specialization", cl::Hidden,
155 cl::desc("Number of possible callees above which an indirect call site is "
156 "left alone rather than specialized into an if-cascade."),
157 cl::init(Val: 3));
158
159STATISTIC(NumOpenMPRuntimeCallsDeduplicated,
160 "Number of OpenMP runtime calls deduplicated");
161STATISTIC(NumOpenMPParallelRegionsDeleted,
162 "Number of OpenMP parallel regions deleted");
163STATISTIC(NumOpenMPRuntimeFunctionsIdentified,
164 "Number of OpenMP runtime functions identified");
165STATISTIC(NumOpenMPRuntimeFunctionUsesIdentified,
166 "Number of OpenMP runtime function uses identified");
167STATISTIC(NumOpenMPTargetRegionKernels,
168 "Number of OpenMP target region entry points (=kernels) identified");
169STATISTIC(NumNonOpenMPTargetRegionKernels,
170 "Number of non-OpenMP target region kernels identified");
171STATISTIC(NumOpenMPTargetRegionKernelsSPMD,
172 "Number of OpenMP target region entry points (=kernels) executed in "
173 "SPMD-mode instead of generic-mode");
174STATISTIC(NumOpenMPTargetRegionKernelsWithoutStateMachine,
175 "Number of OpenMP target region entry points (=kernels) executed in "
176 "generic-mode without a state machines");
177STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback,
178 "Number of OpenMP target region entry points (=kernels) executed in "
179 "generic-mode with customized state machines with fallback");
180STATISTIC(NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback,
181 "Number of OpenMP target region entry points (=kernels) executed in "
182 "generic-mode with customized state machines without fallback");
183STATISTIC(
184 NumOpenMPParallelRegionsReplacedInGPUStateMachine,
185 "Number of OpenMP parallel regions replaced with ID in GPU state machines");
186STATISTIC(NumOpenMPParallelRegionsMerged,
187 "Number of OpenMP parallel regions merged");
188STATISTIC(NumBytesMovedToSharedMemory,
189 "Amount of memory pushed to shared memory");
190STATISTIC(NumBarriersEliminated, "Number of redundant barriers eliminated");
191
192#if !defined(NDEBUG)
193static constexpr auto TAG = "[" DEBUG_TYPE "]";
194#endif
195
196namespace KernelInfo {
197
198// struct ConfigurationEnvironmentTy {
199// uint8_t UseGenericStateMachine;
200// uint8_t MayUseNestedParallelism;
201// llvm::omp::OMPTgtExecModeFlags ExecMode;
202// int32_t MinThreads;
203// int32_t MaxThreads;
204// int32_t MinTeams;
205// int32_t MaxTeams;
206// };
207
208// struct DynamicEnvironmentTy {
209// uint16_t DebugIndentionLevel;
210// };
211
212// struct KernelEnvironmentTy {
213// ConfigurationEnvironmentTy Configuration;
214// IdentTy *Ident;
215// DynamicEnvironmentTy *DynamicEnv;
216// };
217
218#define KERNEL_ENVIRONMENT_IDX(MEMBER, IDX) \
219 constexpr unsigned MEMBER##Idx = IDX;
220
221KERNEL_ENVIRONMENT_IDX(Configuration, 0)
222KERNEL_ENVIRONMENT_IDX(Ident, 1)
223
224#undef KERNEL_ENVIRONMENT_IDX
225
226#define KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MEMBER, IDX) \
227 constexpr unsigned MEMBER##Idx = IDX;
228
229KERNEL_ENVIRONMENT_CONFIGURATION_IDX(UseGenericStateMachine, 0)
230KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MayUseNestedParallelism, 1)
231KERNEL_ENVIRONMENT_CONFIGURATION_IDX(ExecMode, 2)
232KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MinThreads, 3)
233KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MaxThreads, 4)
234KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MinTeams, 5)
235KERNEL_ENVIRONMENT_CONFIGURATION_IDX(MaxTeams, 6)
236
237#undef KERNEL_ENVIRONMENT_CONFIGURATION_IDX
238
239#define KERNEL_ENVIRONMENT_GETTER(MEMBER, RETURNTYPE) \
240 RETURNTYPE *get##MEMBER##FromKernelEnvironment(ConstantStruct *KernelEnvC) { \
241 return cast<RETURNTYPE>(KernelEnvC->getAggregateElement(MEMBER##Idx)); \
242 }
243
244KERNEL_ENVIRONMENT_GETTER(Ident, Constant)
245KERNEL_ENVIRONMENT_GETTER(Configuration, ConstantStruct)
246
247#undef KERNEL_ENVIRONMENT_GETTER
248
249#define KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MEMBER) \
250 ConstantInt *get##MEMBER##FromKernelEnvironment( \
251 ConstantStruct *KernelEnvC) { \
252 ConstantStruct *ConfigC = \
253 getConfigurationFromKernelEnvironment(KernelEnvC); \
254 return dyn_cast<ConstantInt>(ConfigC->getAggregateElement(MEMBER##Idx)); \
255 }
256
257KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(UseGenericStateMachine)
258KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MayUseNestedParallelism)
259KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(ExecMode)
260KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MinThreads)
261KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MaxThreads)
262KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MinTeams)
263KERNEL_ENVIRONMENT_CONFIGURATION_GETTER(MaxTeams)
264
265#undef KERNEL_ENVIRONMENT_CONFIGURATION_GETTER
266
267GlobalVariable *
268getKernelEnvironementGVFromKernelInitCB(CallBase *KernelInitCB) {
269 constexpr int InitKernelEnvironmentArgNo = 0;
270 return cast<GlobalVariable>(
271 Val: KernelInitCB->getArgOperand(i: InitKernelEnvironmentArgNo)
272 ->stripPointerCasts());
273}
274
275ConstantStruct *getKernelEnvironementFromKernelInitCB(CallBase *KernelInitCB) {
276 GlobalVariable *KernelEnvGV =
277 getKernelEnvironementGVFromKernelInitCB(KernelInitCB);
278 return cast<ConstantStruct>(Val: KernelEnvGV->getInitializer());
279}
280} // namespace KernelInfo
281
282namespace {
283
284struct AAHeapToShared;
285
286struct AAICVTracker;
287
288/// OpenMP specific information. For now, stores RFIs and ICVs also needed for
289/// Attributor runs.
290struct OMPInformationCache : public InformationCache {
291 OMPInformationCache(Module &M, AnalysisGetter &AG,
292 BumpPtrAllocator &Allocator, SetVector<Function *> *CGSCC,
293 bool OpenMPPostLink)
294 : InformationCache(M, AG, Allocator, CGSCC), OMPBuilder(M),
295 OpenMPPostLink(OpenMPPostLink) {
296
297 OMPBuilder.Config.IsTargetDevice = isOpenMPDevice(M&: OMPBuilder.M);
298 const Triple T(OMPBuilder.M.getTargetTriple());
299 switch (T.getArch()) {
300 case llvm::Triple::nvptx:
301 case llvm::Triple::nvptx64:
302 case llvm::Triple::amdgpu:
303 assert(OMPBuilder.Config.IsTargetDevice &&
304 "OpenMP AMDGPU/NVPTX is only prepared to deal with device code.");
305 OMPBuilder.Config.IsGPU = true;
306 break;
307 default:
308 OMPBuilder.Config.IsGPU = false;
309 break;
310 }
311 OMPBuilder.initialize();
312 initializeRuntimeFunctions(M);
313 initializeInternalControlVars();
314 }
315
316 /// Generic information that describes an internal control variable.
317 struct InternalControlVarInfo {
318 /// The kind, as described by InternalControlVar enum.
319 InternalControlVar Kind;
320
321 /// The name of the ICV.
322 StringRef Name;
323
324 /// Environment variable associated with this ICV.
325 StringRef EnvVarName;
326
327 /// Initial value kind.
328 ICVInitValue InitKind;
329
330 /// Initial value.
331 ConstantInt *InitValue;
332
333 /// Setter RTL function associated with this ICV.
334 RuntimeFunction Setter;
335
336 /// Getter RTL function associated with this ICV.
337 RuntimeFunction Getter;
338
339 /// RTL Function corresponding to the override clause of this ICV
340 RuntimeFunction Clause;
341 };
342
343 /// Generic information that describes a runtime function
344 struct RuntimeFunctionInfo {
345
346 /// The kind, as described by the RuntimeFunction enum.
347 RuntimeFunction Kind;
348
349 /// The name of the function.
350 StringRef Name;
351
352 /// Flag to indicate a variadic function.
353 bool IsVarArg;
354
355 /// The return type of the function.
356 Type *ReturnType;
357
358 /// The argument types of the function.
359 SmallVector<Type *, 8> ArgumentTypes;
360
361 /// The declaration if available.
362 Function *Declaration = nullptr;
363
364 /// Uses of this runtime function per function containing the use.
365 using UseVector = SmallVector<Use *, 16>;
366
367 /// Clear UsesMap for runtime function.
368 void clearUsesMap() { UsesMap.clear(); }
369
370 /// Boolean conversion that is true if the runtime function was found.
371 operator bool() const { return Declaration; }
372
373 /// Return the vector of uses in function \p F.
374 UseVector &getOrCreateUseVector(Function *F) {
375 std::shared_ptr<UseVector> &UV = UsesMap[F];
376 if (!UV)
377 UV = std::make_shared<UseVector>();
378 return *UV;
379 }
380
381 /// Return the vector of uses in function \p F or `nullptr` if there are
382 /// none.
383 const UseVector *getUseVector(Function &F) const {
384 auto I = UsesMap.find(Val: &F);
385 if (I != UsesMap.end())
386 return I->second.get();
387 return nullptr;
388 }
389
390 /// Return how many functions contain uses of this runtime function.
391 size_t getNumFunctionsWithUses() const { return UsesMap.size(); }
392
393 /// Return the number of arguments (or the minimal number for variadic
394 /// functions).
395 size_t getNumArgs() const { return ArgumentTypes.size(); }
396
397 /// Run the callback \p CB on each use and forget the use if the result is
398 /// true. The callback will be fed the function in which the use was
399 /// encountered as second argument.
400 void foreachUse(SmallVectorImpl<Function *> &SCC,
401 function_ref<bool(Use &, Function &)> CB) {
402 for (Function *F : SCC)
403 foreachUse(CB, F);
404 }
405
406 /// Run the callback \p CB on each use within the function \p F and forget
407 /// the use if the result is true.
408 void foreachUse(function_ref<bool(Use &, Function &)> CB, Function *F) {
409 SmallVector<unsigned, 8> ToBeDeleted;
410 ToBeDeleted.clear();
411
412 unsigned Idx = 0;
413 UseVector &UV = getOrCreateUseVector(F);
414
415 for (Use *U : UV) {
416 if (CB(*U, *F))
417 ToBeDeleted.push_back(Elt: Idx);
418 ++Idx;
419 }
420
421 // Remove the to-be-deleted indices in reverse order as prior
422 // modifications will not modify the smaller indices.
423 while (!ToBeDeleted.empty()) {
424 unsigned Idx = ToBeDeleted.pop_back_val();
425 UV[Idx] = UV.back();
426 UV.pop_back();
427 }
428 }
429
430 private:
431 /// Map from functions to all uses of this runtime function contained in
432 /// them.
433 DenseMap<Function *, std::shared_ptr<UseVector>> UsesMap;
434
435 public:
436 /// Iterators for the uses of this runtime function.
437 decltype(UsesMap)::iterator begin() { return UsesMap.begin(); }
438 decltype(UsesMap)::iterator end() { return UsesMap.end(); }
439 };
440
441 /// An OpenMP-IR-Builder instance
442 OpenMPIRBuilder OMPBuilder;
443
444 /// Map from runtime function kind to the runtime function description.
445 EnumeratedArray<RuntimeFunctionInfo, RuntimeFunction,
446 RuntimeFunction::OMPRTL___last>
447 RFIs;
448
449 /// Map from function declarations/definitions to their runtime enum type.
450 DenseMap<Function *, RuntimeFunction> RuntimeFunctionIDMap;
451
452 /// Map from ICV kind to the ICV description.
453 EnumeratedArray<InternalControlVarInfo, InternalControlVar,
454 InternalControlVar::ICV___last>
455 ICVs;
456
457 /// Helper to initialize all internal control variable information for those
458 /// defined in OMPKinds.def.
459 void initializeInternalControlVars() {
460#define ICV_RT_SET(_Name, RTL) \
461 { \
462 auto &ICV = ICVs[_Name]; \
463 ICV.Setter = RTL; \
464 }
465#define ICV_RT_GET(Name, RTL) \
466 { \
467 auto &ICV = ICVs[Name]; \
468 ICV.Getter = RTL; \
469 }
470#define ICV_DATA_ENV(Enum, _Name, _EnvVarName, Init) \
471 { \
472 auto &ICV = ICVs[Enum]; \
473 ICV.Name = _Name; \
474 ICV.Kind = Enum; \
475 ICV.InitKind = Init; \
476 ICV.EnvVarName = _EnvVarName; \
477 switch (ICV.InitKind) { \
478 case ICV_IMPLEMENTATION_DEFINED: \
479 ICV.InitValue = nullptr; \
480 break; \
481 case ICV_ZERO: \
482 ICV.InitValue = ConstantInt::get( \
483 Type::getInt32Ty(OMPBuilder.Int32->getContext()), 0); \
484 break; \
485 case ICV_FALSE: \
486 ICV.InitValue = ConstantInt::getFalse(OMPBuilder.Int1->getContext()); \
487 break; \
488 case ICV_LAST: \
489 break; \
490 } \
491 }
492#include "llvm/Frontend/OpenMP/OMPKinds.def"
493 }
494
495 /// Returns true if the function declaration \p F matches the runtime
496 /// function types, that is, return type \p RTFRetType, and argument types
497 /// \p RTFArgTypes.
498 static bool declMatchesRTFTypes(Function *F, Type *RTFRetType,
499 SmallVector<Type *, 8> &RTFArgTypes) {
500 // TODO: We should output information to the user (under debug output
501 // and via remarks).
502
503 if (!F)
504 return false;
505 if (F->getReturnType() != RTFRetType)
506 return false;
507 if (F->arg_size() != RTFArgTypes.size())
508 return false;
509
510 auto *RTFTyIt = RTFArgTypes.begin();
511 for (Argument &Arg : F->args()) {
512 if (Arg.getType() != *RTFTyIt)
513 return false;
514
515 ++RTFTyIt;
516 }
517
518 return true;
519 }
520
521 // Helper to collect all uses of the declaration in the UsesMap.
522 unsigned collectUses(RuntimeFunctionInfo &RFI, bool CollectStats = true) {
523 unsigned NumUses = 0;
524 if (!RFI.Declaration)
525 return NumUses;
526 OMPBuilder.addAttributes(FnID: RFI.Kind, Fn&: *RFI.Declaration);
527
528 if (CollectStats) {
529 NumOpenMPRuntimeFunctionsIdentified += 1;
530 NumOpenMPRuntimeFunctionUsesIdentified += RFI.Declaration->getNumUses();
531 }
532
533 // TODO: We directly convert uses into proper calls and unknown uses.
534 for (Use &U : RFI.Declaration->uses()) {
535 if (Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser())) {
536 if (!CGSCC || CGSCC->empty() || CGSCC->contains(key: UserI->getFunction())) {
537 RFI.getOrCreateUseVector(F: UserI->getFunction()).push_back(Elt: &U);
538 ++NumUses;
539 }
540 } else {
541 RFI.getOrCreateUseVector(F: nullptr).push_back(Elt: &U);
542 ++NumUses;
543 }
544 }
545 return NumUses;
546 }
547
548 // Helper function to recollect uses of a runtime function.
549 void recollectUsesForFunction(RuntimeFunction RTF) {
550 auto &RFI = RFIs[RTF];
551 RFI.clearUsesMap();
552 collectUses(RFI, /*CollectStats*/ false);
553 }
554
555 /// Attach !callback metadata to a runtime function that takes one, so that
556 /// the Attributor sees the edge from the runtime call to the callback and
557 /// AAKernelInfo can look inside it. The runtime declares these functions
558 /// without the metadata, so OpenMPOpt supplies it from the table in
559 /// OMPKinds.def.
560 void setCallbackMetadata(Function *F, unsigned ArgNo, ArrayRef<int> Indices,
561 bool IsVarArg) {
562 if (!F || F->hasMetadata(KindID: LLVMContext::MD_callback))
563 return;
564
565 LLVMContext &Ctx = F->getContext();
566 MDBuilder MDB(Ctx);
567 F->addMetadata(KindID: LLVMContext::MD_callback,
568 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(CalleeArgNo: ArgNo, Arguments: Indices,
569 VarArgsArePassed: IsVarArg)}));
570 }
571
572 /// The callback a runtime function was handed, if it is one we can analyze.
573 /// Returns null when the call takes no callback, or when the callback is not
574 /// a definition this module can see, in which case its contents are unknown
575 /// and callers have to stay conservative.
576 static Function *getAnalyzableCallback(const CallBase &CB) {
577 Function *Callee = CB.getCalledFunction();
578 if (!Callee)
579 return nullptr;
580 MDNode *CallbackMD = Callee->getMetadata(KindID: LLVMContext::MD_callback);
581 if (!CallbackMD || CallbackMD->getNumOperands() == 0)
582 return nullptr;
583 // TODO: A runtime function with more than one callback would need each of
584 // them checked; none of the ones in the table have more than one.
585 auto *Encoding = dyn_cast<MDNode>(Val: CallbackMD->getOperand(I: 0));
586 if (!Encoding || Encoding->getNumOperands() == 0)
587 return nullptr;
588 auto *ArgNoMD = dyn_cast<ConstantAsMetadata>(Val: Encoding->getOperand(I: 0));
589 if (!ArgNoMD)
590 return nullptr;
591 uint64_t ArgNo =
592 cast<ConstantInt>(Val: ArgNoMD->getValue())->getLimitedValue(UINT64_MAX);
593 if (ArgNo >= CB.arg_size())
594 return nullptr;
595 auto *Callback =
596 dyn_cast<Function>(Val: CB.getArgOperand(i: ArgNo)->stripPointerCasts());
597 if (!Callback || Callback->isDeclaration())
598 return nullptr;
599 return Callback;
600 }
601
602 // Helper function to recollect uses of all runtime functions.
603 void recollectUses() {
604 for (int Idx = 0; Idx < RFIs.size(); ++Idx)
605 recollectUsesForFunction(RTF: static_cast<RuntimeFunction>(Idx));
606 }
607
608 // Helper function to inherit the calling convention of the function callee.
609 void setCallingConvention(FunctionCallee Callee, CallInst *CI) {
610 if (Function *Fn = dyn_cast<Function>(Val: Callee.getCallee()))
611 CI->setCallingConv(Fn->getCallingConv());
612 }
613
614 // Helper function to determine if it's legal to create a call to the runtime
615 // functions.
616 bool runtimeFnsAvailable(ArrayRef<RuntimeFunction> Fns) {
617 // We can always emit calls if we haven't yet linked in the runtime.
618 if (!OpenMPPostLink)
619 return true;
620
621 // Once the runtime has been already been linked in we cannot emit calls to
622 // any undefined functions.
623 for (RuntimeFunction Fn : Fns) {
624 RuntimeFunctionInfo &RFI = RFIs[Fn];
625
626 if (!RFI.Declaration || RFI.Declaration->isDeclaration())
627 return false;
628 }
629 return true;
630 }
631
632 /// Helper to initialize all runtime function information for those defined
633 /// in OpenMPKinds.def.
634 void initializeRuntimeFunctions(Module &M) {
635
636 // Helper macros for handling __VA_ARGS__ in OMP_RTL
637#define OMP_TYPE(VarName, ...) \
638 Type *VarName = OMPBuilder.VarName; \
639 (void)VarName;
640
641#define OMP_ARRAY_TYPE(VarName, ...) \
642 ArrayType *VarName##Ty = OMPBuilder.VarName##Ty; \
643 (void)VarName##Ty; \
644 PointerType *VarName##PtrTy = OMPBuilder.VarName##PtrTy; \
645 (void)VarName##PtrTy;
646
647#define OMP_FUNCTION_TYPE(VarName, ...) \
648 FunctionType *VarName = OMPBuilder.VarName; \
649 (void)VarName; \
650 PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr; \
651 (void)VarName##Ptr;
652
653#define OMP_STRUCT_TYPE(VarName, ...) \
654 StructType *VarName = OMPBuilder.VarName; \
655 (void)VarName; \
656 PointerType *VarName##Ptr = OMPBuilder.VarName##Ptr; \
657 (void)VarName##Ptr;
658
659#define OMP_RTL(_Enum, _Name, _IsVarArg, _ReturnType, ...) \
660 { \
661 SmallVector<Type *, 8> ArgsTypes({__VA_ARGS__}); \
662 Function *F = M.getFunction(_Name); \
663 RTLFunctions.insert(F); \
664 if (declMatchesRTFTypes(F, OMPBuilder._ReturnType, ArgsTypes)) { \
665 RuntimeFunctionIDMap[F] = _Enum; \
666 auto &RFI = RFIs[_Enum]; \
667 RFI.Kind = _Enum; \
668 RFI.Name = _Name; \
669 RFI.IsVarArg = _IsVarArg; \
670 RFI.ReturnType = OMPBuilder._ReturnType; \
671 RFI.ArgumentTypes = std::move(ArgsTypes); \
672 RFI.Declaration = F; \
673 unsigned NumUses = collectUses(RFI); \
674 (void)NumUses; \
675 LLVM_DEBUG({ \
676 dbgs() << TAG << RFI.Name << (RFI.Declaration ? "" : " not") \
677 << " found\n"; \
678 if (RFI.Declaration) \
679 dbgs() << TAG << "-> got " << NumUses << " uses in " \
680 << RFI.getNumFunctionsWithUses() \
681 << " different functions.\n"; \
682 }); \
683 } \
684 }
685
686#define OMP_RTL_CB_INFO(_Enum, _Name, _ArgNo, _ArgIndices, _IsVarArg) \
687 setCallbackMetadata(M.getFunction(_Name), _ArgNo, _ArgIndices, _IsVarArg);
688
689#include "llvm/Frontend/OpenMP/OMPKinds.def"
690
691 // Remove the `noinline` attribute from `__kmpc`, `ompx::` and `omp_`
692 // functions, except if `optnone` is present.
693 if (isOpenMPDevice(M)) {
694 for (Function &F : M) {
695 for (StringRef Prefix : {"__kmpc", "_ZN4ompx", "omp_"})
696 if (F.hasFnAttribute(Kind: Attribute::NoInline) &&
697 F.getName().starts_with(Prefix) &&
698 !F.hasFnAttribute(Kind: Attribute::OptimizeNone))
699 F.removeFnAttr(Kind: Attribute::NoInline);
700 }
701 }
702
703 // TODO: We should attach the attributes defined in OMPKinds.def.
704 }
705
706 /// Collection of known OpenMP runtime functions..
707 DenseSet<const Function *> RTLFunctions;
708
709 /// Indicates if we have already linked in the OpenMP device library.
710 bool OpenMPPostLink = false;
711
712 /// Kernels that OpenMPOpt transformed from generic to SPMD mode. Recorded at
713 /// the transform (changeToSPMDMode) so later cleanup does not have to
714 /// re-derive the mode. Such kernels no longer run a generic-mode state
715 /// machine, so the parallel data-sharing wrapper passed to __kmpc_parallel_60
716 /// is dead in them.
717 SmallPtrSet<Function *, 8> SPMDizedKernels;
718};
719
720template <typename Ty, bool InsertInvalidates = true>
721struct BooleanStateWithSetVector : public BooleanState {
722 bool contains(const Ty &Elem) const { return Set.contains(Elem); }
723 bool insert(const Ty &Elem) {
724 if (InsertInvalidates)
725 BooleanState::indicatePessimisticFixpoint();
726 return Set.insert(Elem);
727 }
728
729 const Ty &operator[](int Idx) const { return Set[Idx]; }
730 bool operator==(const BooleanStateWithSetVector &RHS) const {
731 return BooleanState::operator==(R: RHS) && Set == RHS.Set;
732 }
733 bool operator!=(const BooleanStateWithSetVector &RHS) const {
734 return !(*this == RHS);
735 }
736
737 bool empty() const { return Set.empty(); }
738 size_t size() const { return Set.size(); }
739
740 /// "Clamp" this state with \p RHS.
741 BooleanStateWithSetVector &operator^=(const BooleanStateWithSetVector &RHS) {
742 BooleanState::operator^=(R: RHS);
743 Set.insert_range(RHS.Set);
744 return *this;
745 }
746
747private:
748 /// A set to keep track of elements.
749 SetVector<Ty> Set;
750
751public:
752 typename decltype(Set)::iterator begin() { return Set.begin(); }
753 typename decltype(Set)::iterator end() { return Set.end(); }
754 typename decltype(Set)::const_iterator begin() const { return Set.begin(); }
755 typename decltype(Set)::const_iterator end() const { return Set.end(); }
756};
757
758template <typename Ty, bool InsertInvalidates = true>
759using BooleanStateWithPtrSetVector =
760 BooleanStateWithSetVector<Ty *, InsertInvalidates>;
761
762struct KernelInfoState : AbstractState {
763 /// Flag to track if we reached a fixpoint.
764 bool IsAtFixpoint = false;
765
766 /// The parallel regions (identified by the outlined parallel functions) that
767 /// can be reached from the associated function.
768 BooleanStateWithPtrSetVector<CallBase, /* InsertInvalidates */ false>
769 ReachedKnownParallelRegions;
770
771 /// State to track what parallel region we might reach.
772 BooleanStateWithPtrSetVector<CallBase> ReachedUnknownParallelRegions;
773
774 /// State to track if we are in SPMD-mode, assumed or know, and why we decided
775 /// we cannot be. If it is assumed, then RequiresFullRuntime should also be
776 /// false.
777 BooleanStateWithPtrSetVector<Instruction, false> SPMDCompatibilityTracker;
778
779 /// The __kmpc_target_init call in this kernel, if any. If we find more than
780 /// one we abort as the kernel is malformed.
781 CallBase *KernelInitCB = nullptr;
782
783 /// The constant kernel environement as taken from and passed to
784 /// __kmpc_target_init.
785 ConstantStruct *KernelEnvC = nullptr;
786
787 /// The __kmpc_target_deinit call in this kernel, if any. If we find more than
788 /// one we abort as the kernel is malformed.
789 CallBase *KernelDeinitCB = nullptr;
790
791 /// Flag to indicate if the associated function is a kernel entry.
792 bool IsKernelEntry = false;
793
794 /// State to track what kernel entries can reach the associated function.
795 BooleanStateWithPtrSetVector<Function, false> ReachingKernelEntries;
796
797 /// State to indicate if we can track parallel level of the associated
798 /// function. We will give up tracking if we encounter unknown caller or the
799 /// caller is __kmpc_parallel_60.
800 BooleanStateWithSetVector<uint8_t> ParallelLevels;
801
802 /// Flag that indicates if the kernel has nested Parallelism
803 bool NestedParallelism = false;
804
805 /// Abstract State interface
806 ///{
807
808 KernelInfoState() = default;
809 KernelInfoState(bool BestState) {
810 if (!BestState)
811 indicatePessimisticFixpoint();
812 }
813
814 /// See AbstractState::isValidState(...)
815 bool isValidState() const override { return true; }
816
817 /// See AbstractState::isAtFixpoint(...)
818 bool isAtFixpoint() const override { return IsAtFixpoint; }
819
820 /// See AbstractState::indicatePessimisticFixpoint(...)
821 ChangeStatus indicatePessimisticFixpoint() override {
822 IsAtFixpoint = true;
823 ParallelLevels.indicatePessimisticFixpoint();
824 ReachingKernelEntries.indicatePessimisticFixpoint();
825 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
826 ReachedKnownParallelRegions.indicatePessimisticFixpoint();
827 ReachedUnknownParallelRegions.indicatePessimisticFixpoint();
828 NestedParallelism = true;
829 return ChangeStatus::CHANGED;
830 }
831
832 /// See AbstractState::indicateOptimisticFixpoint(...)
833 ChangeStatus indicateOptimisticFixpoint() override {
834 IsAtFixpoint = true;
835 ParallelLevels.indicateOptimisticFixpoint();
836 ReachingKernelEntries.indicateOptimisticFixpoint();
837 SPMDCompatibilityTracker.indicateOptimisticFixpoint();
838 ReachedKnownParallelRegions.indicateOptimisticFixpoint();
839 ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
840 return ChangeStatus::UNCHANGED;
841 }
842
843 /// Return the assumed state
844 KernelInfoState &getAssumed() { return *this; }
845 const KernelInfoState &getAssumed() const { return *this; }
846
847 bool operator==(const KernelInfoState &RHS) const {
848 if (SPMDCompatibilityTracker != RHS.SPMDCompatibilityTracker)
849 return false;
850 if (ReachedKnownParallelRegions != RHS.ReachedKnownParallelRegions)
851 return false;
852 if (ReachedUnknownParallelRegions != RHS.ReachedUnknownParallelRegions)
853 return false;
854 if (ReachingKernelEntries != RHS.ReachingKernelEntries)
855 return false;
856 if (ParallelLevels != RHS.ParallelLevels)
857 return false;
858 if (NestedParallelism != RHS.NestedParallelism)
859 return false;
860 return true;
861 }
862
863 /// Returns true if this kernel contains any OpenMP parallel regions.
864 bool mayContainParallelRegion() {
865 return !ReachedKnownParallelRegions.empty() ||
866 !ReachedUnknownParallelRegions.empty();
867 }
868
869 /// Return empty set as the best state of potential values.
870 static KernelInfoState getBestState() { return KernelInfoState(true); }
871
872 static KernelInfoState getBestState(KernelInfoState &KIS) {
873 return getBestState();
874 }
875
876 /// Return full set as the worst state of potential values.
877 static KernelInfoState getWorstState() { return KernelInfoState(false); }
878
879 /// "Clamp" this state with \p KIS.
880 KernelInfoState operator^=(const KernelInfoState &KIS) {
881 // Do not merge two different _init and _deinit call sites.
882 if (KIS.KernelInitCB) {
883 if (KernelInitCB && KernelInitCB != KIS.KernelInitCB)
884 llvm_unreachable("Kernel that calls another kernel violates OpenMP-Opt "
885 "assumptions.");
886 KernelInitCB = KIS.KernelInitCB;
887 }
888 if (KIS.KernelDeinitCB) {
889 if (KernelDeinitCB && KernelDeinitCB != KIS.KernelDeinitCB)
890 llvm_unreachable("Kernel that calls another kernel violates OpenMP-Opt "
891 "assumptions.");
892 KernelDeinitCB = KIS.KernelDeinitCB;
893 }
894 if (KIS.KernelEnvC) {
895 if (KernelEnvC && KernelEnvC != KIS.KernelEnvC)
896 llvm_unreachable("Kernel that calls another kernel violates OpenMP-Opt "
897 "assumptions.");
898 KernelEnvC = KIS.KernelEnvC;
899 }
900 SPMDCompatibilityTracker ^= KIS.SPMDCompatibilityTracker;
901 ReachedKnownParallelRegions ^= KIS.ReachedKnownParallelRegions;
902 ReachedUnknownParallelRegions ^= KIS.ReachedUnknownParallelRegions;
903 NestedParallelism |= KIS.NestedParallelism;
904 return *this;
905 }
906
907 KernelInfoState operator&=(const KernelInfoState &KIS) {
908 return (*this ^= KIS);
909 }
910
911 ///}
912};
913
914/// Used to map the values physically (in the IR) stored in an offload
915/// array, to a vector in memory.
916struct OffloadArray {
917 /// Physical array (in the IR).
918 AllocaInst *Array = nullptr;
919 /// Mapped values.
920 SmallVector<Value *, 8> StoredValues;
921 /// Last stores made in the offload array.
922 SmallVector<StoreInst *, 8> LastAccesses;
923
924 OffloadArray() = default;
925
926 /// Initializes the OffloadArray with the values stored in \p Array before
927 /// instruction \p Before is reached. Returns false if the initialization
928 /// fails.
929 /// This MUST be used immediately after the construction of the object.
930 bool initialize(AllocaInst &Array, Instruction &Before) {
931 if (!getValues(Array, Before))
932 return false;
933
934 this->Array = &Array;
935 return true;
936 }
937
938 static const unsigned DeviceIDArgNum = 1;
939 static const unsigned BasePtrsArgNum = 3;
940 static const unsigned PtrsArgNum = 4;
941 static const unsigned SizesArgNum = 5;
942
943private:
944 /// Traverses the BasicBlock where \p Array is, collecting the stores made to
945 /// \p Array, leaving StoredValues with the values stored before the
946 /// instruction \p Before is reached.
947 bool getValues(AllocaInst &Array, Instruction &Before) {
948 // Initialize containers.
949 const DataLayout &DL = Array.getDataLayout();
950 std::optional<TypeSize> ArraySize = Array.getAllocationSize(DL);
951 if (!ArraySize || !ArraySize->isFixed())
952 return false;
953 const unsigned int PointerSize = DL.getPointerSize();
954 const uint64_t NumValues = ArraySize->getFixedValue() / PointerSize;
955 StoredValues.assign(NumElts: NumValues, Elt: nullptr);
956 LastAccesses.assign(NumElts: NumValues, Elt: nullptr);
957
958 // TODO: This assumes the instruction \p Before is in the same
959 // BasicBlock as Array. Make it general, for any control flow graph.
960 BasicBlock *BB = Array.getParent();
961 if (BB != Before.getParent())
962 return false;
963
964 for (Instruction &I : *BB) {
965 if (&I == &Before)
966 break;
967
968 if (!isa<StoreInst>(Val: &I))
969 continue;
970
971 auto *S = cast<StoreInst>(Val: &I);
972 int64_t Offset = -1;
973 auto *Dst =
974 GetPointerBaseWithConstantOffset(Ptr: S->getPointerOperand(), Offset, DL);
975 if (Dst == &Array) {
976 int64_t Idx = Offset / PointerSize;
977 // Ignore updates that must be UB (probably in dead code at runtime)
978 if ((uint64_t)Idx < NumValues) {
979 StoredValues[Idx] = getUnderlyingObject(V: S->getValueOperand());
980 LastAccesses[Idx] = S;
981 }
982 }
983 }
984
985 return isFilled();
986 }
987
988 /// Returns true if all values in StoredValues and
989 /// LastAccesses are not nullptrs.
990 bool isFilled() {
991 const unsigned NumValues = StoredValues.size();
992 for (unsigned I = 0; I < NumValues; ++I) {
993 if (!StoredValues[I] || !LastAccesses[I])
994 return false;
995 }
996
997 return true;
998 }
999};
1000
1001struct OpenMPOpt {
1002
1003 using OptimizationRemarkGetter =
1004 function_ref<OptimizationRemarkEmitter &(Function *)>;
1005
1006 OpenMPOpt(SmallVectorImpl<Function *> &SCC, CallGraphUpdater &CGUpdater,
1007 OptimizationRemarkGetter OREGetter,
1008 OMPInformationCache &OMPInfoCache, Attributor &A)
1009 : M(*(*SCC.begin())->getParent()), SCC(SCC), CGUpdater(CGUpdater),
1010 OREGetter(OREGetter), OMPInfoCache(OMPInfoCache), A(A) {}
1011
1012 /// Check if any remarks are enabled for openmp-opt
1013 bool remarksEnabled() {
1014 auto &Ctx = M.getContext();
1015 return Ctx.getDiagHandlerPtr()->isAnyRemarkEnabled(DEBUG_TYPE);
1016 }
1017
1018 /// Run all OpenMP optimizations on the underlying SCC.
1019 bool run(bool IsModulePass) {
1020 if (SCC.empty())
1021 return false;
1022
1023 bool Changed = false;
1024
1025 LLVM_DEBUG(dbgs() << TAG << "Run on SCC with " << SCC.size()
1026 << " functions\n");
1027
1028 if (IsModulePass) {
1029 Changed |= runAttributor(IsModulePass);
1030
1031 // Recollect uses, in case Attributor deleted any.
1032 OMPInfoCache.recollectUses();
1033
1034 // TODO: This should be folded into buildCustomStateMachine.
1035 Changed |= rewriteDeviceCodeStateMachine();
1036
1037 // Drop the parallel data-sharing wrapper from __kmpc_parallel_60 calls in
1038 // SPMD kernels, where the runtime never uses it, so the (otherwise dead)
1039 // wrapper can be eliminated instead of lingering as a non-kernel LDS
1040 // user.
1041 Changed |= removeSPMDParallelWrappers();
1042
1043 if (remarksEnabled())
1044 analysisGlobalization();
1045 } else {
1046 if (PrintICVValues)
1047 printICVs();
1048 if (PrintOpenMPKernels)
1049 printKernels();
1050
1051 Changed |= runAttributor(IsModulePass);
1052
1053 // Recollect uses, in case Attributor deleted any.
1054 OMPInfoCache.recollectUses();
1055
1056 Changed |= deleteParallelRegions();
1057
1058 if (HideMemoryTransferLatency)
1059 Changed |= hideMemTransfersLatency();
1060 Changed |= deduplicateRuntimeCalls();
1061 if (EnableParallelRegionMerging) {
1062 if (mergeParallelRegions()) {
1063 deduplicateRuntimeCalls();
1064 Changed = true;
1065 }
1066 }
1067 }
1068
1069 if (OMPInfoCache.OpenMPPostLink)
1070 Changed |= removeRuntimeSymbols();
1071
1072 return Changed;
1073 }
1074
1075 /// Print initial ICV values for testing.
1076 /// FIXME: This should be done from the Attributor once it is added.
1077 void printICVs() const {
1078 InternalControlVar ICVs[] = {ICV_nthreads, ICV_active_levels, ICV_cancel,
1079 ICV_proc_bind};
1080
1081 for (Function *F : SCC) {
1082 for (auto ICV : ICVs) {
1083 auto ICVInfo = OMPInfoCache.ICVs[ICV];
1084 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
1085 return ORA << "OpenMP ICV " << ore::NV("OpenMPICV", ICVInfo.Name)
1086 << " Value: "
1087 << (ICVInfo.InitValue
1088 ? toString(I: ICVInfo.InitValue->getValue(), Radix: 10, Signed: true)
1089 : "IMPLEMENTATION_DEFINED");
1090 };
1091
1092 emitRemark<OptimizationRemarkAnalysis>(F, RemarkName: "OpenMPICVTracker", RemarkCB&: Remark);
1093 }
1094 }
1095 }
1096
1097 /// Print OpenMP GPU kernels for testing.
1098 void printKernels() const {
1099 for (Function *F : SCC) {
1100 if (!omp::isOpenMPKernel(Fn&: *F))
1101 continue;
1102
1103 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
1104 return ORA << "OpenMP GPU kernel "
1105 << ore::NV("OpenMPGPUKernel", F->getName()) << "\n";
1106 };
1107
1108 emitRemark<OptimizationRemarkAnalysis>(F, RemarkName: "OpenMPGPU", RemarkCB&: Remark);
1109 }
1110 }
1111
1112 /// Return the call if \p U is a callee use in a regular call. If \p RFI is
1113 /// given it has to be the callee or a nullptr is returned.
1114 static CallInst *getCallIfRegularCall(
1115 Use &U, OMPInformationCache::RuntimeFunctionInfo *RFI = nullptr) {
1116 CallInst *CI = dyn_cast<CallInst>(Val: U.getUser());
1117 if (CI && CI->isCallee(U: &U) && !CI->hasOperandBundles() &&
1118 (!RFI ||
1119 (RFI->Declaration && CI->getCalledFunction() == RFI->Declaration)))
1120 return CI;
1121 return nullptr;
1122 }
1123
1124 /// Return the call if \p V is a regular call. If \p RFI is given it has to be
1125 /// the callee or a nullptr is returned.
1126 static CallInst *getCallIfRegularCall(
1127 Value &V, OMPInformationCache::RuntimeFunctionInfo *RFI = nullptr) {
1128 CallInst *CI = dyn_cast<CallInst>(Val: &V);
1129 if (CI && !CI->hasOperandBundles() &&
1130 (!RFI ||
1131 (RFI->Declaration && CI->getCalledFunction() == RFI->Declaration)))
1132 return CI;
1133 return nullptr;
1134 }
1135
1136private:
1137 /// Merge parallel regions when it is safe.
1138 bool mergeParallelRegions() {
1139 const unsigned CallbackCalleeOperand = 2;
1140 const unsigned CallbackFirstArgOperand = 3;
1141 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1142
1143 // Check if there are any __kmpc_fork_call calls to merge.
1144 OMPInformationCache::RuntimeFunctionInfo &RFI =
1145 OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
1146
1147 if (!RFI.Declaration)
1148 return false;
1149
1150 // Unmergable calls that prevent merging a parallel region.
1151 OMPInformationCache::RuntimeFunctionInfo UnmergableCallsInfo[] = {
1152 OMPInfoCache.RFIs[OMPRTL___kmpc_push_proc_bind],
1153 OMPInfoCache.RFIs[OMPRTL___kmpc_push_num_threads],
1154 };
1155
1156 bool Changed = false;
1157 LoopInfo *LI = nullptr;
1158 DominatorTree *DT = nullptr;
1159
1160 SmallDenseMap<BasicBlock *, SmallPtrSet<Instruction *, 4>> BB2PRMap;
1161
1162 BasicBlock *StartBB = nullptr, *EndBB = nullptr;
1163 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1164 ArrayRef<BasicBlock *> DeallocBlocks) {
1165 BasicBlock *CGStartBB = CodeGenIP.getBlock();
1166 BasicBlock *CGEndBB =
1167 SplitBlock(Old: CGStartBB, SplitPt: &*CodeGenIP.getPoint(), DT, LI);
1168 assert(StartBB != nullptr && "StartBB should not be null");
1169 CGStartBB->getTerminator()->setSuccessor(Idx: 0, BB: StartBB);
1170 assert(EndBB != nullptr && "EndBB should not be null");
1171 EndBB->getTerminator()->setSuccessor(Idx: 0, BB: CGEndBB);
1172 return Error::success();
1173 };
1174
1175 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
1176 Value &Inner, Value *&ReplacementValue) -> InsertPointTy {
1177 ReplacementValue = &Inner;
1178 return CodeGenIP;
1179 };
1180
1181 auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };
1182
1183 /// Create a sequential execution region within a merged parallel region,
1184 /// encapsulated in a master construct with a barrier for synchronization.
1185 auto CreateSequentialRegion = [&](Function *OuterFn,
1186 BasicBlock *OuterPredBB,
1187 Instruction *SeqStartI,
1188 Instruction *SeqEndI) {
1189 // Isolate the instructions of the sequential region to a separate
1190 // block.
1191 BasicBlock *ParentBB = SeqStartI->getParent();
1192 BasicBlock *SeqEndBB =
1193 SplitBlock(Old: ParentBB, SplitPt: SeqEndI->getNextNode(), DT, LI);
1194 BasicBlock *SeqAfterBB =
1195 SplitBlock(Old: SeqEndBB, SplitPt: &*SeqEndBB->getFirstInsertionPt(), DT, LI);
1196 BasicBlock *SeqStartBB =
1197 SplitBlock(Old: ParentBB, SplitPt: SeqStartI, DT, LI, MSSAU: nullptr, BBName: "seq.par.merged");
1198
1199 assert(ParentBB->getUniqueSuccessor() == SeqStartBB &&
1200 "Expected a different CFG");
1201 const DebugLoc DL = ParentBB->getTerminator()->getDebugLoc();
1202 ParentBB->getTerminator()->eraseFromParent();
1203
1204 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1205 ArrayRef<BasicBlock *> DeallocBlocks) {
1206 BasicBlock *CGStartBB = CodeGenIP.getBlock();
1207 BasicBlock *CGEndBB =
1208 SplitBlock(Old: CGStartBB, SplitPt: &*CodeGenIP.getPoint(), DT, LI);
1209 assert(SeqStartBB != nullptr && "SeqStartBB should not be null");
1210 CGStartBB->getTerminator()->setSuccessor(Idx: 0, BB: SeqStartBB);
1211 assert(SeqEndBB != nullptr && "SeqEndBB should not be null");
1212 SeqEndBB->getTerminator()->setSuccessor(Idx: 0, BB: CGEndBB);
1213 return Error::success();
1214 };
1215 auto FiniCB = [&](InsertPointTy CodeGenIP) { return Error::success(); };
1216
1217 // Find outputs from the sequential region to outside users and
1218 // broadcast their values to them.
1219 for (Instruction &I : *SeqStartBB) {
1220 SmallPtrSet<Instruction *, 4> OutsideUsers;
1221 for (User *Usr : I.users()) {
1222 Instruction &UsrI = *cast<Instruction>(Val: Usr);
1223 // Ignore outputs to LT intrinsics, code extraction for the merged
1224 // parallel region will fix them.
1225 if (UsrI.isLifetimeStartOrEnd())
1226 continue;
1227
1228 if (UsrI.getParent() != SeqStartBB)
1229 OutsideUsers.insert(Ptr: &UsrI);
1230 }
1231
1232 if (OutsideUsers.empty())
1233 continue;
1234
1235 // Emit an alloca in the outer region to store the broadcasted
1236 // value.
1237 const DataLayout &DL = M.getDataLayout();
1238 AllocaInst *AllocaI = new AllocaInst(
1239 I.getType(), DL.getAllocaAddrSpace(), nullptr,
1240 I.getName() + ".seq.output.alloc", OuterFn->front().begin());
1241
1242 // Emit a store instruction in the sequential BB to update the
1243 // value.
1244 new StoreInst(&I, AllocaI, SeqStartBB->getTerminator()->getIterator());
1245
1246 // Emit a load instruction and replace the use of the output value
1247 // with it.
1248 for (Instruction *UsrI : OutsideUsers) {
1249 LoadInst *LoadI = new LoadInst(I.getType(), AllocaI,
1250 I.getName() + ".seq.output.load",
1251 UsrI->getIterator());
1252 UsrI->replaceUsesOfWith(From: &I, To: LoadI);
1253 }
1254 }
1255
1256 OpenMPIRBuilder::LocationDescription Loc(
1257 InsertPointTy(ParentBB, ParentBB->end()), DL);
1258 OpenMPIRBuilder::InsertPointTy SeqAfterIP = cantFail(
1259 ValOrErr: OMPInfoCache.OMPBuilder.createMaster(Loc, BodyGenCB, FiniCB));
1260 cantFail(ValOrErr: OMPInfoCache.OMPBuilder.createBarrier(Loc: {SeqAfterIP, DL},
1261 Kind: OMPD_parallel));
1262
1263 UncondBrInst::Create(Target: SeqAfterBB, InsertBefore: SeqAfterIP.getBlock());
1264
1265 LLVM_DEBUG(dbgs() << TAG << "After sequential inlining " << *OuterFn
1266 << "\n");
1267 };
1268
1269 // Helper to merge the __kmpc_fork_call calls in MergableCIs. They are all
1270 // contained in BB and only separated by instructions that can be
1271 // redundantly executed in parallel. The block BB is split before the first
1272 // call (in MergableCIs) and after the last so the entire region we merge
1273 // into a single parallel region is contained in a single basic block
1274 // without any other instructions. We use the OpenMPIRBuilder to outline
1275 // that block and call the resulting function via __kmpc_fork_call.
1276 auto Merge = [&](const SmallVectorImpl<CallInst *> &MergableCIs,
1277 BasicBlock *BB) {
1278 // TODO: Change the interface to allow single CIs expanded, e.g, to
1279 // include an outer loop.
1280 assert(MergableCIs.size() > 1 && "Assumed multiple mergable CIs");
1281
1282 auto Remark = [&](OptimizationRemark OR) {
1283 OR << "Parallel region merged with parallel region"
1284 << (MergableCIs.size() > 2 ? "s" : "") << " at ";
1285 for (auto *CI : llvm::drop_begin(RangeOrContainer: MergableCIs)) {
1286 OR << ore::NV("OpenMPParallelMerge", CI->getDebugLoc());
1287 if (CI != MergableCIs.back())
1288 OR << ", ";
1289 }
1290 return OR << ".";
1291 };
1292
1293 emitRemark<OptimizationRemark>(I: MergableCIs.front(), RemarkName: "OMP150", RemarkCB&: Remark);
1294
1295 Function *OriginalFn = BB->getParent();
1296 LLVM_DEBUG(dbgs() << TAG << "Merge " << MergableCIs.size()
1297 << " parallel regions in " << OriginalFn->getName()
1298 << "\n");
1299
1300 // Isolate the calls to merge in a separate block.
1301 EndBB = SplitBlock(Old: BB, SplitPt: MergableCIs.back()->getNextNode(), DT, LI);
1302 BasicBlock *AfterBB =
1303 SplitBlock(Old: EndBB, SplitPt: &*EndBB->getFirstInsertionPt(), DT, LI);
1304 StartBB = SplitBlock(Old: BB, SplitPt: MergableCIs.front(), DT, LI, MSSAU: nullptr,
1305 BBName: "omp.par.merged");
1306
1307 assert(BB->getUniqueSuccessor() == StartBB && "Expected a different CFG");
1308 const DebugLoc DL = BB->getTerminator()->getDebugLoc();
1309 BB->getTerminator()->eraseFromParent();
1310
1311 // Create sequential regions for sequential instructions that are
1312 // in-between mergable parallel regions.
1313 for (auto *It = MergableCIs.begin(), *End = MergableCIs.end() - 1;
1314 It != End; ++It) {
1315 Instruction *ForkCI = *It;
1316 Instruction *NextForkCI = *(It + 1);
1317
1318 // Continue if there are not in-between instructions.
1319 if (ForkCI->getNextNode() == NextForkCI)
1320 continue;
1321
1322 CreateSequentialRegion(OriginalFn, BB, ForkCI->getNextNode(),
1323 NextForkCI->getPrevNode());
1324 }
1325
1326 OpenMPIRBuilder::LocationDescription Loc(InsertPointTy(BB, BB->end()),
1327 DL);
1328 IRBuilder<>::InsertPoint AllocaIP(
1329 &OriginalFn->getEntryBlock(),
1330 OriginalFn->getEntryBlock().getFirstInsertionPt());
1331 // Create the merged parallel region with default proc binding, to
1332 // avoid overriding binding settings, and without explicit cancellation.
1333 OpenMPIRBuilder::InsertPointTy AfterIP =
1334 cantFail(ValOrErr: OMPInfoCache.OMPBuilder.createParallel(
1335 Loc, AllocaIP, /* DeallocBlocks */ {}, BodyGenCB, PrivCB, FiniCB,
1336 IfCondition: nullptr, NumThreads: nullptr, ProcBind: OMP_PROC_BIND_default,
1337 /* IsCancellable */ false));
1338 UncondBrInst::Create(Target: AfterBB, InsertBefore: AfterIP.getBlock());
1339
1340 // Perform the actual outlining.
1341 OMPInfoCache.OMPBuilder.finalize(Fn: OriginalFn);
1342
1343 Function *OutlinedFn = MergableCIs.front()->getCaller();
1344
1345 // Replace the __kmpc_fork_call calls with direct calls to the outlined
1346 // callbacks.
1347 SmallVector<Value *, 8> Args;
1348 for (auto *CI : MergableCIs) {
1349 Value *Callee = CI->getArgOperand(i: CallbackCalleeOperand);
1350 FunctionType *FT = OMPInfoCache.OMPBuilder.ParallelTask;
1351 Args.clear();
1352 Args.push_back(Elt: OutlinedFn->getArg(i: 0));
1353 Args.push_back(Elt: OutlinedFn->getArg(i: 1));
1354 for (unsigned U = CallbackFirstArgOperand, E = CI->arg_size(); U < E;
1355 ++U)
1356 Args.push_back(Elt: CI->getArgOperand(i: U));
1357
1358 CallInst *NewCI =
1359 CallInst::Create(Ty: FT, Func: Callee, Args, NameStr: "", InsertBefore: CI->getIterator());
1360 if (CI->getDebugLoc())
1361 NewCI->setDebugLoc(CI->getDebugLoc());
1362
1363 // Forward parameter attributes from the callback to the callee.
1364 for (unsigned U = CallbackFirstArgOperand, E = CI->arg_size(); U < E;
1365 ++U)
1366 for (const Attribute &A : CI->getAttributes().getParamAttrs(ArgNo: U))
1367 NewCI->addParamAttr(
1368 ArgNo: U - (CallbackFirstArgOperand - CallbackCalleeOperand), Attr: A);
1369
1370 // Emit an explicit barrier to replace the implicit fork-join barrier.
1371 if (CI != MergableCIs.back()) {
1372 // TODO: Remove barrier if the merged parallel region includes the
1373 // 'nowait' clause.
1374 cantFail(ValOrErr: OMPInfoCache.OMPBuilder.createBarrier(
1375 Loc: {InsertPointTy(NewCI->getParent(),
1376 NewCI->getNextNode()->getIterator()),
1377 NewCI->getDebugLoc()},
1378 Kind: OMPD_parallel));
1379 }
1380
1381 CI->eraseFromParent();
1382 }
1383
1384 assert(OutlinedFn != OriginalFn && "Outlining failed");
1385 CGUpdater.registerOutlinedFunction(OriginalFn&: *OriginalFn, NewFn&: *OutlinedFn);
1386 CGUpdater.reanalyzeFunction(Fn&: *OriginalFn);
1387
1388 NumOpenMPParallelRegionsMerged += MergableCIs.size();
1389
1390 return true;
1391 };
1392
1393 // Helper function that identifes sequences of
1394 // __kmpc_fork_call uses in a basic block.
1395 auto DetectPRsCB = [&](Use &U, Function &F) {
1396 CallInst *CI = getCallIfRegularCall(U, RFI: &RFI);
1397 BB2PRMap[CI->getParent()].insert(Ptr: CI);
1398
1399 return false;
1400 };
1401
1402 BB2PRMap.clear();
1403 RFI.foreachUse(SCC, CB: DetectPRsCB);
1404 SmallVector<SmallVector<CallInst *, 4>, 4> MergableCIsVector;
1405 // Find mergable parallel regions within a basic block that are
1406 // safe to merge, that is any in-between instructions can safely
1407 // execute in parallel after merging.
1408 // TODO: support merging across basic-blocks.
1409 for (auto &It : BB2PRMap) {
1410 auto &CIs = It.getSecond();
1411 if (CIs.size() < 2)
1412 continue;
1413
1414 BasicBlock *BB = It.getFirst();
1415 SmallVector<CallInst *, 4> MergableCIs;
1416
1417 /// Returns true if the instruction is mergable, false otherwise.
1418 /// A terminator instruction is unmergable by definition since merging
1419 /// works within a BB. Instructions before the mergable region are
1420 /// mergable if they are not calls to OpenMP runtime functions that may
1421 /// set different execution parameters for subsequent parallel regions.
1422 /// Instructions in-between parallel regions are mergable if they are not
1423 /// calls to any non-intrinsic function since that may call a non-mergable
1424 /// OpenMP runtime function.
1425 auto IsMergable = [&](Instruction &I, bool IsBeforeMergableRegion) {
1426 // We do not merge across BBs, hence return false (unmergable) if the
1427 // instruction is a terminator.
1428 if (I.isTerminator())
1429 return false;
1430
1431 if (!isa<CallInst>(Val: &I))
1432 return true;
1433
1434 CallInst *CI = cast<CallInst>(Val: &I);
1435 if (IsBeforeMergableRegion) {
1436 Function *CalledFunction = CI->getCalledFunction();
1437 if (!CalledFunction)
1438 return false;
1439 // Return false (unmergable) if the call before the parallel
1440 // region calls an explicit affinity (proc_bind) or number of
1441 // threads (num_threads) compiler-generated function. Those settings
1442 // may be incompatible with following parallel regions.
1443 // TODO: ICV tracking to detect compatibility.
1444 for (const auto &RFI : UnmergableCallsInfo) {
1445 if (CalledFunction == RFI.Declaration)
1446 return false;
1447 }
1448 } else {
1449 // Return false (unmergable) if there is a call instruction
1450 // in-between parallel regions when it is not an intrinsic. It
1451 // may call an unmergable OpenMP runtime function in its callpath.
1452 // TODO: Keep track of possible OpenMP calls in the callpath.
1453 if (!isa<IntrinsicInst>(Val: CI))
1454 return false;
1455 }
1456
1457 return true;
1458 };
1459 // Find maximal number of parallel region CIs that are safe to merge.
1460 for (auto It = BB->begin(), End = BB->end(); It != End;) {
1461 Instruction &I = *It;
1462 ++It;
1463
1464 if (CIs.count(Ptr: &I)) {
1465 MergableCIs.push_back(Elt: cast<CallInst>(Val: &I));
1466 continue;
1467 }
1468
1469 // Continue expanding if the instruction is mergable.
1470 if (IsMergable(I, MergableCIs.empty()))
1471 continue;
1472
1473 // Forward the instruction iterator to skip the next parallel region
1474 // since there is an unmergable instruction which can affect it.
1475 for (; It != End; ++It) {
1476 Instruction &SkipI = *It;
1477 if (CIs.count(Ptr: &SkipI)) {
1478 LLVM_DEBUG(dbgs() << TAG << "Skip parallel region " << SkipI
1479 << " due to " << I << "\n");
1480 ++It;
1481 break;
1482 }
1483 }
1484
1485 // Store mergable regions found.
1486 if (MergableCIs.size() > 1) {
1487 MergableCIsVector.push_back(Elt: MergableCIs);
1488 LLVM_DEBUG(dbgs() << TAG << "Found " << MergableCIs.size()
1489 << " parallel regions in block " << BB->getName()
1490 << " of function " << BB->getParent()->getName()
1491 << "\n";);
1492 }
1493
1494 MergableCIs.clear();
1495 }
1496
1497 if (!MergableCIsVector.empty()) {
1498 Changed = true;
1499
1500 for (auto &MergableCIs : MergableCIsVector)
1501 Merge(MergableCIs, BB);
1502 MergableCIsVector.clear();
1503 }
1504 }
1505
1506 if (Changed) {
1507 /// Re-collect use for fork calls, emitted barrier calls, and
1508 /// any emitted master/end_master calls.
1509 OMPInfoCache.recollectUsesForFunction(RTF: OMPRTL___kmpc_fork_call);
1510 OMPInfoCache.recollectUsesForFunction(RTF: OMPRTL___kmpc_barrier);
1511 OMPInfoCache.recollectUsesForFunction(RTF: OMPRTL___kmpc_master);
1512 OMPInfoCache.recollectUsesForFunction(RTF: OMPRTL___kmpc_end_master);
1513 }
1514
1515 return Changed;
1516 }
1517
1518 /// Try to delete parallel regions if possible.
1519 bool deleteParallelRegions() {
1520 const unsigned CallbackCalleeOperand = 2;
1521
1522 OMPInformationCache::RuntimeFunctionInfo &RFI =
1523 OMPInfoCache.RFIs[OMPRTL___kmpc_fork_call];
1524
1525 if (!RFI.Declaration)
1526 return false;
1527
1528 bool Changed = false;
1529 auto DeleteCallCB = [&](Use &U, Function &) {
1530 CallInst *CI = getCallIfRegularCall(U);
1531 if (!CI)
1532 return false;
1533 auto *Fn = dyn_cast<Function>(
1534 Val: CI->getArgOperand(i: CallbackCalleeOperand)->stripPointerCasts());
1535 if (!Fn)
1536 return false;
1537 if (!Fn->onlyReadsMemory())
1538 return false;
1539 if (!Fn->hasFnAttribute(Kind: Attribute::WillReturn))
1540 return false;
1541
1542 LLVM_DEBUG(dbgs() << TAG << "Delete read-only parallel region in "
1543 << CI->getCaller()->getName() << "\n");
1544
1545 auto Remark = [&](OptimizationRemark OR) {
1546 return OR << "Removing parallel region with no side-effects.";
1547 };
1548 emitRemark<OptimizationRemark>(I: CI, RemarkName: "OMP160", RemarkCB&: Remark);
1549
1550 CI->eraseFromParent();
1551 Changed = true;
1552 ++NumOpenMPParallelRegionsDeleted;
1553 return true;
1554 };
1555
1556 RFI.foreachUse(SCC, CB: DeleteCallCB);
1557
1558 return Changed;
1559 }
1560
1561 /// Try to eliminate runtime calls by reusing existing ones.
1562 bool deduplicateRuntimeCalls() {
1563 bool Changed = false;
1564
1565 RuntimeFunction DeduplicableRuntimeCallIDs[] = {
1566 OMPRTL_omp_get_num_threads,
1567 OMPRTL_omp_in_parallel,
1568 OMPRTL_omp_get_cancellation,
1569 OMPRTL_omp_get_supported_active_levels,
1570 OMPRTL_omp_get_level,
1571 OMPRTL_omp_get_ancestor_thread_num,
1572 OMPRTL_omp_get_team_size,
1573 OMPRTL_omp_get_active_level,
1574 OMPRTL_omp_in_final,
1575 OMPRTL_omp_get_proc_bind,
1576 OMPRTL_omp_get_num_places,
1577 OMPRTL_omp_get_num_procs,
1578 OMPRTL_omp_get_place_num,
1579 OMPRTL_omp_get_partition_num_places,
1580 OMPRTL_omp_get_partition_place_nums};
1581
1582 // Global-tid is handled separately.
1583 SmallSetVector<Value *, 16> GTIdArgs;
1584 collectGlobalThreadIdArguments(GTIdArgs);
1585 LLVM_DEBUG(dbgs() << TAG << "Found " << GTIdArgs.size()
1586 << " global thread ID arguments\n");
1587
1588 for (Function *F : SCC) {
1589 for (auto DeduplicableRuntimeCallID : DeduplicableRuntimeCallIDs)
1590 Changed |= deduplicateRuntimeCalls(
1591 F&: *F, RFI&: OMPInfoCache.RFIs[DeduplicableRuntimeCallID]);
1592
1593 // __kmpc_global_thread_num is special as we can replace it with an
1594 // argument in enough cases to make it worth trying.
1595 Value *GTIdArg = nullptr;
1596 for (Argument &Arg : F->args())
1597 if (GTIdArgs.count(key: &Arg)) {
1598 GTIdArg = &Arg;
1599 break;
1600 }
1601 Changed |= deduplicateRuntimeCalls(
1602 F&: *F, RFI&: OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num], ReplVal: GTIdArg);
1603 }
1604
1605 return Changed;
1606 }
1607
1608 /// Tries to remove known runtime symbols that are optional from the module.
1609 bool removeRuntimeSymbols() {
1610 // The RPC client symbol is defined in `libc` and indicates that something
1611 // required an RPC server. If its users were all optimized out then we can
1612 // safely remove it.
1613 // TODO: This should be somewhere more common in the future.
1614 if (GlobalVariable *GV = M.getNamedGlobal(Name: "__llvm_rpc_client")) {
1615 if (GV->hasNUsesOrMore(N: 1))
1616 return false;
1617
1618 GV->replaceAllUsesWith(V: PoisonValue::get(T: GV->getType()));
1619 GV->eraseFromParent();
1620 return true;
1621 }
1622 return false;
1623 }
1624
1625 /// Tries to hide the latency of runtime calls that involve host to
1626 /// device memory transfers by splitting them into their "issue" and "wait"
1627 /// versions. The "issue" is moved upwards as much as possible. The "wait" is
1628 /// moved downards as much as possible. The "issue" issues the memory transfer
1629 /// asynchronously, returning a handle. The "wait" waits in the returned
1630 /// handle for the memory transfer to finish.
1631 bool hideMemTransfersLatency() {
1632 auto &RFI = OMPInfoCache.RFIs[OMPRTL___tgt_target_data_begin_mapper];
1633 bool Changed = false;
1634 auto SplitMemTransfers = [&](Use &U, Function &Decl) {
1635 auto *RTCall = getCallIfRegularCall(U, RFI: &RFI);
1636 if (!RTCall)
1637 return false;
1638
1639 OffloadArray OffloadArrays[3];
1640 if (!getValuesInOffloadArrays(RuntimeCall&: *RTCall, OAs: OffloadArrays))
1641 return false;
1642
1643 LLVM_DEBUG(dumpValuesInOffloadArrays(OffloadArrays));
1644
1645 // TODO: Check if can be moved upwards.
1646 bool WasSplit = false;
1647 Instruction *WaitMovementPoint = canBeMovedDownwards(RuntimeCall&: *RTCall);
1648 if (WaitMovementPoint)
1649 WasSplit = splitTargetDataBeginRTC(RuntimeCall&: *RTCall, WaitMovementPoint&: *WaitMovementPoint);
1650
1651 Changed |= WasSplit;
1652 return WasSplit;
1653 };
1654 if (OMPInfoCache.runtimeFnsAvailable(
1655 Fns: {OMPRTL___tgt_target_data_begin_mapper_issue,
1656 OMPRTL___tgt_target_data_begin_mapper_wait}))
1657 RFI.foreachUse(SCC, CB: SplitMemTransfers);
1658
1659 return Changed;
1660 }
1661
1662 void analysisGlobalization() {
1663 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
1664
1665 auto CheckGlobalization = [&](Use &U, Function &Decl) {
1666 if (CallInst *CI = getCallIfRegularCall(U, RFI: &RFI)) {
1667 auto Remark = [&](OptimizationRemarkMissed ORM) {
1668 return ORM
1669 << "Found thread data sharing on the GPU. "
1670 << "Expect degraded performance due to data globalization.";
1671 };
1672 emitRemark<OptimizationRemarkMissed>(I: CI, RemarkName: "OMP112", RemarkCB&: Remark);
1673 }
1674
1675 return false;
1676 };
1677
1678 RFI.foreachUse(SCC, CB: CheckGlobalization);
1679 }
1680
1681 /// Maps the values stored in the offload arrays passed as arguments to
1682 /// \p RuntimeCall into the offload arrays in \p OAs.
1683 bool getValuesInOffloadArrays(CallInst &RuntimeCall,
1684 MutableArrayRef<OffloadArray> OAs) {
1685 assert(OAs.size() == 3 && "Need space for three offload arrays!");
1686
1687 // A runtime call that involves memory offloading looks something like:
1688 // call void @__tgt_target_data_begin_mapper(arg0, arg1,
1689 // i8** %offload_baseptrs, i8** %offload_ptrs, i64* %offload_sizes,
1690 // ...)
1691 // So, the idea is to access the allocas that allocate space for these
1692 // offload arrays, offload_baseptrs, offload_ptrs, offload_sizes.
1693 // Therefore:
1694 // i8** %offload_baseptrs.
1695 Value *BasePtrsArg =
1696 RuntimeCall.getArgOperand(i: OffloadArray::BasePtrsArgNum);
1697 // i8** %offload_ptrs.
1698 Value *PtrsArg = RuntimeCall.getArgOperand(i: OffloadArray::PtrsArgNum);
1699 // i8** %offload_sizes.
1700 Value *SizesArg = RuntimeCall.getArgOperand(i: OffloadArray::SizesArgNum);
1701
1702 // Get values stored in **offload_baseptrs.
1703 auto *V = getUnderlyingObject(V: BasePtrsArg);
1704 if (!isa<AllocaInst>(Val: V))
1705 return false;
1706 auto *BasePtrsArray = cast<AllocaInst>(Val: V);
1707 if (!OAs[0].initialize(Array&: *BasePtrsArray, Before&: RuntimeCall))
1708 return false;
1709
1710 // Get values stored in **offload_baseptrs.
1711 V = getUnderlyingObject(V: PtrsArg);
1712 if (!isa<AllocaInst>(Val: V))
1713 return false;
1714 auto *PtrsArray = cast<AllocaInst>(Val: V);
1715 if (!OAs[1].initialize(Array&: *PtrsArray, Before&: RuntimeCall))
1716 return false;
1717
1718 // Get values stored in **offload_sizes.
1719 V = getUnderlyingObject(V: SizesArg);
1720 // If it's a [constant] global array don't analyze it.
1721 if (isa<GlobalValue>(Val: V))
1722 return isa<Constant>(Val: V);
1723 if (!isa<AllocaInst>(Val: V))
1724 return false;
1725
1726 auto *SizesArray = cast<AllocaInst>(Val: V);
1727 if (!OAs[2].initialize(Array&: *SizesArray, Before&: RuntimeCall))
1728 return false;
1729
1730 return true;
1731 }
1732
1733 /// Prints the values in the OffloadArrays \p OAs using LLVM_DEBUG.
1734 /// For now this is a way to test that the function getValuesInOffloadArrays
1735 /// is working properly.
1736 /// TODO: Move this to a unittest when unittests are available for OpenMPOpt.
1737 void dumpValuesInOffloadArrays(ArrayRef<OffloadArray> OAs) {
1738 assert(OAs.size() == 3 && "There are three offload arrays to debug!");
1739
1740 LLVM_DEBUG(dbgs() << TAG << " Successfully got offload values:\n");
1741 std::string ValuesStr;
1742 raw_string_ostream Printer(ValuesStr);
1743 std::string Separator = " --- ";
1744
1745 for (auto *BP : OAs[0].StoredValues) {
1746 BP->print(O&: Printer);
1747 Printer << Separator;
1748 }
1749 LLVM_DEBUG(dbgs() << "\t\toffload_baseptrs: " << ValuesStr << "\n");
1750 ValuesStr.clear();
1751
1752 for (auto *P : OAs[1].StoredValues) {
1753 P->print(O&: Printer);
1754 Printer << Separator;
1755 }
1756 LLVM_DEBUG(dbgs() << "\t\toffload_ptrs: " << ValuesStr << "\n");
1757 ValuesStr.clear();
1758
1759 for (auto *S : OAs[2].StoredValues) {
1760 S->print(O&: Printer);
1761 Printer << Separator;
1762 }
1763 LLVM_DEBUG(dbgs() << "\t\toffload_sizes: " << ValuesStr << "\n");
1764 }
1765
1766 /// Returns the instruction where the "wait" counterpart \p RuntimeCall can be
1767 /// moved. Returns nullptr if the movement is not possible, or not worth it.
1768 Instruction *canBeMovedDownwards(CallInst &RuntimeCall) {
1769 // FIXME: This traverses only the BasicBlock where RuntimeCall is.
1770 // Make it traverse the CFG.
1771
1772 Instruction *CurrentI = &RuntimeCall;
1773 bool IsWorthIt = false;
1774 while ((CurrentI = CurrentI->getNextNode())) {
1775
1776 // TODO: Once we detect the regions to be offloaded we should use the
1777 // alias analysis manager to check if CurrentI may modify one of
1778 // the offloaded regions.
1779 if (CurrentI->mayHaveSideEffects() || CurrentI->mayReadFromMemory()) {
1780 if (IsWorthIt)
1781 return CurrentI;
1782
1783 return nullptr;
1784 }
1785
1786 // FIXME: For now if we move it over anything without side effect
1787 // is worth it.
1788 IsWorthIt = true;
1789 }
1790
1791 // Return end of BasicBlock.
1792 return RuntimeCall.getParent()->getTerminator();
1793 }
1794
1795 /// Splits \p RuntimeCall into its "issue" and "wait" counterparts.
1796 bool splitTargetDataBeginRTC(CallInst &RuntimeCall,
1797 Instruction &WaitMovementPoint) {
1798 // Create stack allocated handle (__tgt_async_info) at the beginning of the
1799 // function. Used for storing information of the async transfer, allowing to
1800 // wait on it later.
1801 auto &IRBuilder = OMPInfoCache.OMPBuilder;
1802 Function *F = RuntimeCall.getCaller();
1803 BasicBlock &Entry = F->getEntryBlock();
1804 IRBuilder.Builder.SetInsertPoint(TheBB: &Entry,
1805 IP: Entry.getFirstNonPHIOrDbgOrAlloca());
1806 Value *Handle = IRBuilder.Builder.CreateAlloca(
1807 Ty: IRBuilder.AsyncInfo, /*ArraySize=*/nullptr, Name: "handle");
1808 Handle =
1809 IRBuilder.Builder.CreateAddrSpaceCast(V: Handle, DestTy: IRBuilder.AsyncInfoPtr);
1810
1811 // Add "issue" runtime call declaration:
1812 // declare %struct.tgt_async_info @__tgt_target_data_begin_issue(i64, i32,
1813 // i8**, i8**, i64*, i64*)
1814 FunctionCallee IssueDecl = IRBuilder.getOrCreateRuntimeFunction(
1815 M, FnID: OMPRTL___tgt_target_data_begin_mapper_issue);
1816
1817 // Change RuntimeCall call site for its asynchronous version.
1818 SmallVector<Value *, 16> Args;
1819 for (auto &Arg : RuntimeCall.args())
1820 Args.push_back(Elt: Arg.get());
1821 Args.push_back(Elt: Handle);
1822
1823 CallInst *IssueCallsite = CallInst::Create(Func: IssueDecl, Args, /*NameStr=*/"",
1824 InsertBefore: RuntimeCall.getIterator());
1825 OMPInfoCache.setCallingConvention(Callee: IssueDecl, CI: IssueCallsite);
1826 RuntimeCall.eraseFromParent();
1827
1828 // Add "wait" runtime call declaration:
1829 // declare void @__tgt_target_data_begin_wait(i64, %struct.__tgt_async_info)
1830 FunctionCallee WaitDecl = IRBuilder.getOrCreateRuntimeFunction(
1831 M, FnID: OMPRTL___tgt_target_data_begin_mapper_wait);
1832
1833 Value *WaitParams[2] = {
1834 IssueCallsite->getArgOperand(
1835 i: OffloadArray::DeviceIDArgNum), // device_id.
1836 Handle // handle to wait on.
1837 };
1838 CallInst *WaitCallsite = CallInst::Create(
1839 Func: WaitDecl, Args: WaitParams, /*NameStr=*/"", InsertBefore: WaitMovementPoint.getIterator());
1840 OMPInfoCache.setCallingConvention(Callee: WaitDecl, CI: WaitCallsite);
1841
1842 return true;
1843 }
1844
1845 static Value *combinedIdentStruct(Value *CurrentIdent, Value *NextIdent,
1846 bool GlobalOnly, bool &SingleChoice) {
1847 if (CurrentIdent == NextIdent)
1848 return CurrentIdent;
1849
1850 // TODO: Figure out how to actually combine multiple debug locations. For
1851 // now we just keep an existing one if there is a single choice.
1852 if (!GlobalOnly || isa<GlobalValue>(Val: NextIdent)) {
1853 SingleChoice = !CurrentIdent;
1854 return NextIdent;
1855 }
1856 return nullptr;
1857 }
1858
1859 /// Return an `struct ident_t*` value that represents the ones used in the
1860 /// calls of \p RFI inside of \p F. If \p GlobalOnly is true, we will not
1861 /// return a local `struct ident_t*`. For now, if we cannot find a suitable
1862 /// return value we create one from scratch. We also do not yet combine
1863 /// information, e.g., the source locations, see combinedIdentStruct.
1864 Value *
1865 getCombinedIdentFromCallUsesIn(OMPInformationCache::RuntimeFunctionInfo &RFI,
1866 Function &F, bool GlobalOnly) {
1867 bool SingleChoice = true;
1868 Value *Ident = nullptr;
1869 auto CombineIdentStruct = [&](Use &U, Function &Caller) {
1870 CallInst *CI = getCallIfRegularCall(U, RFI: &RFI);
1871 if (!CI || &F != &Caller)
1872 return false;
1873 Ident = combinedIdentStruct(CurrentIdent: Ident, NextIdent: CI->getArgOperand(i: 0),
1874 /* GlobalOnly */ true, SingleChoice);
1875 return false;
1876 };
1877 RFI.foreachUse(SCC, CB: CombineIdentStruct);
1878
1879 if (!Ident || !SingleChoice) {
1880 // The IRBuilder uses the insertion block to get to the module, this is
1881 // unfortunate but we work around it for now. No instruction is emitted
1882 // here, so there is no debug location to preserve.
1883 if (!OMPInfoCache.OMPBuilder.getInsertionPoint().getBlock())
1884 OMPInfoCache.OMPBuilder.updateToLocation(
1885 Loc: {OpenMPIRBuilder::InsertPointTy(&F.getEntryBlock(),
1886 F.getEntryBlock().begin()),
1887 DebugLoc()});
1888 // Create a fallback location if non was found.
1889 // TODO: Use the debug locations of the calls instead.
1890 uint32_t SrcLocStrSize;
1891 Constant *Loc =
1892 OMPInfoCache.OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1893 Ident = OMPInfoCache.OMPBuilder.getOrCreateIdent(SrcLocStr: Loc, SrcLocStrSize);
1894 }
1895 return Ident;
1896 }
1897
1898 /// Try to eliminate calls of \p RFI in \p F by reusing an existing one or
1899 /// \p ReplVal if given.
1900 bool deduplicateRuntimeCalls(Function &F,
1901 OMPInformationCache::RuntimeFunctionInfo &RFI,
1902 Value *ReplVal = nullptr) {
1903 auto *UV = RFI.getUseVector(F);
1904 if (!UV || UV->size() + (ReplVal != nullptr) < 2)
1905 return false;
1906
1907 LLVM_DEBUG(
1908 dbgs() << TAG << "Deduplicate " << UV->size() << " uses of " << RFI.Name
1909 << (ReplVal ? " with an existing value\n" : "\n") << "\n");
1910
1911 assert((!ReplVal || (isa<Argument>(ReplVal) &&
1912 cast<Argument>(ReplVal)->getParent() == &F)) &&
1913 "Unexpected replacement value!");
1914
1915 // TODO: Use dominance to find a good position instead.
1916 auto CanBeMoved = [this](CallBase &CB) {
1917 unsigned NumArgs = CB.arg_size();
1918 if (NumArgs == 0)
1919 return true;
1920 if (CB.getArgOperand(i: 0)->getType() != OMPInfoCache.OMPBuilder.IdentPtr)
1921 return false;
1922 for (unsigned U = 1; U < NumArgs; ++U)
1923 if (isa<Instruction>(Val: CB.getArgOperand(i: U)))
1924 return false;
1925 return true;
1926 };
1927
1928 if (!ReplVal) {
1929 auto *DT =
1930 OMPInfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F);
1931 if (!DT)
1932 return false;
1933 Instruction *IP = nullptr;
1934 for (Use *U : *UV) {
1935 if (CallInst *CI = getCallIfRegularCall(U&: *U, RFI: &RFI)) {
1936 if (IP)
1937 IP = DT->findNearestCommonDominator(I1: IP, I2: CI);
1938 else
1939 IP = CI;
1940 if (!CanBeMoved(*CI))
1941 continue;
1942 if (!ReplVal)
1943 ReplVal = CI;
1944 }
1945 }
1946 if (!ReplVal)
1947 return false;
1948 assert(IP && "Expected insertion point!");
1949 cast<Instruction>(Val: ReplVal)->moveBefore(InsertPos: IP->getIterator());
1950 }
1951
1952 // If we use a call as a replacement value we need to make sure the ident is
1953 // valid at the new location. For now we just pick a global one, either
1954 // existing and used by one of the calls, or created from scratch.
1955 if (CallBase *CI = dyn_cast<CallBase>(Val: ReplVal)) {
1956 if (!CI->arg_empty() &&
1957 CI->getArgOperand(i: 0)->getType() == OMPInfoCache.OMPBuilder.IdentPtr) {
1958 Value *Ident = getCombinedIdentFromCallUsesIn(RFI, F,
1959 /* GlobalOnly */ true);
1960 CI->setArgOperand(i: 0, v: Ident);
1961 }
1962 }
1963
1964 bool Changed = false;
1965 auto ReplaceAndDeleteCB = [&](Use &U, Function &Caller) {
1966 CallInst *CI = getCallIfRegularCall(U, RFI: &RFI);
1967 if (!CI || CI == ReplVal || &F != &Caller)
1968 return false;
1969 assert(CI->getCaller() == &F && "Unexpected call!");
1970
1971 auto Remark = [&](OptimizationRemark OR) {
1972 return OR << "OpenMP runtime call "
1973 << ore::NV("OpenMPOptRuntime", RFI.Name) << " deduplicated.";
1974 };
1975 if (CI->getDebugLoc())
1976 emitRemark<OptimizationRemark>(I: CI, RemarkName: "OMP170", RemarkCB&: Remark);
1977 else
1978 emitRemark<OptimizationRemark>(F: &F, RemarkName: "OMP170", RemarkCB&: Remark);
1979
1980 CI->replaceAllUsesWith(V: ReplVal);
1981 CI->eraseFromParent();
1982 ++NumOpenMPRuntimeCallsDeduplicated;
1983 Changed = true;
1984 return true;
1985 };
1986 RFI.foreachUse(SCC, CB: ReplaceAndDeleteCB);
1987
1988 return Changed;
1989 }
1990
1991 /// Collect arguments that represent the global thread id in \p GTIdArgs.
1992 void collectGlobalThreadIdArguments(SmallSetVector<Value *, 16> &GTIdArgs) {
1993 // TODO: Below we basically perform a fixpoint iteration with a pessimistic
1994 // initialization. We could define an AbstractAttribute instead and
1995 // run the Attributor here once it can be run as an SCC pass.
1996
1997 // Helper to check the argument \p ArgNo at all call sites of \p F for
1998 // a GTId.
1999 auto CallArgOpIsGTId = [&](Function &F, unsigned ArgNo, CallInst &RefCI) {
2000 if (!F.hasLocalLinkage())
2001 return false;
2002 for (Use &U : F.uses()) {
2003 if (CallInst *CI = getCallIfRegularCall(U)) {
2004 Value *ArgOp = CI->getArgOperand(i: ArgNo);
2005 if (CI == &RefCI || GTIdArgs.count(key: ArgOp) ||
2006 getCallIfRegularCall(
2007 V&: *ArgOp, RFI: &OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num]))
2008 continue;
2009 }
2010 return false;
2011 }
2012 return true;
2013 };
2014
2015 // Helper to identify uses of a GTId as GTId arguments.
2016 auto AddUserArgs = [&](Value &GTId) {
2017 for (Use &U : GTId.uses())
2018 if (CallInst *CI = dyn_cast<CallInst>(Val: U.getUser()))
2019 if (CI->isArgOperand(U: &U))
2020 if (Function *Callee = CI->getCalledFunction())
2021 if (CallArgOpIsGTId(*Callee, U.getOperandNo(), *CI))
2022 GTIdArgs.insert(X: Callee->getArg(i: U.getOperandNo()));
2023 };
2024
2025 // The argument users of __kmpc_global_thread_num calls are GTIds.
2026 OMPInformationCache::RuntimeFunctionInfo &GlobThreadNumRFI =
2027 OMPInfoCache.RFIs[OMPRTL___kmpc_global_thread_num];
2028
2029 GlobThreadNumRFI.foreachUse(SCC, CB: [&](Use &U, Function &F) {
2030 if (CallInst *CI = getCallIfRegularCall(U, RFI: &GlobThreadNumRFI))
2031 AddUserArgs(*CI);
2032 return false;
2033 });
2034
2035 // Transitively search for more arguments by looking at the users of the
2036 // ones we know already. During the search the GTIdArgs vector is extended
2037 // so we cannot cache the size nor can we use a range based for.
2038 for (unsigned U = 0; U < GTIdArgs.size(); ++U)
2039 AddUserArgs(*GTIdArgs[U]);
2040 }
2041
2042 /// Kernel (=GPU) optimizations and utility functions
2043 ///
2044 ///{{
2045
2046 /// Cache to remember the unique kernel for a function.
2047 DenseMap<Function *, std::optional<Kernel>> UniqueKernelMap;
2048
2049 /// Find the unique kernel that will execute \p F, if any.
2050 Kernel getUniqueKernelFor(Function &F);
2051
2052 /// Find the unique kernel that will execute \p I, if any.
2053 Kernel getUniqueKernelFor(Instruction &I) {
2054 return getUniqueKernelFor(F&: *I.getFunction());
2055 }
2056
2057 /// Rewrite the device (=GPU) code state machine create in non-SPMD mode in
2058 /// the cases we can avoid taking the address of a function.
2059 bool rewriteDeviceCodeStateMachine();
2060
2061 /// In SPMD kernels the parallel data-sharing wrapper passed to
2062 /// __kmpc_parallel_60 is never used by the runtime; null it out so the dead
2063 /// wrapper (and any LDS it references) can be removed.
2064 bool removeSPMDParallelWrappers();
2065
2066 ///
2067 ///}}
2068
2069 /// Emit a remark generically
2070 ///
2071 /// This template function can be used to generically emit a remark. The
2072 /// RemarkKind should be one of the following:
2073 /// - OptimizationRemark to indicate a successful optimization attempt
2074 /// - OptimizationRemarkMissed to report a failed optimization attempt
2075 /// - OptimizationRemarkAnalysis to provide additional information about an
2076 /// optimization attempt
2077 ///
2078 /// The remark is built using a callback function provided by the caller that
2079 /// takes a RemarkKind as input and returns a RemarkKind.
2080 template <typename RemarkKind, typename RemarkCallBack>
2081 void emitRemark(Instruction *I, StringRef RemarkName,
2082 RemarkCallBack &&RemarkCB) const {
2083 Function *F = I->getParent()->getParent();
2084 auto &ORE = OREGetter(F);
2085
2086 if (RemarkName.starts_with(Prefix: "OMP"))
2087 ORE.emit([&]() {
2088 return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, I))
2089 << " [" << RemarkName << "]";
2090 });
2091 else
2092 ORE.emit(
2093 [&]() { return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, I)); });
2094 }
2095
2096 /// Emit a remark on a function.
2097 template <typename RemarkKind, typename RemarkCallBack>
2098 void emitRemark(Function *F, StringRef RemarkName,
2099 RemarkCallBack &&RemarkCB) const {
2100 auto &ORE = OREGetter(F);
2101
2102 if (RemarkName.starts_with(Prefix: "OMP"))
2103 ORE.emit([&]() {
2104 return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, F))
2105 << " [" << RemarkName << "]";
2106 });
2107 else
2108 ORE.emit(
2109 [&]() { return RemarkCB(RemarkKind(DEBUG_TYPE, RemarkName, F)); });
2110 }
2111
2112 /// The underlying module.
2113 Module &M;
2114
2115 /// The SCC we are operating on.
2116 SmallVectorImpl<Function *> &SCC;
2117
2118 /// Callback to update the call graph, the first argument is a removed call,
2119 /// the second an optional replacement call.
2120 CallGraphUpdater &CGUpdater;
2121
2122 /// Callback to get an OptimizationRemarkEmitter from a Function *
2123 OptimizationRemarkGetter OREGetter;
2124
2125 /// OpenMP-specific information cache. Also Used for Attributor runs.
2126 OMPInformationCache &OMPInfoCache;
2127
2128 /// Attributor instance.
2129 Attributor &A;
2130
2131 /// Helper function to run Attributor on SCC.
2132 bool runAttributor(bool IsModulePass) {
2133 if (SCC.empty())
2134 return false;
2135
2136 registerAAs(IsModulePass);
2137
2138 ChangeStatus Changed = A.run();
2139
2140 LLVM_DEBUG(dbgs() << "[Attributor] Done with " << SCC.size()
2141 << " functions, result: " << Changed << ".\n");
2142
2143 if (Changed == ChangeStatus::CHANGED)
2144 OMPInfoCache.invalidateAnalyses();
2145
2146 return Changed == ChangeStatus::CHANGED;
2147 }
2148
2149 void registerFoldRuntimeCall(RuntimeFunction RF);
2150
2151 /// Populate the Attributor with abstract attribute opportunities in the
2152 /// functions.
2153 void registerAAs(bool IsModulePass);
2154
2155public:
2156 /// Callback to register AAs for live functions, including internal functions
2157 /// marked live during the traversal.
2158 static void registerAAsForFunction(Attributor &A, const Function &F);
2159};
2160
2161Kernel OpenMPOpt::getUniqueKernelFor(Function &F) {
2162 if (OMPInfoCache.CGSCC && !OMPInfoCache.CGSCC->empty() &&
2163 !OMPInfoCache.CGSCC->contains(key: &F))
2164 return nullptr;
2165
2166 // Use a scope to keep the lifetime of the CachedKernel short.
2167 {
2168 std::optional<Kernel> &CachedKernel = UniqueKernelMap[&F];
2169 if (CachedKernel)
2170 return *CachedKernel;
2171
2172 // TODO: We should use an AA to create an (optimistic and callback
2173 // call-aware) call graph. For now we stick to simple patterns that
2174 // are less powerful, basically the worst fixpoint.
2175 if (isOpenMPKernel(Fn&: F)) {
2176 CachedKernel = Kernel(&F);
2177 return *CachedKernel;
2178 }
2179
2180 CachedKernel = nullptr;
2181 if (!F.hasLocalLinkage()) {
2182
2183 // See https://openmp.llvm.org/remarks/OptimizationRemarks.html
2184 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
2185 return ORA << "Potentially unknown OpenMP target region caller.";
2186 };
2187 emitRemark<OptimizationRemarkAnalysis>(F: &F, RemarkName: "OMP100", RemarkCB&: Remark);
2188
2189 return nullptr;
2190 }
2191 }
2192
2193 auto GetUniqueKernelForUse = [&](const Use &U) -> Kernel {
2194 if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
2195 // Allow use in equality comparisons.
2196 if (Cmp->isEquality())
2197 return getUniqueKernelFor(I&: *Cmp);
2198 return nullptr;
2199 }
2200 if (auto *CB = dyn_cast<CallBase>(Val: U.getUser())) {
2201 // Allow direct calls.
2202 if (CB->isCallee(U: &U))
2203 return getUniqueKernelFor(I&: *CB);
2204
2205 OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
2206 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
2207 // Allow the use in __kmpc_parallel_60 calls.
2208 if (OpenMPOpt::getCallIfRegularCall(V&: *U.getUser(), RFI: &KernelParallelRFI))
2209 return getUniqueKernelFor(I&: *CB);
2210 return nullptr;
2211 }
2212 // Disallow every other use.
2213 return nullptr;
2214 };
2215
2216 // TODO: In the future we want to track more than just a unique kernel.
2217 SmallPtrSet<Kernel, 2> PotentialKernels;
2218 OMPInformationCache::foreachUse(F, CB: [&](const Use &U) {
2219 PotentialKernels.insert(Ptr: GetUniqueKernelForUse(U));
2220 });
2221
2222 Kernel K = nullptr;
2223 if (PotentialKernels.size() == 1)
2224 K = *PotentialKernels.begin();
2225
2226 // Cache the result.
2227 UniqueKernelMap[&F] = K;
2228
2229 return K;
2230}
2231
2232bool OpenMPOpt::rewriteDeviceCodeStateMachine() {
2233 OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
2234 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
2235
2236 bool Changed = false;
2237 if (!KernelParallelRFI)
2238 return Changed;
2239
2240 // If we have disabled state machine changes, exit
2241 if (DisableOpenMPOptStateMachineRewrite)
2242 return Changed;
2243
2244 for (Function *F : SCC) {
2245
2246 // Check if the function is a use in a __kmpc_parallel_60 call at
2247 // all.
2248 bool UnknownUse = false;
2249 bool KernelParallelUse = false;
2250 unsigned NumDirectCalls = 0;
2251
2252 SmallVector<Use *, 2> ToBeReplacedStateMachineUses;
2253 OMPInformationCache::foreachUse(F&: *F, CB: [&](Use &U) {
2254 if (auto *CB = dyn_cast<CallBase>(Val: U.getUser()))
2255 if (CB->isCallee(U: &U)) {
2256 ++NumDirectCalls;
2257 return;
2258 }
2259
2260 if (isa<ICmpInst>(Val: U.getUser())) {
2261 ToBeReplacedStateMachineUses.push_back(Elt: &U);
2262 return;
2263 }
2264
2265 // Find wrapper functions that represent parallel kernels.
2266 CallInst *CI =
2267 OpenMPOpt::getCallIfRegularCall(V&: *U.getUser(), RFI: &KernelParallelRFI);
2268 const unsigned int WrapperFunctionArgNo = 6;
2269 if (!KernelParallelUse && CI &&
2270 CI->getArgOperandNo(U: &U) == WrapperFunctionArgNo) {
2271 KernelParallelUse = true;
2272 ToBeReplacedStateMachineUses.push_back(Elt: &U);
2273 return;
2274 }
2275 UnknownUse = true;
2276 });
2277
2278 // Do not emit a remark if we haven't seen a __kmpc_parallel_60
2279 // use.
2280 if (!KernelParallelUse)
2281 continue;
2282
2283 // If this ever hits, we should investigate.
2284 // TODO: Checking the number of uses is not a necessary restriction and
2285 // should be lifted.
2286 if (UnknownUse || NumDirectCalls != 1 ||
2287 ToBeReplacedStateMachineUses.size() > 2) {
2288 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
2289 return ORA << "Parallel region is used in "
2290 << (UnknownUse ? "unknown" : "unexpected")
2291 << " ways. Will not attempt to rewrite the state machine.";
2292 };
2293 emitRemark<OptimizationRemarkAnalysis>(F, RemarkName: "OMP101", RemarkCB&: Remark);
2294 continue;
2295 }
2296
2297 // Even if we have __kmpc_parallel_60 calls, we (for now) give
2298 // up if the function is not called from a unique kernel.
2299 Kernel K = getUniqueKernelFor(F&: *F);
2300 if (!K) {
2301 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
2302 return ORA << "Parallel region is not called from a unique kernel. "
2303 "Will not attempt to rewrite the state machine.";
2304 };
2305 emitRemark<OptimizationRemarkAnalysis>(F, RemarkName: "OMP102", RemarkCB&: Remark);
2306 continue;
2307 }
2308
2309 // We now know F is a parallel body function called only from the kernel K.
2310 // We also identified the state machine uses in which we replace the
2311 // function pointer by a new global symbol for identification purposes. This
2312 // ensures only direct calls to the function are left.
2313
2314 Module &M = *F->getParent();
2315 Type *Int8Ty = Type::getInt8Ty(C&: M.getContext());
2316
2317 auto *ID = new GlobalVariable(
2318 M, Int8Ty, /* isConstant */ true, GlobalValue::PrivateLinkage,
2319 UndefValue::get(T: Int8Ty), F->getName() + ".ID");
2320
2321 for (Use *U : ToBeReplacedStateMachineUses)
2322 U->set(ConstantExpr::getPointerBitCastOrAddrSpaceCast(
2323 C: ID, Ty: U->get()->getType()));
2324
2325 ++NumOpenMPParallelRegionsReplacedInGPUStateMachine;
2326
2327 Changed = true;
2328 }
2329
2330 return Changed;
2331}
2332
2333bool OpenMPOpt::removeSPMDParallelWrappers() {
2334 // Nothing to clean up unless we SPMD-ized at least one kernel.
2335 if (OMPInfoCache.SPMDizedKernels.empty())
2336 return false;
2337
2338 OMPInformationCache::RuntimeFunctionInfo &KernelParallelRFI =
2339 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
2340 if (!KernelParallelRFI || !KernelParallelRFI.Declaration)
2341 return false;
2342
2343 constexpr unsigned WrapperFunctionArgNo = 6;
2344 bool Changed = false;
2345 for (User *U : KernelParallelRFI.Declaration->users()) {
2346 auto *CI = dyn_cast<CallInst>(Val: U);
2347 if (!CI || CI->getCalledOperand() != KernelParallelRFI.Declaration ||
2348 CI->arg_size() <= WrapperFunctionArgNo)
2349 continue;
2350
2351 Value *Wrapper = CI->getArgOperand(i: WrapperFunctionArgNo);
2352 if (isa<ConstantPointerNull>(Val: Wrapper))
2353 continue;
2354
2355 // Only drop the wrapper for a parallel region reached from a single kernel
2356 // that we transformed to SPMD mode. A region also reachable from a
2357 // generic-mode kernel still needs its wrapper for that kernel's state
2358 // machine, and getUniqueKernelFor conservatively bails on such shared
2359 // regions. (Mirrors the unique-kernel requirement in
2360 // rewriteDeviceCodeStateMachine.)
2361 Kernel K = getUniqueKernelFor(F&: *CI->getFunction());
2362 if (!K || !OMPInfoCache.SPMDizedKernels.contains(Ptr: K))
2363 continue;
2364
2365 CI->setArgOperand(
2366 i: WrapperFunctionArgNo,
2367 v: ConstantPointerNull::get(T: cast<PointerType>(Val: Wrapper->getType())));
2368 Changed = true;
2369 }
2370
2371 return Changed;
2372}
2373
2374/// Abstract Attribute for tracking ICV values.
2375struct AAICVTracker : public StateWrapper<BooleanState, AbstractAttribute> {
2376 using Base = StateWrapper<BooleanState, AbstractAttribute>;
2377 AAICVTracker(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
2378
2379 /// Returns true if value is assumed to be tracked.
2380 bool isAssumedTracked() const { return getAssumed(); }
2381
2382 /// Returns true if value is known to be tracked.
2383 bool isKnownTracked() const { return getAssumed(); }
2384
2385 /// Create an abstract attribute biew for the position \p IRP.
2386 static AAICVTracker &createForPosition(const IRPosition &IRP, Attributor &A);
2387
2388 /// Return the value with which \p I can be replaced for specific \p ICV.
2389 virtual std::optional<Value *> getReplacementValue(InternalControlVar ICV,
2390 const Instruction *I,
2391 Attributor &A) const {
2392 return std::nullopt;
2393 }
2394
2395 /// Return an assumed unique ICV value if a single candidate is found. If
2396 /// there cannot be one, return a nullptr. If it is not clear yet, return
2397 /// std::nullopt.
2398 virtual std::optional<Value *>
2399 getUniqueReplacementValue(InternalControlVar ICV) const = 0;
2400
2401 // Currently only nthreads is being tracked.
2402 // this array will only grow with time.
2403 InternalControlVar TrackableICVs[1] = {ICV_nthreads};
2404
2405 /// See AbstractAttribute::getName()
2406 StringRef getName() const override { return "AAICVTracker"; }
2407
2408 /// See AbstractAttribute::getIdAddr()
2409 const char *getIdAddr() const override { return &ID; }
2410
2411 /// This function should return true if the type of the \p AA is AAICVTracker
2412 static bool classof(const AbstractAttribute *AA) {
2413 return (AA->getIdAddr() == &ID);
2414 }
2415
2416 static const char ID;
2417};
2418
2419struct AAICVTrackerFunction : public AAICVTracker {
2420 AAICVTrackerFunction(const IRPosition &IRP, Attributor &A)
2421 : AAICVTracker(IRP, A) {}
2422
2423 // FIXME: come up with better string.
2424 const std::string getAsStr(Attributor *) const override {
2425 return "ICVTrackerFunction";
2426 }
2427
2428 // FIXME: come up with some stats.
2429 void trackStatistics() const override {}
2430
2431 /// We don't manifest anything for this AA.
2432 ChangeStatus manifest(Attributor &A) override {
2433 return ChangeStatus::UNCHANGED;
2434 }
2435
2436 // Map of ICV to their values at specific program point.
2437 EnumeratedArray<DenseMap<Instruction *, Value *>, InternalControlVar,
2438 InternalControlVar::ICV___last>
2439 ICVReplacementValuesMap;
2440
2441 ChangeStatus updateImpl(Attributor &A) override {
2442 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
2443
2444 Function *F = getAnchorScope();
2445
2446 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2447
2448 for (InternalControlVar ICV : TrackableICVs) {
2449 auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
2450
2451 auto &ValuesMap = ICVReplacementValuesMap[ICV];
2452 auto TrackValues = [&](Use &U, Function &) {
2453 CallInst *CI = OpenMPOpt::getCallIfRegularCall(U);
2454 if (!CI)
2455 return false;
2456
2457 // FIXME: handle setters with more that 1 arguments.
2458 /// Track new value.
2459 if (ValuesMap.insert(KV: std::make_pair(x&: CI, y: CI->getArgOperand(i: 0))).second)
2460 HasChanged = ChangeStatus::CHANGED;
2461
2462 return false;
2463 };
2464
2465 auto CallCheck = [&](Instruction &I) {
2466 std::optional<Value *> ReplVal = getValueForCall(A, I, ICV);
2467 if (ReplVal && ValuesMap.insert(KV: std::make_pair(x: &I, y&: *ReplVal)).second)
2468 HasChanged = ChangeStatus::CHANGED;
2469
2470 return true;
2471 };
2472
2473 // Track all changes of an ICV.
2474 SetterRFI.foreachUse(CB: TrackValues, F);
2475
2476 bool UsedAssumedInformation = false;
2477 A.checkForAllInstructions(Pred: CallCheck, QueryingAA: *this, Opcodes: {Instruction::Call},
2478 UsedAssumedInformation,
2479 /* CheckBBLivenessOnly */ true);
2480
2481 /// TODO: Figure out a way to avoid adding entry in
2482 /// ICVReplacementValuesMap
2483 Instruction *Entry = &F->getEntryBlock().front();
2484 if (HasChanged == ChangeStatus::CHANGED)
2485 ValuesMap.try_emplace(Key: Entry);
2486 }
2487
2488 return HasChanged;
2489 }
2490
2491 /// Helper to check if \p I is a call and get the value for it if it is
2492 /// unique.
2493 std::optional<Value *> getValueForCall(Attributor &A, const Instruction &I,
2494 InternalControlVar &ICV) const {
2495
2496 const auto *CB = dyn_cast<CallBase>(Val: &I);
2497 if (!CB || CB->hasFnAttr(Kind: "no_openmp") ||
2498 CB->hasFnAttr(Kind: "no_openmp_routines") ||
2499 CB->hasFnAttr(Kind: "no_openmp_constructs"))
2500 return std::nullopt;
2501
2502 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2503 auto &GetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Getter];
2504 auto &SetterRFI = OMPInfoCache.RFIs[OMPInfoCache.ICVs[ICV].Setter];
2505 Function *CalledFunction = CB->getCalledFunction();
2506
2507 // Indirect call, assume ICV changes.
2508 if (CalledFunction == nullptr)
2509 return nullptr;
2510 if (CalledFunction == GetterRFI.Declaration)
2511 return std::nullopt;
2512 if (CalledFunction == SetterRFI.Declaration) {
2513 if (ICVReplacementValuesMap[ICV].count(Val: &I))
2514 return ICVReplacementValuesMap[ICV].lookup(Val: &I);
2515
2516 return nullptr;
2517 }
2518
2519 // Since we don't know, assume it changes the ICV.
2520 if (CalledFunction->isDeclaration())
2521 return nullptr;
2522
2523 const auto *ICVTrackingAA = A.getAAFor<AAICVTracker>(
2524 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::REQUIRED);
2525
2526 if (ICVTrackingAA->isAssumedTracked()) {
2527 std::optional<Value *> URV =
2528 ICVTrackingAA->getUniqueReplacementValue(ICV);
2529 if (!URV || (*URV && AA::isValidAtPosition(VAC: AA::ValueAndContext(**URV, I),
2530 InfoCache&: OMPInfoCache)))
2531 return URV;
2532 }
2533
2534 // If we don't know, assume it changes.
2535 return nullptr;
2536 }
2537
2538 // We don't check unique value for a function, so return std::nullopt.
2539 std::optional<Value *>
2540 getUniqueReplacementValue(InternalControlVar ICV) const override {
2541 return std::nullopt;
2542 }
2543
2544 /// Return the value with which \p I can be replaced for specific \p ICV.
2545 std::optional<Value *> getReplacementValue(InternalControlVar ICV,
2546 const Instruction *I,
2547 Attributor &A) const override {
2548 const auto &ValuesMap = ICVReplacementValuesMap[ICV];
2549 if (ValuesMap.count(Val: I))
2550 return ValuesMap.lookup(Val: I);
2551
2552 SmallVector<const Instruction *, 16> Worklist;
2553 SmallPtrSet<const Instruction *, 16> Visited;
2554 Worklist.push_back(Elt: I);
2555
2556 std::optional<Value *> ReplVal;
2557
2558 while (!Worklist.empty()) {
2559 const Instruction *CurrInst = Worklist.pop_back_val();
2560 if (!Visited.insert(Ptr: CurrInst).second)
2561 continue;
2562
2563 const BasicBlock *CurrBB = CurrInst->getParent();
2564
2565 // Go up and look for all potential setters/calls that might change the
2566 // ICV.
2567 while ((CurrInst = CurrInst->getPrevNode())) {
2568 if (ValuesMap.count(Val: CurrInst)) {
2569 std::optional<Value *> NewReplVal = ValuesMap.lookup(Val: CurrInst);
2570 // Unknown value, track new.
2571 if (!ReplVal) {
2572 ReplVal = NewReplVal;
2573 break;
2574 }
2575
2576 // If we found a new value, we can't know the icv value anymore.
2577 if (NewReplVal)
2578 if (ReplVal != NewReplVal)
2579 return nullptr;
2580
2581 break;
2582 }
2583
2584 std::optional<Value *> NewReplVal = getValueForCall(A, I: *CurrInst, ICV);
2585 if (!NewReplVal)
2586 continue;
2587
2588 // Unknown value, track new.
2589 if (!ReplVal) {
2590 ReplVal = NewReplVal;
2591 break;
2592 }
2593
2594 // if (NewReplVal.hasValue())
2595 // We found a new value, we can't know the icv value anymore.
2596 if (ReplVal != NewReplVal)
2597 return nullptr;
2598 }
2599
2600 // If we are in the same BB and we have a value, we are done.
2601 if (CurrBB == I->getParent() && ReplVal)
2602 return ReplVal;
2603
2604 // Go through all predecessors and add terminators for analysis.
2605 for (const BasicBlock *Pred : predecessors(BB: CurrBB))
2606 if (const Instruction *Terminator = Pred->getTerminator())
2607 Worklist.push_back(Elt: Terminator);
2608 }
2609
2610 return ReplVal;
2611 }
2612};
2613
2614struct AAICVTrackerFunctionReturned : AAICVTracker {
2615 AAICVTrackerFunctionReturned(const IRPosition &IRP, Attributor &A)
2616 : AAICVTracker(IRP, A) {}
2617
2618 // FIXME: come up with better string.
2619 const std::string getAsStr(Attributor *) const override {
2620 return "ICVTrackerFunctionReturned";
2621 }
2622
2623 // FIXME: come up with some stats.
2624 void trackStatistics() const override {}
2625
2626 /// We don't manifest anything for this AA.
2627 ChangeStatus manifest(Attributor &A) override {
2628 return ChangeStatus::UNCHANGED;
2629 }
2630
2631 // Map of ICV to their values at specific program point.
2632 EnumeratedArray<std::optional<Value *>, InternalControlVar,
2633 InternalControlVar::ICV___last>
2634 ICVReplacementValuesMap;
2635
2636 /// Return the value with which \p I can be replaced for specific \p ICV.
2637 std::optional<Value *>
2638 getUniqueReplacementValue(InternalControlVar ICV) const override {
2639 return ICVReplacementValuesMap[ICV];
2640 }
2641
2642 ChangeStatus updateImpl(Attributor &A) override {
2643 ChangeStatus Changed = ChangeStatus::UNCHANGED;
2644 const auto *ICVTrackingAA = A.getAAFor<AAICVTracker>(
2645 QueryingAA: *this, IRP: IRPosition::function(F: *getAnchorScope()), DepClass: DepClassTy::REQUIRED);
2646
2647 if (!ICVTrackingAA->isAssumedTracked())
2648 return indicatePessimisticFixpoint();
2649
2650 for (InternalControlVar ICV : TrackableICVs) {
2651 std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2652 std::optional<Value *> UniqueICVValue;
2653
2654 auto CheckReturnInst = [&](Instruction &I) {
2655 std::optional<Value *> NewReplVal =
2656 ICVTrackingAA->getReplacementValue(ICV, I: &I, A);
2657
2658 // If we found a second ICV value there is no unique returned value.
2659 if (UniqueICVValue && UniqueICVValue != NewReplVal)
2660 return false;
2661
2662 UniqueICVValue = NewReplVal;
2663
2664 return true;
2665 };
2666
2667 bool UsedAssumedInformation = false;
2668 if (!A.checkForAllInstructions(Pred: CheckReturnInst, QueryingAA: *this, Opcodes: {Instruction::Ret},
2669 UsedAssumedInformation,
2670 /* CheckBBLivenessOnly */ true))
2671 UniqueICVValue = nullptr;
2672
2673 if (UniqueICVValue == ReplVal)
2674 continue;
2675
2676 ReplVal = UniqueICVValue;
2677 Changed = ChangeStatus::CHANGED;
2678 }
2679
2680 return Changed;
2681 }
2682};
2683
2684struct AAICVTrackerCallSite : AAICVTracker {
2685 AAICVTrackerCallSite(const IRPosition &IRP, Attributor &A)
2686 : AAICVTracker(IRP, A) {}
2687
2688 void initialize(Attributor &A) override {
2689 assert(getAnchorScope() && "Expected anchor function");
2690
2691 // We only initialize this AA for getters, so we need to know which ICV it
2692 // gets.
2693 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
2694 for (InternalControlVar ICV : TrackableICVs) {
2695 auto ICVInfo = OMPInfoCache.ICVs[ICV];
2696 auto &Getter = OMPInfoCache.RFIs[ICVInfo.Getter];
2697 if (Getter.Declaration == getAssociatedFunction()) {
2698 AssociatedICV = ICVInfo.Kind;
2699 return;
2700 }
2701 }
2702
2703 /// Unknown ICV.
2704 indicatePessimisticFixpoint();
2705 }
2706
2707 ChangeStatus manifest(Attributor &A) override {
2708 if (!ReplVal || !*ReplVal)
2709 return ChangeStatus::UNCHANGED;
2710
2711 A.changeAfterManifest(IRP: IRPosition::inst(I: *getCtxI()), NV&: **ReplVal);
2712 A.deleteAfterManifest(I&: *getCtxI());
2713
2714 return ChangeStatus::CHANGED;
2715 }
2716
2717 // FIXME: come up with better string.
2718 const std::string getAsStr(Attributor *) const override {
2719 return "ICVTrackerCallSite";
2720 }
2721
2722 // FIXME: come up with some stats.
2723 void trackStatistics() const override {}
2724
2725 InternalControlVar AssociatedICV;
2726 std::optional<Value *> ReplVal;
2727
2728 ChangeStatus updateImpl(Attributor &A) override {
2729 const auto *ICVTrackingAA = A.getAAFor<AAICVTracker>(
2730 QueryingAA: *this, IRP: IRPosition::function(F: *getAnchorScope()), DepClass: DepClassTy::REQUIRED);
2731
2732 // We don't have any information, so we assume it changes the ICV.
2733 if (!ICVTrackingAA->isAssumedTracked())
2734 return indicatePessimisticFixpoint();
2735
2736 std::optional<Value *> NewReplVal =
2737 ICVTrackingAA->getReplacementValue(ICV: AssociatedICV, I: getCtxI(), A);
2738
2739 if (ReplVal == NewReplVal)
2740 return ChangeStatus::UNCHANGED;
2741
2742 ReplVal = NewReplVal;
2743 return ChangeStatus::CHANGED;
2744 }
2745
2746 // Return the value with which associated value can be replaced for specific
2747 // \p ICV.
2748 std::optional<Value *>
2749 getUniqueReplacementValue(InternalControlVar ICV) const override {
2750 return ReplVal;
2751 }
2752};
2753
2754struct AAICVTrackerCallSiteReturned : AAICVTracker {
2755 AAICVTrackerCallSiteReturned(const IRPosition &IRP, Attributor &A)
2756 : AAICVTracker(IRP, A) {}
2757
2758 // FIXME: come up with better string.
2759 const std::string getAsStr(Attributor *) const override {
2760 return "ICVTrackerCallSiteReturned";
2761 }
2762
2763 // FIXME: come up with some stats.
2764 void trackStatistics() const override {}
2765
2766 /// We don't manifest anything for this AA.
2767 ChangeStatus manifest(Attributor &A) override {
2768 return ChangeStatus::UNCHANGED;
2769 }
2770
2771 // Map of ICV to their values at specific program point.
2772 EnumeratedArray<std::optional<Value *>, InternalControlVar,
2773 InternalControlVar::ICV___last>
2774 ICVReplacementValuesMap;
2775
2776 /// Return the value with which associated value can be replaced for specific
2777 /// \p ICV.
2778 std::optional<Value *>
2779 getUniqueReplacementValue(InternalControlVar ICV) const override {
2780 return ICVReplacementValuesMap[ICV];
2781 }
2782
2783 ChangeStatus updateImpl(Attributor &A) override {
2784 ChangeStatus Changed = ChangeStatus::UNCHANGED;
2785 const auto *ICVTrackingAA = A.getAAFor<AAICVTracker>(
2786 QueryingAA: *this, IRP: IRPosition::returned(F: *getAssociatedFunction()),
2787 DepClass: DepClassTy::REQUIRED);
2788
2789 // We don't have any information, so we assume it changes the ICV.
2790 if (!ICVTrackingAA->isAssumedTracked())
2791 return indicatePessimisticFixpoint();
2792
2793 for (InternalControlVar ICV : TrackableICVs) {
2794 std::optional<Value *> &ReplVal = ICVReplacementValuesMap[ICV];
2795 std::optional<Value *> NewReplVal =
2796 ICVTrackingAA->getUniqueReplacementValue(ICV);
2797
2798 if (ReplVal == NewReplVal)
2799 continue;
2800
2801 ReplVal = NewReplVal;
2802 Changed = ChangeStatus::CHANGED;
2803 }
2804 return Changed;
2805 }
2806};
2807
2808/// Determines if \p BB exits the function unconditionally itself or reaches a
2809/// block that does through only unique successors.
2810static bool hasFunctionEndAsUniqueSuccessor(const BasicBlock *BB) {
2811 if (succ_empty(BB))
2812 return true;
2813 const BasicBlock *const Successor = BB->getUniqueSuccessor();
2814 if (!Successor)
2815 return false;
2816 return hasFunctionEndAsUniqueSuccessor(BB: Successor);
2817}
2818
2819struct AAExecutionDomainFunction : public AAExecutionDomain {
2820 AAExecutionDomainFunction(const IRPosition &IRP, Attributor &A)
2821 : AAExecutionDomain(IRP, A) {}
2822
2823 ~AAExecutionDomainFunction() override { delete RPOT; }
2824
2825 void initialize(Attributor &A) override {
2826 Function *F = getAnchorScope();
2827 assert(F && "Expected anchor function");
2828 RPOT = new ReversePostOrderTraversal<Function *>(F);
2829 }
2830
2831 const std::string getAsStr(Attributor *) const override {
2832 unsigned TotalBlocks = 0, InitialThreadBlocks = 0, AlignedBlocks = 0;
2833 for (auto &It : BEDMap) {
2834 if (!It.getFirst())
2835 continue;
2836 TotalBlocks++;
2837 InitialThreadBlocks += It.getSecond().IsExecutedByInitialThreadOnly;
2838 AlignedBlocks += It.getSecond().IsReachedFromAlignedBarrierOnly &&
2839 It.getSecond().IsReachingAlignedBarrierOnly;
2840 }
2841 return "[AAExecutionDomain] " + std::to_string(val: InitialThreadBlocks) + "/" +
2842 std::to_string(val: AlignedBlocks) + " of " +
2843 std::to_string(val: TotalBlocks) +
2844 " executed by initial thread / aligned";
2845 }
2846
2847 /// See AbstractAttribute::trackStatistics().
2848 void trackStatistics() const override {}
2849
2850 ChangeStatus manifest(Attributor &A) override {
2851 LLVM_DEBUG({
2852 for (const BasicBlock &BB : *getAnchorScope()) {
2853 if (!isExecutedByInitialThreadOnly(BB))
2854 continue;
2855 dbgs() << TAG << " Basic block @" << getAnchorScope()->getName() << " "
2856 << BB.getName() << " is executed by a single thread.\n";
2857 }
2858 });
2859
2860 ChangeStatus Changed = ChangeStatus::UNCHANGED;
2861
2862 if (DisableOpenMPOptBarrierElimination)
2863 return Changed;
2864
2865 SmallPtrSet<CallBase *, 16> DeletedBarriers;
2866 auto HandleAlignedBarrier = [&](CallBase *CB) {
2867 const ExecutionDomainTy &ED = CB ? CEDMap[{CB, PRE}] : BEDMap[nullptr];
2868 if (!ED.IsReachedFromAlignedBarrierOnly ||
2869 ED.EncounteredNonLocalSideEffect)
2870 return;
2871 if (!ED.EncounteredAssumes.empty() && !A.isModulePass())
2872 return;
2873
2874 // We can remove this barrier, if it is one, or aligned barriers reaching
2875 // the kernel end (if CB is nullptr). Aligned barriers reaching the kernel
2876 // end should only be removed if the kernel end is their unique successor;
2877 // otherwise, they may have side-effects that aren't accounted for in the
2878 // kernel end in their other successors. If those barriers have other
2879 // barriers reaching them, those can be transitively removed as well as
2880 // long as the kernel end is also their unique successor.
2881 if (CB) {
2882 DeletedBarriers.insert(Ptr: CB);
2883 A.deleteAfterManifest(I&: *CB);
2884 ++NumBarriersEliminated;
2885 Changed = ChangeStatus::CHANGED;
2886 } else if (!ED.AlignedBarriers.empty()) {
2887 Changed = ChangeStatus::CHANGED;
2888 SmallVector<CallBase *> Worklist(ED.AlignedBarriers.begin(),
2889 ED.AlignedBarriers.end());
2890 SmallSetVector<CallBase *, 16> Visited;
2891 while (!Worklist.empty()) {
2892 CallBase *LastCB = Worklist.pop_back_val();
2893 if (!Visited.insert(X: LastCB))
2894 continue;
2895 if (LastCB->getFunction() != getAnchorScope())
2896 continue;
2897 if (!hasFunctionEndAsUniqueSuccessor(BB: LastCB->getParent()))
2898 continue;
2899 if (!DeletedBarriers.count(Ptr: LastCB)) {
2900 ++NumBarriersEliminated;
2901 A.deleteAfterManifest(I&: *LastCB);
2902 continue;
2903 }
2904 // The final aligned barrier (LastCB) reaching the kernel end was
2905 // removed already. This means we can go one step further and remove
2906 // the barriers encoutered last before (LastCB).
2907 const ExecutionDomainTy &LastED = CEDMap[{LastCB, PRE}];
2908 Worklist.append(in_start: LastED.AlignedBarriers.begin(),
2909 in_end: LastED.AlignedBarriers.end());
2910 }
2911 }
2912
2913 // If we actually eliminated a barrier we need to eliminate the associated
2914 // llvm.assumes as well to avoid creating UB.
2915 if (!ED.EncounteredAssumes.empty() && (CB || !ED.AlignedBarriers.empty()))
2916 for (auto *AssumeCB : ED.EncounteredAssumes)
2917 A.deleteAfterManifest(I&: *AssumeCB);
2918 };
2919
2920 for (auto *CB : AlignedBarriers)
2921 HandleAlignedBarrier(CB);
2922
2923 // Handle the "kernel end barrier" for kernels too.
2924 if (omp::isOpenMPKernel(Fn&: *getAnchorScope()))
2925 HandleAlignedBarrier(nullptr);
2926
2927 return Changed;
2928 }
2929
2930 bool isNoOpFence(const FenceInst &FI) const override {
2931 return getState().isValidState() && !NonNoOpFences.count(Ptr: &FI);
2932 }
2933
2934 /// Merge barrier and assumption information from \p PredED into the successor
2935 /// \p ED.
2936 void
2937 mergeInPredecessorBarriersAndAssumptions(Attributor &A, ExecutionDomainTy &ED,
2938 const ExecutionDomainTy &PredED);
2939
2940 /// Merge all information from \p PredED into the successor \p ED. If
2941 /// \p InitialEdgeOnly is set, only the initial edge will enter the block
2942 /// represented by \p ED from this predecessor.
2943 bool mergeInPredecessor(Attributor &A, ExecutionDomainTy &ED,
2944 const ExecutionDomainTy &PredED,
2945 bool InitialEdgeOnly = false);
2946
2947 /// Accumulate information for the entry block in \p EntryBBED.
2948 bool handleCallees(Attributor &A, ExecutionDomainTy &EntryBBED);
2949
2950 /// See AbstractAttribute::updateImpl.
2951 ChangeStatus updateImpl(Attributor &A) override;
2952
2953 /// Query interface, see AAExecutionDomain
2954 ///{
2955 bool isExecutedByInitialThreadOnly(const BasicBlock &BB) const override {
2956 if (!isValidState())
2957 return false;
2958 assert(BB.getParent() == getAnchorScope() && "Block is out of scope!");
2959 return BEDMap.lookup(Val: &BB).IsExecutedByInitialThreadOnly;
2960 }
2961
2962 bool isExecutedInAlignedRegion(Attributor &A,
2963 const Instruction &I) const override {
2964 assert(I.getFunction() == getAnchorScope() &&
2965 "Instruction is out of scope!");
2966 if (!isValidState())
2967 return false;
2968
2969 bool ForwardIsOk = true;
2970 const Instruction *CurI;
2971
2972 // Check forward until a call or the block end is reached.
2973 CurI = &I;
2974 do {
2975 auto *CB = dyn_cast<CallBase>(Val: CurI);
2976 if (!CB)
2977 continue;
2978 if (CB != &I && AlignedBarriers.contains(key: const_cast<CallBase *>(CB)))
2979 return true;
2980 const auto &It = CEDMap.find(Val: {CB, PRE});
2981 if (It == CEDMap.end())
2982 continue;
2983 if (!It->getSecond().IsReachingAlignedBarrierOnly)
2984 ForwardIsOk = false;
2985 break;
2986 } while ((CurI = CurI->getNextNode()));
2987
2988 if (!CurI && !BEDMap.lookup(Val: I.getParent()).IsReachingAlignedBarrierOnly)
2989 ForwardIsOk = false;
2990
2991 // Check backward until a call or the block beginning is reached.
2992 CurI = &I;
2993 do {
2994 auto *CB = dyn_cast<CallBase>(Val: CurI);
2995 if (!CB)
2996 continue;
2997 if (CB != &I && AlignedBarriers.contains(key: const_cast<CallBase *>(CB)))
2998 return true;
2999 const auto &It = CEDMap.find(Val: {CB, POST});
3000 if (It == CEDMap.end())
3001 continue;
3002 if (It->getSecond().IsReachedFromAlignedBarrierOnly)
3003 break;
3004 return false;
3005 } while ((CurI = CurI->getPrevNode()));
3006
3007 // Delayed decision on the forward pass to allow aligned barrier detection
3008 // in the backwards traversal.
3009 if (!ForwardIsOk)
3010 return false;
3011
3012 if (!CurI) {
3013 const BasicBlock *BB = I.getParent();
3014 if (BB == &BB->getParent()->getEntryBlock())
3015 return BEDMap.lookup(Val: nullptr).IsReachedFromAlignedBarrierOnly;
3016 if (!llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PredBB) {
3017 return BEDMap.lookup(Val: PredBB).IsReachedFromAlignedBarrierOnly;
3018 })) {
3019 return false;
3020 }
3021 }
3022
3023 // On neither traversal we found a anything but aligned barriers.
3024 return true;
3025 }
3026
3027 ExecutionDomainTy getExecutionDomain(const BasicBlock &BB) const override {
3028 assert(isValidState() &&
3029 "No request should be made against an invalid state!");
3030 return BEDMap.lookup(Val: &BB);
3031 }
3032 std::pair<ExecutionDomainTy, ExecutionDomainTy>
3033 getExecutionDomain(const CallBase &CB) const override {
3034 assert(isValidState() &&
3035 "No request should be made against an invalid state!");
3036 return {CEDMap.lookup(Val: {&CB, PRE}), CEDMap.lookup(Val: {&CB, POST})};
3037 }
3038 ExecutionDomainTy getFunctionExecutionDomain() const override {
3039 assert(isValidState() &&
3040 "No request should be made against an invalid state!");
3041 return InterProceduralED;
3042 }
3043 ///}
3044
3045 // Check if the edge into the successor block contains a condition that only
3046 // lets the main thread execute it.
3047 static bool isInitialThreadOnlyEdge(Attributor &A, CondBrInst *Edge,
3048 BasicBlock &SuccessorBB) {
3049 if (!Edge)
3050 return false;
3051 if (Edge->getSuccessor(i: 0) != &SuccessorBB)
3052 return false;
3053
3054 auto *Cmp = dyn_cast<CmpInst>(Val: Edge->getCondition());
3055 if (!Cmp || !Cmp->isTrueWhenEqual() || !Cmp->isEquality())
3056 return false;
3057
3058 ConstantInt *C = dyn_cast<ConstantInt>(Val: Cmp->getOperand(i_nocapture: 1));
3059 if (!C)
3060 return false;
3061
3062 // Match: -1 == __kmpc_target_init (for non-SPMD kernels only!)
3063 if (C->isAllOnesValue()) {
3064 auto *CB = dyn_cast<CallBase>(Val: Cmp->getOperand(i_nocapture: 0));
3065 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3066 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
3067 CB = CB ? OpenMPOpt::getCallIfRegularCall(V&: *CB, RFI: &RFI) : nullptr;
3068 if (!CB)
3069 return false;
3070 ConstantStruct *KernelEnvC =
3071 KernelInfo::getKernelEnvironementFromKernelInitCB(KernelInitCB: CB);
3072 ConstantInt *ExecModeC =
3073 KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC);
3074 return ExecModeC->getSExtValue() & OMP_TGT_EXEC_MODE_GENERIC;
3075 }
3076
3077 if (C->isZero()) {
3078 // Match: 0 == llvm.nvvm.read.ptx.sreg.tid.x()
3079 if (auto *II = dyn_cast<IntrinsicInst>(Val: Cmp->getOperand(i_nocapture: 0)))
3080 if (II->getIntrinsicID() == Intrinsic::nvvm_read_ptx_sreg_tid_x)
3081 return true;
3082
3083 // Match: 0 == llvm.amdgcn.workitem.id.x()
3084 if (auto *II = dyn_cast<IntrinsicInst>(Val: Cmp->getOperand(i_nocapture: 0)))
3085 if (II->getIntrinsicID() == Intrinsic::amdgcn_workitem_id_x)
3086 return true;
3087 }
3088
3089 return false;
3090 };
3091
3092 /// Mapping containing information about the function for other AAs.
3093 ExecutionDomainTy InterProceduralED;
3094
3095 enum Direction { PRE = 0, POST = 1 };
3096 /// Mapping containing information per block.
3097 DenseMap<const BasicBlock *, ExecutionDomainTy> BEDMap;
3098 DenseMap<PointerIntPair<const CallBase *, 1, Direction>, ExecutionDomainTy>
3099 CEDMap;
3100 SmallSetVector<CallBase *, 16> AlignedBarriers;
3101
3102 ReversePostOrderTraversal<Function *> *RPOT = nullptr;
3103
3104 /// Set \p R to \V and report true if that changed \p R.
3105 static bool setAndRecord(bool &R, bool V) {
3106 bool Eq = (R == V);
3107 R = V;
3108 return !Eq;
3109 }
3110
3111 /// Collection of fences known to be non-no-opt. All fences not in this set
3112 /// can be assumed no-opt.
3113 SmallPtrSet<const FenceInst *, 8> NonNoOpFences;
3114};
3115
3116void AAExecutionDomainFunction::mergeInPredecessorBarriersAndAssumptions(
3117 Attributor &A, ExecutionDomainTy &ED, const ExecutionDomainTy &PredED) {
3118 for (auto *EA : PredED.EncounteredAssumes)
3119 ED.addAssumeInst(A, AI&: *EA);
3120
3121 for (auto *AB : PredED.AlignedBarriers)
3122 ED.addAlignedBarrier(A, CB&: *AB);
3123}
3124
3125bool AAExecutionDomainFunction::mergeInPredecessor(
3126 Attributor &A, ExecutionDomainTy &ED, const ExecutionDomainTy &PredED,
3127 bool InitialEdgeOnly) {
3128
3129 bool Changed = false;
3130 Changed |=
3131 setAndRecord(R&: ED.IsExecutedByInitialThreadOnly,
3132 V: InitialEdgeOnly || (PredED.IsExecutedByInitialThreadOnly &&
3133 ED.IsExecutedByInitialThreadOnly));
3134
3135 Changed |= setAndRecord(R&: ED.IsReachedFromAlignedBarrierOnly,
3136 V: ED.IsReachedFromAlignedBarrierOnly &&
3137 PredED.IsReachedFromAlignedBarrierOnly);
3138 Changed |= setAndRecord(R&: ED.EncounteredNonLocalSideEffect,
3139 V: ED.EncounteredNonLocalSideEffect |
3140 PredED.EncounteredNonLocalSideEffect);
3141 // Do not track assumptions and barriers as part of Changed.
3142 if (ED.IsReachedFromAlignedBarrierOnly)
3143 mergeInPredecessorBarriersAndAssumptions(A, ED, PredED);
3144 else
3145 ED.clearAssumeInstAndAlignedBarriers();
3146 return Changed;
3147}
3148
3149bool AAExecutionDomainFunction::handleCallees(Attributor &A,
3150 ExecutionDomainTy &EntryBBED) {
3151 SmallVector<std::pair<ExecutionDomainTy, ExecutionDomainTy>, 4> CallSiteEDs;
3152 auto PredForCallSite = [&](AbstractCallSite ACS) {
3153 const auto *EDAA = A.getAAFor<AAExecutionDomain>(
3154 QueryingAA: *this, IRP: IRPosition::function(F: *ACS.getInstruction()->getFunction()),
3155 DepClass: DepClassTy::OPTIONAL);
3156 if (!EDAA || !EDAA->getState().isValidState())
3157 return false;
3158 CallSiteEDs.emplace_back(
3159 Args: EDAA->getExecutionDomain(CB: *cast<CallBase>(Val: ACS.getInstruction())));
3160 return true;
3161 };
3162
3163 ExecutionDomainTy ExitED;
3164 bool AllCallSitesKnown;
3165 if (A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this,
3166 /* RequiresAllCallSites */ RequireAllCallSites: true,
3167 UsedAssumedInformation&: AllCallSitesKnown)) {
3168 for (const auto &[CSInED, CSOutED] : CallSiteEDs) {
3169 mergeInPredecessor(A, ED&: EntryBBED, PredED: CSInED);
3170 ExitED.IsReachingAlignedBarrierOnly &=
3171 CSOutED.IsReachingAlignedBarrierOnly;
3172 }
3173
3174 } else {
3175 // We could not find all predecessors, so this is either a kernel or a
3176 // function with external linkage (or with some other weird uses).
3177 if (omp::isOpenMPKernel(Fn&: *getAnchorScope())) {
3178 EntryBBED.IsExecutedByInitialThreadOnly = false;
3179 EntryBBED.IsReachedFromAlignedBarrierOnly = true;
3180 EntryBBED.EncounteredNonLocalSideEffect = false;
3181 ExitED.IsReachingAlignedBarrierOnly = false;
3182 } else {
3183 EntryBBED.IsExecutedByInitialThreadOnly = false;
3184 EntryBBED.IsReachedFromAlignedBarrierOnly = false;
3185 EntryBBED.EncounteredNonLocalSideEffect = true;
3186 ExitED.IsReachingAlignedBarrierOnly = false;
3187 }
3188 }
3189
3190 bool Changed = false;
3191 auto &FnED = BEDMap[nullptr];
3192 Changed |= setAndRecord(R&: FnED.IsReachedFromAlignedBarrierOnly,
3193 V: FnED.IsReachedFromAlignedBarrierOnly &
3194 EntryBBED.IsReachedFromAlignedBarrierOnly);
3195 Changed |= setAndRecord(R&: FnED.IsReachingAlignedBarrierOnly,
3196 V: FnED.IsReachingAlignedBarrierOnly &
3197 ExitED.IsReachingAlignedBarrierOnly);
3198 Changed |= setAndRecord(R&: FnED.IsExecutedByInitialThreadOnly,
3199 V: EntryBBED.IsExecutedByInitialThreadOnly);
3200 return Changed;
3201}
3202
3203ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
3204
3205 bool Changed = false;
3206
3207 // Helper to deal with an aligned barrier encountered during the forward
3208 // traversal. \p CB is the aligned barrier, \p ED is the execution domain when
3209 // it was encountered.
3210 auto HandleAlignedBarrier = [&](CallBase &CB, ExecutionDomainTy &ED) {
3211 Changed |= AlignedBarriers.insert(X: &CB);
3212 // First, update the barrier ED kept in the separate CEDMap.
3213 auto &CallInED = CEDMap[{&CB, PRE}];
3214 Changed |= mergeInPredecessor(A, ED&: CallInED, PredED: ED);
3215 CallInED.IsReachingAlignedBarrierOnly = true;
3216 // Next adjust the ED we use for the traversal.
3217 ED.EncounteredNonLocalSideEffect = false;
3218 ED.IsReachedFromAlignedBarrierOnly = true;
3219 // Aligned barrier collection has to come last.
3220 ED.clearAssumeInstAndAlignedBarriers();
3221 ED.addAlignedBarrier(A, CB);
3222 auto &CallOutED = CEDMap[{&CB, POST}];
3223 Changed |= mergeInPredecessor(A, ED&: CallOutED, PredED: ED);
3224 };
3225
3226 auto *LivenessAA =
3227 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3228
3229 Function *F = getAnchorScope();
3230 BasicBlock &EntryBB = F->getEntryBlock();
3231 bool IsKernel = omp::isOpenMPKernel(Fn&: *F);
3232
3233 SmallVector<Instruction *> SyncInstWorklist;
3234 for (auto &RIt : *RPOT) {
3235 BasicBlock &BB = *RIt;
3236
3237 bool IsEntryBB = &BB == &EntryBB;
3238 // TODO: We use local reasoning since we don't have a divergence analysis
3239 // running as well. We could basically allow uniform branches here.
3240 bool AlignedBarrierLastInBlock = IsEntryBB && IsKernel;
3241 bool IsExplicitlyAligned = IsEntryBB && IsKernel;
3242 ExecutionDomainTy ED;
3243 // Propagate "incoming edges" into information about this block.
3244 if (IsEntryBB) {
3245 Changed |= handleCallees(A, EntryBBED&: ED);
3246 } else {
3247 // For live non-entry blocks we only propagate
3248 // information via live edges.
3249 if (LivenessAA && LivenessAA->isAssumedDead(BB: &BB))
3250 continue;
3251
3252 for (auto *PredBB : predecessors(BB: &BB)) {
3253 if (LivenessAA && LivenessAA->isEdgeDead(From: PredBB, To: &BB))
3254 continue;
3255 bool InitialEdgeOnly = isInitialThreadOnlyEdge(
3256 A, Edge: dyn_cast<CondBrInst>(Val: PredBB->getTerminator()), SuccessorBB&: BB);
3257 mergeInPredecessor(A, ED, PredED: BEDMap[PredBB], InitialEdgeOnly);
3258 }
3259 }
3260
3261 // Now we traverse the block, accumulate effects in ED and attach
3262 // information to calls.
3263 for (Instruction &I : BB) {
3264 bool UsedAssumedInformation;
3265 if (A.isAssumedDead(I, QueryingAA: *this, LivenessAA, UsedAssumedInformation,
3266 /* CheckBBLivenessOnly */ false, DepClass: DepClassTy::OPTIONAL,
3267 /* CheckForDeadStore */ true))
3268 continue;
3269
3270 // Asummes and "assume-like" (dbg, lifetime, ...) are handled first, the
3271 // former is collected the latter is ignored.
3272 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
3273 if (auto *AI = dyn_cast_or_null<AssumeInst>(Val: II)) {
3274 ED.addAssumeInst(A, AI&: *AI);
3275 continue;
3276 }
3277 // TODO: Should we also collect and delete lifetime markers?
3278 if (II->isAssumeLikeIntrinsic())
3279 continue;
3280 }
3281
3282 if (auto *FI = dyn_cast<FenceInst>(Val: &I)) {
3283 if (!ED.EncounteredNonLocalSideEffect) {
3284 // An aligned fence without non-local side-effects is a no-op.
3285 if (ED.IsReachedFromAlignedBarrierOnly)
3286 continue;
3287 // A non-aligned fence without non-local side-effects is a no-op
3288 // if the ordering only publishes non-local side-effects (or less).
3289 switch (FI->getOrdering()) {
3290 case AtomicOrdering::NotAtomic:
3291 continue;
3292 case AtomicOrdering::Unordered:
3293 continue;
3294 case AtomicOrdering::Monotonic:
3295 continue;
3296 case AtomicOrdering::Acquire:
3297 break;
3298 case AtomicOrdering::Release:
3299 continue;
3300 case AtomicOrdering::AcquireRelease:
3301 break;
3302 case AtomicOrdering::SequentiallyConsistent:
3303 break;
3304 };
3305 }
3306 NonNoOpFences.insert(Ptr: FI);
3307 }
3308
3309 auto *CB = dyn_cast<CallBase>(Val: &I);
3310 bool IsNoSync = AA::isNoSyncInst(A, I, QueryingAA: *this);
3311 bool IsAlignedBarrier =
3312 !IsNoSync && CB &&
3313 AANoSync::isAlignedBarrier(CB: *CB, ExecutedAligned: AlignedBarrierLastInBlock);
3314
3315 AlignedBarrierLastInBlock &= IsNoSync;
3316 IsExplicitlyAligned &= IsNoSync;
3317
3318 // Next we check for calls. Aligned barriers are handled
3319 // explicitly, everything else is kept for the backward traversal and will
3320 // also affect our state.
3321 if (CB) {
3322 if (IsAlignedBarrier) {
3323 HandleAlignedBarrier(*CB, ED);
3324 AlignedBarrierLastInBlock = true;
3325 IsExplicitlyAligned = true;
3326 continue;
3327 }
3328
3329 // Check the pointer(s) of a memory intrinsic explicitly.
3330 if (isa<MemIntrinsic>(Val: &I)) {
3331 if (!ED.EncounteredNonLocalSideEffect &&
3332 AA::isPotentiallyAffectedByBarrier(A, I, QueryingAA: *this))
3333 ED.EncounteredNonLocalSideEffect = true;
3334 if (!IsNoSync) {
3335 ED.IsReachedFromAlignedBarrierOnly = false;
3336 SyncInstWorklist.push_back(Elt: &I);
3337 }
3338 continue;
3339 }
3340
3341 // Record how we entered the call, then accumulate the effect of the
3342 // call in ED for potential use by the callee.
3343 auto &CallInED = CEDMap[{CB, PRE}];
3344 Changed |= mergeInPredecessor(A, ED&: CallInED, PredED: ED);
3345
3346 // If we have a sync-definition we can check if it starts/ends in an
3347 // aligned barrier. If we are unsure we assume any sync breaks
3348 // alignment.
3349 Function *Callee = CB->getCalledFunction();
3350 if (!IsNoSync && Callee && !Callee->isDeclaration()) {
3351 const auto *EDAA = A.getAAFor<AAExecutionDomain>(
3352 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::OPTIONAL);
3353 if (EDAA && EDAA->getState().isValidState()) {
3354 const auto &CalleeED = EDAA->getFunctionExecutionDomain();
3355 ED.IsReachedFromAlignedBarrierOnly =
3356 CalleeED.IsReachedFromAlignedBarrierOnly;
3357 AlignedBarrierLastInBlock = ED.IsReachedFromAlignedBarrierOnly;
3358 if (IsNoSync || !CalleeED.IsReachedFromAlignedBarrierOnly)
3359 ED.EncounteredNonLocalSideEffect |=
3360 CalleeED.EncounteredNonLocalSideEffect;
3361 else
3362 ED.EncounteredNonLocalSideEffect =
3363 CalleeED.EncounteredNonLocalSideEffect;
3364 if (!CalleeED.IsReachingAlignedBarrierOnly) {
3365 Changed |=
3366 setAndRecord(R&: CallInED.IsReachingAlignedBarrierOnly, V: false);
3367 SyncInstWorklist.push_back(Elt: &I);
3368 }
3369 if (CalleeED.IsReachedFromAlignedBarrierOnly)
3370 mergeInPredecessorBarriersAndAssumptions(A, ED, PredED: CalleeED);
3371 auto &CallOutED = CEDMap[{CB, POST}];
3372 Changed |= mergeInPredecessor(A, ED&: CallOutED, PredED: ED);
3373 continue;
3374 }
3375 }
3376 if (!IsNoSync) {
3377 ED.IsReachedFromAlignedBarrierOnly = false;
3378 Changed |= setAndRecord(R&: CallInED.IsReachingAlignedBarrierOnly, V: false);
3379 SyncInstWorklist.push_back(Elt: &I);
3380 }
3381 AlignedBarrierLastInBlock &= ED.IsReachedFromAlignedBarrierOnly;
3382 ED.EncounteredNonLocalSideEffect |= !CB->doesNotAccessMemory();
3383 auto &CallOutED = CEDMap[{CB, POST}];
3384 Changed |= mergeInPredecessor(A, ED&: CallOutED, PredED: ED);
3385 }
3386
3387 if (!I.mayHaveSideEffects() && !I.mayReadFromMemory())
3388 continue;
3389
3390 // If we have a callee we try to use fine-grained information to
3391 // determine local side-effects.
3392 if (CB) {
3393 const auto *MemAA = A.getAAFor<AAMemoryLocation>(
3394 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
3395
3396 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
3397 AAMemoryLocation::AccessKind,
3398 AAMemoryLocation::MemoryLocationsKind) {
3399 return !AA::isPotentiallyAffectedByBarrier(A, Ptrs: {Ptr}, QueryingAA: *this, CtxI: I);
3400 };
3401 if (MemAA && MemAA->getState().isValidState() &&
3402 MemAA->checkForAllAccessesToMemoryKind(
3403 Pred: AccessPred, MLK: AAMemoryLocation::ALL_LOCATIONS))
3404 continue;
3405 }
3406
3407 auto &InfoCache = A.getInfoCache();
3408 if (!I.mayHaveSideEffects() && InfoCache.isOnlyUsedByAssume(I))
3409 continue;
3410
3411 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
3412 if (LI->hasMetadata(KindID: LLVMContext::MD_invariant_load))
3413 continue;
3414
3415 if (!ED.EncounteredNonLocalSideEffect &&
3416 AA::isPotentiallyAffectedByBarrier(A, I, QueryingAA: *this))
3417 ED.EncounteredNonLocalSideEffect = true;
3418 }
3419
3420 bool IsEndAndNotReachingAlignedBarriersOnly = false;
3421 if (!isa<UnreachableInst>(Val: BB.getTerminator()) &&
3422 !BB.getTerminator()->getNumSuccessors()) {
3423
3424 Changed |= mergeInPredecessor(A, ED&: InterProceduralED, PredED: ED);
3425
3426 auto &FnED = BEDMap[nullptr];
3427 if (IsKernel && !IsExplicitlyAligned)
3428 FnED.IsReachingAlignedBarrierOnly = false;
3429 Changed |= mergeInPredecessor(A, ED&: FnED, PredED: ED);
3430
3431 if (!FnED.IsReachingAlignedBarrierOnly) {
3432 IsEndAndNotReachingAlignedBarriersOnly = true;
3433 SyncInstWorklist.push_back(Elt: BB.getTerminator());
3434 auto &BBED = BEDMap[&BB];
3435 Changed |= setAndRecord(R&: BBED.IsReachingAlignedBarrierOnly, V: false);
3436 }
3437 }
3438
3439 ExecutionDomainTy &StoredED = BEDMap[&BB];
3440 ED.IsReachingAlignedBarrierOnly = StoredED.IsReachingAlignedBarrierOnly &&
3441 !IsEndAndNotReachingAlignedBarriersOnly;
3442
3443 // Check if we computed anything different as part of the forward
3444 // traversal. We do not take assumptions and aligned barriers into account
3445 // as they do not influence the state we iterate. Backward traversal values
3446 // are handled later on.
3447 if (ED.IsExecutedByInitialThreadOnly !=
3448 StoredED.IsExecutedByInitialThreadOnly ||
3449 ED.IsReachedFromAlignedBarrierOnly !=
3450 StoredED.IsReachedFromAlignedBarrierOnly ||
3451 ED.EncounteredNonLocalSideEffect !=
3452 StoredED.EncounteredNonLocalSideEffect)
3453 Changed = true;
3454
3455 // Update the state with the new value.
3456 StoredED = std::move(ED);
3457 }
3458
3459 // Propagate (non-aligned) sync instruction effects backwards until the
3460 // entry is hit or an aligned barrier.
3461 SmallSetVector<BasicBlock *, 16> Visited;
3462 while (!SyncInstWorklist.empty()) {
3463 Instruction *SyncInst = SyncInstWorklist.pop_back_val();
3464 Instruction *CurInst = SyncInst;
3465 bool HitAlignedBarrierOrKnownEnd = false;
3466 while ((CurInst = CurInst->getPrevNode())) {
3467 auto *CB = dyn_cast<CallBase>(Val: CurInst);
3468 if (!CB)
3469 continue;
3470 auto &CallOutED = CEDMap[{CB, POST}];
3471 Changed |= setAndRecord(R&: CallOutED.IsReachingAlignedBarrierOnly, V: false);
3472 auto &CallInED = CEDMap[{CB, PRE}];
3473 HitAlignedBarrierOrKnownEnd =
3474 AlignedBarriers.count(key: CB) || !CallInED.IsReachingAlignedBarrierOnly;
3475 if (HitAlignedBarrierOrKnownEnd)
3476 break;
3477 Changed |= setAndRecord(R&: CallInED.IsReachingAlignedBarrierOnly, V: false);
3478 }
3479 if (HitAlignedBarrierOrKnownEnd)
3480 continue;
3481 BasicBlock *SyncBB = SyncInst->getParent();
3482 for (auto *PredBB : predecessors(BB: SyncBB)) {
3483 if (LivenessAA && LivenessAA->isEdgeDead(From: PredBB, To: SyncBB))
3484 continue;
3485 if (!Visited.insert(X: PredBB))
3486 continue;
3487 auto &PredED = BEDMap[PredBB];
3488 if (setAndRecord(R&: PredED.IsReachingAlignedBarrierOnly, V: false)) {
3489 Changed = true;
3490 SyncInstWorklist.push_back(Elt: PredBB->getTerminator());
3491 }
3492 }
3493 if (SyncBB != &EntryBB)
3494 continue;
3495 Changed |=
3496 setAndRecord(R&: InterProceduralED.IsReachingAlignedBarrierOnly, V: false);
3497 }
3498
3499 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
3500}
3501
3502/// Try to replace memory allocation calls called by a single thread with a
3503/// static buffer of shared memory.
3504struct AAHeapToShared : public StateWrapper<BooleanState, AbstractAttribute> {
3505 using Base = StateWrapper<BooleanState, AbstractAttribute>;
3506 AAHeapToShared(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
3507
3508 /// Create an abstract attribute view for the position \p IRP.
3509 static AAHeapToShared &createForPosition(const IRPosition &IRP,
3510 Attributor &A);
3511
3512 /// Returns true if HeapToShared conversion is assumed to be possible.
3513 virtual bool isAssumedHeapToShared(CallBase &CB) const = 0;
3514
3515 /// Returns true if HeapToShared conversion is assumed and the CB is a
3516 /// callsite to a free operation to be removed.
3517 virtual bool isAssumedHeapToSharedRemovedFree(CallBase &CB) const = 0;
3518
3519 /// See AbstractAttribute::getName().
3520 StringRef getName() const override { return "AAHeapToShared"; }
3521
3522 /// See AbstractAttribute::getIdAddr().
3523 const char *getIdAddr() const override { return &ID; }
3524
3525 /// This function should return true if the type of the \p AA is
3526 /// AAHeapToShared.
3527 static bool classof(const AbstractAttribute *AA) {
3528 return (AA->getIdAddr() == &ID);
3529 }
3530
3531 /// Unique ID (due to the unique address)
3532 static const char ID;
3533};
3534
3535struct AAHeapToSharedFunction : public AAHeapToShared {
3536 AAHeapToSharedFunction(const IRPosition &IRP, Attributor &A)
3537 : AAHeapToShared(IRP, A) {}
3538
3539 const std::string getAsStr(Attributor *) const override {
3540 return "[AAHeapToShared] " + std::to_string(val: MallocCalls.size()) +
3541 " malloc calls eligible.";
3542 }
3543
3544 /// See AbstractAttribute::trackStatistics().
3545 void trackStatistics() const override {}
3546
3547 /// This functions finds free calls that will be removed by the
3548 /// HeapToShared transformation.
3549 void findPotentialRemovedFreeCalls(Attributor &A) {
3550 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3551 auto &FreeRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
3552
3553 PotentialRemovedFreeCalls.clear();
3554 // Update free call users of found malloc calls.
3555 for (CallBase *CB : MallocCalls) {
3556 SmallVector<CallBase *, 4> FreeCalls;
3557 for (auto *U : CB->users()) {
3558 CallBase *C = dyn_cast<CallBase>(Val: U);
3559 if (C && C->getCalledFunction() == FreeRFI.Declaration)
3560 FreeCalls.push_back(Elt: C);
3561 }
3562
3563 if (FreeCalls.size() != 1)
3564 continue;
3565
3566 PotentialRemovedFreeCalls.insert(Ptr: FreeCalls.front());
3567 }
3568 }
3569
3570 void initialize(Attributor &A) override {
3571 if (DisableOpenMPOptDeglobalization) {
3572 indicatePessimisticFixpoint();
3573 return;
3574 }
3575
3576 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3577 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3578 if (!RFI.Declaration)
3579 return;
3580
3581 Attributor::SimplifictionCallbackTy SCB =
3582 [](const IRPosition &, const AbstractAttribute *,
3583 bool &) -> std::optional<Value *> { return nullptr; };
3584
3585 Function *F = getAnchorScope();
3586 const OMPInformationCache::RuntimeFunctionInfo::UseVector *Uses =
3587 RFI.getUseVector(F&: *F);
3588 if (!Uses)
3589 return;
3590
3591 for (Use *U : *Uses)
3592 if (CallBase *CB = dyn_cast<CallBase>(Val: U->getUser())) {
3593 MallocCalls.insert(X: CB);
3594 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *CB),
3595 CB: SCB);
3596 }
3597
3598 findPotentialRemovedFreeCalls(A);
3599 }
3600
3601 bool isAssumedHeapToShared(CallBase &CB) const override {
3602 return isValidState() && MallocCalls.count(key: &CB);
3603 }
3604
3605 bool isAssumedHeapToSharedRemovedFree(CallBase &CB) const override {
3606 return isValidState() && PotentialRemovedFreeCalls.count(Ptr: &CB);
3607 }
3608
3609 ChangeStatus manifest(Attributor &A) override {
3610 if (MallocCalls.empty())
3611 return ChangeStatus::UNCHANGED;
3612
3613 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3614 auto &FreeCall = OMPInfoCache.RFIs[OMPRTL___kmpc_free_shared];
3615
3616 Function *F = getAnchorScope();
3617 auto *HS = A.lookupAAFor<AAHeapToStack>(IRP: IRPosition::function(F: *F), QueryingAA: this,
3618 DepClass: DepClassTy::OPTIONAL);
3619
3620 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3621 for (CallBase *CB : MallocCalls) {
3622 // Skip replacing this if HeapToStack has already claimed it.
3623 if (HS && HS->isAssumedHeapToStack(CB: *CB))
3624 continue;
3625
3626 // Find the unique free call to remove it.
3627 SmallVector<CallBase *, 4> FreeCalls;
3628 for (auto *U : CB->users()) {
3629 CallBase *C = dyn_cast<CallBase>(Val: U);
3630 if (C && C->getCalledFunction() == FreeCall.Declaration)
3631 FreeCalls.push_back(Elt: C);
3632 }
3633 if (FreeCalls.size() != 1)
3634 continue;
3635
3636 auto *AllocSize = cast<ConstantInt>(Val: CB->getArgOperand(i: 0));
3637
3638 if (AllocSize->getZExtValue() + SharedMemoryUsed > SharedMemoryLimit) {
3639 LLVM_DEBUG(dbgs() << TAG << "Cannot replace call " << *CB
3640 << " with shared memory."
3641 << " Shared memory usage is limited to "
3642 << SharedMemoryLimit << " bytes\n");
3643 continue;
3644 }
3645
3646 LLVM_DEBUG(dbgs() << TAG << "Replace globalization call " << *CB
3647 << " with " << AllocSize->getZExtValue()
3648 << " bytes of shared memory\n");
3649
3650 // Create a new shared memory buffer of the same size as the allocation
3651 // and replace all the uses of the original allocation with it.
3652 Module *M = CB->getModule();
3653 Type *Int8Ty = Type::getInt8Ty(C&: M->getContext());
3654 Type *Int8ArrTy = ArrayType::get(ElementType: Int8Ty, NumElements: AllocSize->getZExtValue());
3655 auto *SharedMem = new GlobalVariable(
3656 *M, Int8ArrTy, /* IsConstant */ false, GlobalValue::InternalLinkage,
3657 PoisonValue::get(T: Int8ArrTy), CB->getName() + "_shared", nullptr,
3658 GlobalValue::NotThreadLocal,
3659 static_cast<unsigned>(AddressSpace::Shared));
3660 auto *NewBuffer = ConstantExpr::getPointerCast(
3661 C: SharedMem, Ty: PointerType::getUnqual(C&: M->getContext()));
3662
3663 auto Remark = [&](OptimizationRemark OR) {
3664 return OR << "Replaced globalized variable with "
3665 << ore::NV("SharedMemory", AllocSize->getZExtValue())
3666 << (AllocSize->isOne() ? " byte " : " bytes ")
3667 << "of shared memory.";
3668 };
3669 A.emitRemark<OptimizationRemark>(I: CB, RemarkName: "OMP111", RemarkCB&: Remark);
3670
3671 MaybeAlign Alignment = CB->getRetAlign();
3672 assert(Alignment &&
3673 "HeapToShared on allocation without alignment attribute");
3674 SharedMem->setAlignment(*Alignment);
3675
3676 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewBuffer);
3677 A.deleteAfterManifest(I&: *CB);
3678 A.deleteAfterManifest(I&: *FreeCalls.front());
3679
3680 SharedMemoryUsed += AllocSize->getZExtValue();
3681 NumBytesMovedToSharedMemory = SharedMemoryUsed;
3682 Changed = ChangeStatus::CHANGED;
3683 }
3684
3685 return Changed;
3686 }
3687
3688 ChangeStatus updateImpl(Attributor &A) override {
3689 if (MallocCalls.empty())
3690 return indicatePessimisticFixpoint();
3691 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3692 auto &RFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
3693 if (!RFI.Declaration)
3694 return ChangeStatus::UNCHANGED;
3695
3696 Function *F = getAnchorScope();
3697
3698 auto NumMallocCalls = MallocCalls.size();
3699
3700 // Only consider malloc calls executed by a single thread with a constant.
3701 for (User *U : RFI.Declaration->users()) {
3702 if (CallBase *CB = dyn_cast<CallBase>(Val: U)) {
3703 if (CB->getCaller() != F)
3704 continue;
3705 if (!MallocCalls.count(key: CB))
3706 continue;
3707 if (!isa<ConstantInt>(Val: CB->getArgOperand(i: 0))) {
3708 MallocCalls.remove(X: CB);
3709 continue;
3710 }
3711 const auto *ED = A.getAAFor<AAExecutionDomain>(
3712 QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::REQUIRED);
3713 if (!ED || !ED->isExecutedByInitialThreadOnly(I: *CB))
3714 MallocCalls.remove(X: CB);
3715 }
3716 }
3717
3718 findPotentialRemovedFreeCalls(A);
3719
3720 if (NumMallocCalls != MallocCalls.size())
3721 return ChangeStatus::CHANGED;
3722
3723 return ChangeStatus::UNCHANGED;
3724 }
3725
3726 /// Collection of all malloc calls in a function.
3727 SmallSetVector<CallBase *, 4> MallocCalls;
3728 /// Collection of potentially removed free calls in a function.
3729 SmallPtrSet<CallBase *, 4> PotentialRemovedFreeCalls;
3730 /// The total amount of shared memory that has been used for HeapToShared.
3731 unsigned SharedMemoryUsed = 0;
3732};
3733
3734struct AAKernelInfo : public StateWrapper<KernelInfoState, AbstractAttribute> {
3735 using Base = StateWrapper<KernelInfoState, AbstractAttribute>;
3736 AAKernelInfo(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
3737
3738 /// The callee value is tracked beyond a simple stripPointerCasts, so we allow
3739 /// unknown callees.
3740 static bool requiresCalleeForCallBase() { return false; }
3741
3742 /// Statistics are tracked as part of manifest for now.
3743 void trackStatistics() const override {}
3744
3745 /// See AbstractAttribute::getAsStr()
3746 const std::string getAsStr(Attributor *) const override {
3747 if (!isValidState())
3748 return "<invalid>";
3749 return std::string(SPMDCompatibilityTracker.isAssumed() ? "SPMD"
3750 : "generic") +
3751 std::string(SPMDCompatibilityTracker.isAtFixpoint() ? " [FIX]"
3752 : "") +
3753 std::string(" #PRs: ") +
3754 (ReachedKnownParallelRegions.isValidState()
3755 ? std::to_string(val: ReachedKnownParallelRegions.size())
3756 : "<invalid>") +
3757 ", #Unknown PRs: " +
3758 (ReachedUnknownParallelRegions.isValidState()
3759 ? std::to_string(val: ReachedUnknownParallelRegions.size())
3760 : "<invalid>") +
3761 ", #Reaching Kernels: " +
3762 (ReachingKernelEntries.isValidState()
3763 ? std::to_string(val: ReachingKernelEntries.size())
3764 : "<invalid>") +
3765 ", #ParLevels: " +
3766 (ParallelLevels.isValidState()
3767 ? std::to_string(val: ParallelLevels.size())
3768 : "<invalid>") +
3769 ", NestedPar: " + (NestedParallelism ? "yes" : "no");
3770 }
3771
3772 /// Create an abstract attribute biew for the position \p IRP.
3773 static AAKernelInfo &createForPosition(const IRPosition &IRP, Attributor &A);
3774
3775 /// See AbstractAttribute::getName()
3776 StringRef getName() const override { return "AAKernelInfo"; }
3777
3778 /// See AbstractAttribute::getIdAddr()
3779 const char *getIdAddr() const override { return &ID; }
3780
3781 /// This function should return true if the type of the \p AA is AAKernelInfo
3782 static bool classof(const AbstractAttribute *AA) {
3783 return (AA->getIdAddr() == &ID);
3784 }
3785
3786 static const char ID;
3787};
3788
3789/// The function kernel info abstract attribute, basically, what can we say
3790/// about a function with regards to the KernelInfoState.
3791struct AAKernelInfoFunction : AAKernelInfo {
3792 AAKernelInfoFunction(const IRPosition &IRP, Attributor &A)
3793 : AAKernelInfo(IRP, A) {}
3794
3795 SmallPtrSet<Instruction *, 4> GuardedInstructions;
3796
3797 SmallPtrSetImpl<Instruction *> &getGuardedInstructions() {
3798 return GuardedInstructions;
3799 }
3800
3801 void setConfigurationOfKernelEnvironment(ConstantStruct *ConfigC) {
3802 Constant *NewKernelEnvC = ConstantFoldInsertValueInstruction(
3803 Agg: KernelEnvC, Val: ConfigC, Idxs: {KernelInfo::ConfigurationIdx});
3804 assert(NewKernelEnvC && "Failed to create new kernel environment");
3805 KernelEnvC = cast<ConstantStruct>(Val: NewKernelEnvC);
3806 }
3807
3808#define KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MEMBER) \
3809 void set##MEMBER##OfKernelEnvironment(ConstantInt *NewVal) { \
3810 ConstantStruct *ConfigC = \
3811 KernelInfo::getConfigurationFromKernelEnvironment(KernelEnvC); \
3812 Constant *NewConfigC = ConstantFoldInsertValueInstruction( \
3813 ConfigC, NewVal, {KernelInfo::MEMBER##Idx}); \
3814 assert(NewConfigC && "Failed to create new configuration environment"); \
3815 setConfigurationOfKernelEnvironment(cast<ConstantStruct>(NewConfigC)); \
3816 }
3817
3818 KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(UseGenericStateMachine)
3819 KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MayUseNestedParallelism)
3820 KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(ExecMode)
3821 KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MinThreads)
3822 KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MaxThreads)
3823 KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MinTeams)
3824 KERNEL_ENVIRONMENT_CONFIGURATION_SETTER(MaxTeams)
3825
3826#undef KERNEL_ENVIRONMENT_CONFIGURATION_SETTER
3827
3828 /// See AbstractAttribute::initialize(...).
3829 void initialize(Attributor &A) override {
3830 // This is a high-level transform that might change the constant arguments
3831 // of the init and dinit calls. We need to tell the Attributor about this
3832 // to avoid other parts using the current constant value for simpliication.
3833 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
3834
3835 Function *Fn = getAnchorScope();
3836
3837 OMPInformationCache::RuntimeFunctionInfo &InitRFI =
3838 OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
3839 OMPInformationCache::RuntimeFunctionInfo &DeinitRFI =
3840 OMPInfoCache.RFIs[OMPRTL___kmpc_target_deinit];
3841
3842 // For kernels we perform more initialization work, first we find the init
3843 // and deinit calls.
3844 auto StoreCallBase = [](Use &U,
3845 OMPInformationCache::RuntimeFunctionInfo &RFI,
3846 CallBase *&Storage) {
3847 CallBase *CB = OpenMPOpt::getCallIfRegularCall(U, RFI: &RFI);
3848 assert(CB &&
3849 "Unexpected use of __kmpc_target_init or __kmpc_target_deinit!");
3850 assert(!Storage &&
3851 "Multiple uses of __kmpc_target_init or __kmpc_target_deinit!");
3852 Storage = CB;
3853 return false;
3854 };
3855 InitRFI.foreachUse(
3856 CB: [&](Use &U, Function &) {
3857 StoreCallBase(U, InitRFI, KernelInitCB);
3858 return false;
3859 },
3860 F: Fn);
3861 DeinitRFI.foreachUse(
3862 CB: [&](Use &U, Function &) {
3863 StoreCallBase(U, DeinitRFI, KernelDeinitCB);
3864 return false;
3865 },
3866 F: Fn);
3867
3868 // Ignore kernels without initializers such as global constructors.
3869 if (!KernelInitCB || !KernelDeinitCB)
3870 return;
3871
3872 // Add itself to the reaching kernel and set IsKernelEntry.
3873 ReachingKernelEntries.insert(Elem: Fn);
3874 IsKernelEntry = true;
3875
3876 KernelEnvC =
3877 KernelInfo::getKernelEnvironementFromKernelInitCB(KernelInitCB);
3878 GlobalVariable *KernelEnvGV =
3879 KernelInfo::getKernelEnvironementGVFromKernelInitCB(KernelInitCB);
3880
3881 Attributor::GlobalVariableSimplifictionCallbackTy
3882 KernelConfigurationSimplifyCB =
3883 [&](const GlobalVariable &GV, const AbstractAttribute *AA,
3884 bool &UsedAssumedInformation) -> std::optional<Constant *> {
3885 if (!isAtFixpoint()) {
3886 if (!AA)
3887 return nullptr;
3888 UsedAssumedInformation = true;
3889 A.recordDependence(FromAA: *this, ToAA: *AA, DepClass: DepClassTy::OPTIONAL);
3890 }
3891 return KernelEnvC;
3892 };
3893
3894 A.registerGlobalVariableSimplificationCallback(
3895 GV: *KernelEnvGV, CB: KernelConfigurationSimplifyCB);
3896
3897 // We cannot change to SPMD mode if the runtime functions aren't availible.
3898 bool CanChangeToSPMD = OMPInfoCache.runtimeFnsAvailable(
3899 Fns: {OMPRTL___kmpc_get_hardware_thread_id_in_block,
3900 OMPRTL___kmpc_barrier_simple_spmd});
3901
3902 // Check if we know we are in SPMD-mode already.
3903 ConstantInt *ExecModeC =
3904 KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC);
3905 ConstantInt *AssumedExecModeC = ConstantInt::get(
3906 Ty: ExecModeC->getIntegerType(),
3907 V: ExecModeC->getSExtValue() | OMP_TGT_EXEC_MODE_GENERIC_SPMD);
3908 if (ExecModeC->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD)
3909 SPMDCompatibilityTracker.indicateOptimisticFixpoint();
3910 else if (DisableOpenMPOptSPMDization || !CanChangeToSPMD)
3911 // This is a generic region but SPMDization is disabled so stop
3912 // tracking.
3913 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
3914 else
3915 setExecModeOfKernelEnvironment(AssumedExecModeC);
3916
3917 const Triple T(Fn->getParent()->getTargetTriple());
3918 auto *Int32Ty = Type::getInt32Ty(C&: Fn->getContext());
3919 auto [MinThreads, MaxThreads] =
3920 OpenMPIRBuilder::readThreadBoundsForKernel(T, Kernel&: *Fn);
3921 if (MinThreads)
3922 setMinThreadsOfKernelEnvironment(ConstantInt::get(Ty: Int32Ty, V: MinThreads));
3923 if (MaxThreads)
3924 setMaxThreadsOfKernelEnvironment(ConstantInt::get(Ty: Int32Ty, V: MaxThreads));
3925 auto [MinTeams, MaxTeams] =
3926 OpenMPIRBuilder::readTeamBoundsForKernel(T, Kernel&: *Fn);
3927 if (MinTeams)
3928 setMinTeamsOfKernelEnvironment(ConstantInt::get(Ty: Int32Ty, V: MinTeams));
3929 if (MaxTeams)
3930 setMaxTeamsOfKernelEnvironment(ConstantInt::get(Ty: Int32Ty, V: MaxTeams));
3931
3932 ConstantInt *MayUseNestedParallelismC =
3933 KernelInfo::getMayUseNestedParallelismFromKernelEnvironment(KernelEnvC);
3934 ConstantInt *AssumedMayUseNestedParallelismC = ConstantInt::get(
3935 Ty: MayUseNestedParallelismC->getIntegerType(), V: NestedParallelism);
3936 setMayUseNestedParallelismOfKernelEnvironment(
3937 AssumedMayUseNestedParallelismC);
3938
3939 if (!DisableOpenMPOptStateMachineRewrite) {
3940 ConstantInt *UseGenericStateMachineC =
3941 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
3942 KernelEnvC);
3943 ConstantInt *AssumedUseGenericStateMachineC =
3944 ConstantInt::get(Ty: UseGenericStateMachineC->getIntegerType(), V: false);
3945 setUseGenericStateMachineOfKernelEnvironment(
3946 AssumedUseGenericStateMachineC);
3947 }
3948
3949 // Register virtual uses of functions we might need to preserve.
3950 auto RegisterVirtualUse = [&](RuntimeFunction RFKind,
3951 Attributor::VirtualUseCallbackTy &CB) {
3952 if (!OMPInfoCache.RFIs[RFKind].Declaration)
3953 return;
3954 A.registerVirtualUseCallback(V: *OMPInfoCache.RFIs[RFKind].Declaration, CB);
3955 };
3956
3957 // Add a dependence to ensure updates if the state changes.
3958 auto AddDependence = [](Attributor &A, const AAKernelInfo *KI,
3959 const AbstractAttribute *QueryingAA) {
3960 if (QueryingAA) {
3961 A.recordDependence(FromAA: *KI, ToAA: *QueryingAA, DepClass: DepClassTy::OPTIONAL);
3962 }
3963 return true;
3964 };
3965
3966 Attributor::VirtualUseCallbackTy CustomStateMachineUseCB =
3967 [&](Attributor &A, const AbstractAttribute *QueryingAA) {
3968 // Whenever we create a custom state machine we will insert calls to
3969 // __kmpc_get_max_team_threads,
3970 // __kmpc_barrier_simple_generic,
3971 // __kmpc_kernel_parallel, and
3972 // __kmpc_kernel_end_parallel.
3973 // Not needed if we are on track for SPMDzation.
3974 if (SPMDCompatibilityTracker.isValidState())
3975 return AddDependence(A, this, QueryingAA);
3976 // Not needed if we can't rewrite due to an invalid state.
3977 if (!ReachedKnownParallelRegions.isValidState())
3978 return AddDependence(A, this, QueryingAA);
3979 return false;
3980 };
3981
3982 // Not needed if we are pre-runtime merge.
3983 if (!KernelInitCB->getCalledFunction()->isDeclaration()) {
3984 RegisterVirtualUse(OMPRTL___kmpc_get_max_team_threads,
3985 CustomStateMachineUseCB);
3986 RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
3987 CustomStateMachineUseCB);
3988 RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
3989 CustomStateMachineUseCB);
3990 RegisterVirtualUse(OMPRTL___kmpc_kernel_end_parallel,
3991 CustomStateMachineUseCB);
3992 }
3993
3994 // If we do not perform SPMDzation we do not need the virtual uses below.
3995 if (SPMDCompatibilityTracker.isAtFixpoint())
3996 return;
3997
3998 Attributor::VirtualUseCallbackTy HWThreadIdUseCB =
3999 [&](Attributor &A, const AbstractAttribute *QueryingAA) {
4000 // Whenever we perform SPMDzation we will insert
4001 // __kmpc_get_hardware_thread_id_in_block calls.
4002 if (!SPMDCompatibilityTracker.isValidState())
4003 return AddDependence(A, this, QueryingAA);
4004 return false;
4005 };
4006 RegisterVirtualUse(OMPRTL___kmpc_get_hardware_thread_id_in_block,
4007 HWThreadIdUseCB);
4008
4009 Attributor::VirtualUseCallbackTy SPMDBarrierUseCB =
4010 [&](Attributor &A, const AbstractAttribute *QueryingAA) {
4011 // Whenever we perform SPMDzation with guarding we will insert
4012 // __kmpc_simple_barrier_spmd calls. If SPMDzation failed, there is
4013 // nothing to guard, or there are no parallel regions, we don't need
4014 // the calls.
4015 if (!SPMDCompatibilityTracker.isValidState())
4016 return AddDependence(A, this, QueryingAA);
4017 if (SPMDCompatibilityTracker.empty())
4018 return AddDependence(A, this, QueryingAA);
4019 if (!mayContainParallelRegion())
4020 return AddDependence(A, this, QueryingAA);
4021 return false;
4022 };
4023 RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_spmd, SPMDBarrierUseCB);
4024 }
4025
4026 /// Sanitize the string \p S such that it is a suitable global symbol name.
4027 static std::string sanitizeForGlobalName(std::string S) {
4028 std::replace_if(
4029 first: S.begin(), last: S.end(),
4030 pred: [](const char C) {
4031 return !((C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') ||
4032 (C >= '0' && C <= '9') || C == '_');
4033 },
4034 new_value: '.');
4035 return S;
4036 }
4037
4038 /// Modify the IR based on the KernelInfoState as the fixpoint iteration is
4039 /// finished now.
4040 ChangeStatus manifest(Attributor &A) override {
4041 // If we are not looking at a kernel with __kmpc_target_init and
4042 // __kmpc_target_deinit call we cannot actually manifest the information.
4043 if (!KernelInitCB || !KernelDeinitCB)
4044 return ChangeStatus::UNCHANGED;
4045
4046 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4047
4048 bool HasBuiltStateMachine = true;
4049 if (!changeToSPMDMode(A, Changed)) {
4050 if (!KernelInitCB->getCalledFunction()->isDeclaration())
4051 HasBuiltStateMachine = buildCustomStateMachine(A, Changed);
4052 else
4053 HasBuiltStateMachine = false;
4054 }
4055
4056 // We need to reset KernelEnvC if specific rewriting is not done.
4057 ConstantStruct *ExistingKernelEnvC =
4058 KernelInfo::getKernelEnvironementFromKernelInitCB(KernelInitCB);
4059 ConstantInt *OldUseGenericStateMachineVal =
4060 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
4061 KernelEnvC: ExistingKernelEnvC);
4062 if (!HasBuiltStateMachine)
4063 setUseGenericStateMachineOfKernelEnvironment(
4064 OldUseGenericStateMachineVal);
4065
4066 // At last, update the KernelEnvc
4067 GlobalVariable *KernelEnvGV =
4068 KernelInfo::getKernelEnvironementGVFromKernelInitCB(KernelInitCB);
4069 if (KernelEnvGV->getInitializer() != KernelEnvC) {
4070 KernelEnvGV->setInitializer(KernelEnvC);
4071 Changed = ChangeStatus::CHANGED;
4072 }
4073
4074 return Changed;
4075 }
4076
4077 void insertInstructionGuardsHelper(Attributor &A) {
4078 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4079
4080 auto CreateGuardedRegion = [&](Instruction *RegionStartI,
4081 Instruction *RegionEndI) {
4082 LoopInfo *LI = nullptr;
4083 DominatorTree *DT = nullptr;
4084 MemorySSAUpdater *MSU = nullptr;
4085 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4086
4087 BasicBlock *ParentBB = RegionStartI->getParent();
4088 Function *Fn = ParentBB->getParent();
4089 Module &M = *Fn->getParent();
4090
4091 // Create all the blocks and logic.
4092 // ParentBB:
4093 // goto RegionCheckTidBB
4094 // RegionCheckTidBB:
4095 // Tid = __kmpc_hardware_thread_id()
4096 // if (Tid != 0)
4097 // goto RegionBarrierBB
4098 // RegionStartBB:
4099 // <execute instructions guarded>
4100 // goto RegionEndBB
4101 // RegionEndBB:
4102 // <store escaping values to shared mem>
4103 // goto RegionBarrierBB
4104 // RegionBarrierBB:
4105 // __kmpc_simple_barrier_spmd()
4106 // // second barrier is omitted if lacking escaping values.
4107 // <load escaping values from shared mem>
4108 // __kmpc_simple_barrier_spmd()
4109 // goto RegionExitBB
4110 // RegionExitBB:
4111 // <execute rest of instructions>
4112
4113 BasicBlock *RegionEndBB = SplitBlock(Old: ParentBB, SplitPt: RegionEndI->getNextNode(),
4114 DT, LI, MSSAU: MSU, BBName: "region.guarded.end");
4115 BasicBlock *RegionBarrierBB =
4116 SplitBlock(Old: RegionEndBB, SplitPt: &*RegionEndBB->getFirstInsertionPt(), DT, LI,
4117 MSSAU: MSU, BBName: "region.barrier");
4118 BasicBlock *RegionExitBB =
4119 SplitBlock(Old: RegionBarrierBB, SplitPt: &*RegionBarrierBB->getFirstInsertionPt(),
4120 DT, LI, MSSAU: MSU, BBName: "region.exit");
4121 BasicBlock *RegionStartBB =
4122 SplitBlock(Old: ParentBB, SplitPt: RegionStartI, DT, LI, MSSAU: MSU, BBName: "region.guarded");
4123
4124 assert(ParentBB->getUniqueSuccessor() == RegionStartBB &&
4125 "Expected a different CFG");
4126
4127 BasicBlock *RegionCheckTidBB = SplitBlock(
4128 Old: ParentBB, SplitPt: ParentBB->getTerminator(), DT, LI, MSSAU: MSU, BBName: "region.check.tid");
4129
4130 // Register basic blocks with the Attributor.
4131 A.registerManifestAddedBasicBlock(BB&: *RegionEndBB);
4132 A.registerManifestAddedBasicBlock(BB&: *RegionBarrierBB);
4133 A.registerManifestAddedBasicBlock(BB&: *RegionExitBB);
4134 A.registerManifestAddedBasicBlock(BB&: *RegionStartBB);
4135 A.registerManifestAddedBasicBlock(BB&: *RegionCheckTidBB);
4136
4137 bool HasBroadcastValues = false;
4138 // Find escaping outputs from the guarded region to outside users and
4139 // broadcast their values to them.
4140 for (Instruction &I : *RegionStartBB) {
4141 SmallVector<Use *, 4> OutsideUses;
4142 for (Use &U : I.uses()) {
4143 Instruction &UsrI = *cast<Instruction>(Val: U.getUser());
4144 if (UsrI.getParent() != RegionStartBB)
4145 OutsideUses.push_back(Elt: &U);
4146 }
4147
4148 if (OutsideUses.empty())
4149 continue;
4150
4151 HasBroadcastValues = true;
4152
4153 // Emit a global variable in shared memory to store the broadcasted
4154 // value.
4155 auto *SharedMem = new GlobalVariable(
4156 M, I.getType(), /* IsConstant */ false,
4157 GlobalValue::InternalLinkage, UndefValue::get(T: I.getType()),
4158 sanitizeForGlobalName(
4159 S: (I.getName() + ".guarded.output.alloc").str()),
4160 nullptr, GlobalValue::NotThreadLocal,
4161 static_cast<unsigned>(AddressSpace::Shared));
4162
4163 // Emit a store instruction to update the value.
4164 new StoreInst(&I, SharedMem,
4165 RegionEndBB->getTerminator()->getIterator());
4166
4167 LoadInst *LoadI = new LoadInst(
4168 I.getType(), SharedMem, I.getName() + ".guarded.output.load",
4169 RegionBarrierBB->getTerminator()->getIterator());
4170
4171 // Emit a load instruction and replace uses of the output value.
4172 for (Use *U : OutsideUses)
4173 A.changeUseAfterManifest(U&: *U, NV&: *LoadI);
4174 }
4175
4176 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4177
4178 // Go to tid check BB in ParentBB.
4179 const DebugLoc DL = ParentBB->getTerminator()->getDebugLoc();
4180 ParentBB->getTerminator()->eraseFromParent();
4181 OpenMPIRBuilder::LocationDescription Loc(
4182 InsertPointTy(ParentBB, ParentBB->end()), DL);
4183 OMPInfoCache.OMPBuilder.updateToLocation(Loc);
4184 uint32_t SrcLocStrSize;
4185 auto *SrcLocStr =
4186 OMPInfoCache.OMPBuilder.getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4187 Value *Ident =
4188 OMPInfoCache.OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4189 UncondBrInst::Create(Target: RegionCheckTidBB, InsertBefore: ParentBB)->setDebugLoc(DL);
4190
4191 // Add check for Tid in RegionCheckTidBB
4192 RegionCheckTidBB->getTerminator()->eraseFromParent();
4193 OpenMPIRBuilder::LocationDescription LocRegionCheckTid(
4194 InsertPointTy(RegionCheckTidBB, RegionCheckTidBB->end()), DL);
4195 OMPInfoCache.OMPBuilder.updateToLocation(Loc: LocRegionCheckTid);
4196 FunctionCallee HardwareTidFn =
4197 OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4198 M, FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block);
4199 CallInst *Tid =
4200 OMPInfoCache.OMPBuilder.Builder.CreateCall(Callee: HardwareTidFn, Args: {});
4201 Tid->setDebugLoc(DL);
4202 OMPInfoCache.setCallingConvention(Callee: HardwareTidFn, CI: Tid);
4203 Value *TidCheck = OMPInfoCache.OMPBuilder.Builder.CreateIsNull(Arg: Tid);
4204 OMPInfoCache.OMPBuilder.Builder
4205 .CreateCondBr(Cond: TidCheck, True: RegionStartBB, False: RegionBarrierBB)
4206 ->setDebugLoc(DL);
4207
4208 // First barrier for synchronization, ensures main thread has updated
4209 // values.
4210 FunctionCallee BarrierFn =
4211 OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4212 M, FnID: OMPRTL___kmpc_barrier_simple_spmd);
4213 OMPInfoCache.OMPBuilder.updateToLocation(
4214 Loc: {InsertPointTy(RegionBarrierBB,
4215 RegionBarrierBB->getFirstInsertionPt()),
4216 DL});
4217 CallInst *Barrier =
4218 OMPInfoCache.OMPBuilder.Builder.CreateCall(Callee: BarrierFn, Args: {Ident, Tid});
4219 OMPInfoCache.setCallingConvention(Callee: BarrierFn, CI: Barrier);
4220
4221 // Second barrier ensures workers have read broadcast values.
4222 if (HasBroadcastValues) {
4223 CallInst *Barrier =
4224 CallInst::Create(Func: BarrierFn, Args: {Ident, Tid}, NameStr: "",
4225 InsertBefore: RegionBarrierBB->getTerminator()->getIterator());
4226 Barrier->setDebugLoc(DL);
4227 OMPInfoCache.setCallingConvention(Callee: BarrierFn, CI: Barrier);
4228 }
4229 };
4230
4231 auto &AllocSharedRFI = OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared];
4232 SmallPtrSet<BasicBlock *, 8> Visited;
4233 for (Instruction *GuardedI : SPMDCompatibilityTracker) {
4234 BasicBlock *BB = GuardedI->getParent();
4235 if (!Visited.insert(Ptr: BB).second)
4236 continue;
4237
4238 SmallVector<std::pair<Instruction *, Instruction *>> Reorders;
4239 Instruction *LastEffect = nullptr;
4240 BasicBlock::reverse_iterator IP = BB->rbegin(), IPEnd = BB->rend();
4241 while (++IP != IPEnd) {
4242 if (!IP->mayHaveSideEffects() && !IP->mayReadFromMemory())
4243 continue;
4244 Instruction *I = &*IP;
4245 if (OpenMPOpt::getCallIfRegularCall(V&: *I, RFI: &AllocSharedRFI))
4246 continue;
4247 if (!I->user_empty() || !SPMDCompatibilityTracker.contains(Elem: I)) {
4248 LastEffect = nullptr;
4249 continue;
4250 }
4251 if (LastEffect)
4252 Reorders.push_back(Elt: {I, LastEffect});
4253 LastEffect = &*IP;
4254 }
4255 for (auto &Reorder : Reorders)
4256 Reorder.first->moveBefore(InsertPos: Reorder.second->getIterator());
4257 }
4258
4259 SmallVector<std::pair<Instruction *, Instruction *>, 4> GuardedRegions;
4260
4261 for (Instruction *GuardedI : SPMDCompatibilityTracker) {
4262 BasicBlock *BB = GuardedI->getParent();
4263 auto *CalleeAA = A.lookupAAFor<AAKernelInfo>(
4264 IRP: IRPosition::function(F: *GuardedI->getFunction()), QueryingAA: nullptr,
4265 DepClass: DepClassTy::NONE);
4266 assert(CalleeAA != nullptr && "Expected Callee AAKernelInfo");
4267 auto &CalleeAAFunction = *cast<AAKernelInfoFunction>(Val: CalleeAA);
4268 // Continue if instruction is already guarded.
4269 if (CalleeAAFunction.getGuardedInstructions().contains(Ptr: GuardedI))
4270 continue;
4271
4272 Instruction *GuardedRegionStart = nullptr, *GuardedRegionEnd = nullptr;
4273 for (Instruction &I : *BB) {
4274 // If instruction I needs to be guarded update the guarded region
4275 // bounds.
4276 if (SPMDCompatibilityTracker.contains(Elem: &I)) {
4277 CalleeAAFunction.getGuardedInstructions().insert(Ptr: &I);
4278 if (GuardedRegionStart)
4279 GuardedRegionEnd = &I;
4280 else
4281 GuardedRegionStart = GuardedRegionEnd = &I;
4282
4283 continue;
4284 }
4285
4286 // Instruction I does not need guarding, store
4287 // any region found and reset bounds.
4288 if (GuardedRegionStart) {
4289 GuardedRegions.push_back(
4290 Elt: std::make_pair(x&: GuardedRegionStart, y&: GuardedRegionEnd));
4291 GuardedRegionStart = nullptr;
4292 GuardedRegionEnd = nullptr;
4293 }
4294 }
4295 }
4296
4297 for (auto &GR : GuardedRegions)
4298 CreateGuardedRegion(GR.first, GR.second);
4299 }
4300
4301 void forceSingleThreadPerWorkgroupHelper(Attributor &A) {
4302 // Only allow 1 thread per workgroup to continue executing the user code.
4303 //
4304 // InitCB = __kmpc_target_init(...)
4305 // ThreadIdInBlock = __kmpc_get_hardware_thread_id_in_block();
4306 // if (ThreadIdInBlock != 0) return;
4307 // UserCode:
4308 // // user code
4309 //
4310 auto &Ctx = getAnchorValue().getContext();
4311 Function *Kernel = getAssociatedFunction();
4312 assert(Kernel && "Expected an associated function!");
4313
4314 // Create block for user code to branch to from initial block.
4315 BasicBlock *InitBB = KernelInitCB->getParent();
4316 BasicBlock *UserCodeBB = InitBB->splitBasicBlock(
4317 I: KernelInitCB->getNextNode(), BBName: "main.thread.user_code");
4318 BasicBlock *ReturnBB =
4319 BasicBlock::Create(Context&: Ctx, Name: "exit.threads", Parent: Kernel, InsertBefore: UserCodeBB);
4320
4321 // Register blocks with attributor:
4322 A.registerManifestAddedBasicBlock(BB&: *InitBB);
4323 A.registerManifestAddedBasicBlock(BB&: *UserCodeBB);
4324 A.registerManifestAddedBasicBlock(BB&: *ReturnBB);
4325
4326 // Debug location:
4327 const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
4328 ReturnInst::Create(C&: Ctx, InsertAtEnd: ReturnBB)->setDebugLoc(DLoc);
4329 InitBB->getTerminator()->eraseFromParent();
4330
4331 // Prepare call to OMPRTL___kmpc_get_hardware_thread_id_in_block.
4332 Module &M = *Kernel->getParent();
4333 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4334 FunctionCallee ThreadIdInBlockFn =
4335 OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4336 M, FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block);
4337
4338 // Get thread ID in block.
4339 CallInst *ThreadIdInBlock =
4340 CallInst::Create(Func: ThreadIdInBlockFn, NameStr: "thread_id.in.block", InsertBefore: InitBB);
4341 OMPInfoCache.setCallingConvention(Callee: ThreadIdInBlockFn, CI: ThreadIdInBlock);
4342 ThreadIdInBlock->setDebugLoc(DLoc);
4343
4344 // Eliminate all threads in the block with ID not equal to 0:
4345 Instruction *IsMainThread =
4346 ICmpInst::Create(Op: ICmpInst::ICmp, Pred: CmpInst::ICMP_NE, S1: ThreadIdInBlock,
4347 S2: ConstantInt::get(Ty: ThreadIdInBlock->getType(), V: 0),
4348 Name: "thread.is_main", InsertBefore: InitBB);
4349 IsMainThread->setDebugLoc(DLoc);
4350 CondBrInst::Create(Cond: IsMainThread, IfTrue: ReturnBB, IfFalse: UserCodeBB, InsertBefore: InitBB);
4351 }
4352
4353 bool changeToSPMDMode(Attributor &A, ChangeStatus &Changed) {
4354 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4355
4356 if (!SPMDCompatibilityTracker.isAssumed()) {
4357 for (Instruction *NonCompatibleI : SPMDCompatibilityTracker) {
4358 if (!NonCompatibleI)
4359 continue;
4360
4361 // Skip diagnostics on calls to known OpenMP runtime functions for now.
4362 if (auto *CB = dyn_cast<CallBase>(Val: NonCompatibleI))
4363 if (OMPInfoCache.RTLFunctions.contains(V: CB->getCalledFunction()))
4364 continue;
4365
4366 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
4367 ORA << "Value has potential side effects preventing SPMD-mode "
4368 "execution";
4369 if (isa<CallBase>(Val: NonCompatibleI)) {
4370 ORA << ". Add `[[omp::assume(\"ompx_spmd_amenable\")]]` to "
4371 "the called function to override";
4372 }
4373 return ORA << ".";
4374 };
4375 A.emitRemark<OptimizationRemarkAnalysis>(I: NonCompatibleI, RemarkName: "OMP121",
4376 RemarkCB&: Remark);
4377
4378 LLVM_DEBUG(dbgs() << TAG << "SPMD-incompatible side-effect: "
4379 << *NonCompatibleI << "\n");
4380 }
4381
4382 return false;
4383 }
4384
4385 // Get the actual kernel, could be the caller of the anchor scope if we have
4386 // a debug wrapper.
4387 Function *Kernel = getAnchorScope();
4388 if (Kernel->hasLocalLinkage()) {
4389 assert(Kernel->hasOneUse() && "Unexpected use of debug kernel wrapper.");
4390 auto *CB = cast<CallBase>(Val: Kernel->user_back());
4391 Kernel = CB->getCaller();
4392 }
4393 assert(omp::isOpenMPKernel(*Kernel) && "Expected kernel function!");
4394
4395 // Check if the kernel is already in SPMD mode, if so, return success.
4396 ConstantStruct *ExistingKernelEnvC =
4397 KernelInfo::getKernelEnvironementFromKernelInitCB(KernelInitCB);
4398 auto *ExecModeC =
4399 KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC: ExistingKernelEnvC);
4400 const int8_t ExecModeVal = ExecModeC->getSExtValue();
4401 if (ExecModeVal != OMP_TGT_EXEC_MODE_GENERIC)
4402 return true;
4403
4404 // We will now unconditionally modify the IR, indicate a change.
4405 Changed = ChangeStatus::CHANGED;
4406
4407 // Do not use instruction guards when no parallel is present inside
4408 // the target region.
4409 if (mayContainParallelRegion())
4410 insertInstructionGuardsHelper(A);
4411 else
4412 forceSingleThreadPerWorkgroupHelper(A);
4413
4414 // Adjust the global exec mode flag that tells the runtime what mode this
4415 // kernel is executed in.
4416 assert(ExecModeVal == OMP_TGT_EXEC_MODE_GENERIC &&
4417 "Initially non-SPMD kernel has SPMD exec mode!");
4418 setExecModeOfKernelEnvironment(
4419 ConstantInt::get(Ty: ExecModeC->getIntegerType(),
4420 V: ExecModeVal | OMP_TGT_EXEC_MODE_GENERIC_SPMD));
4421
4422 ++NumOpenMPTargetRegionKernelsSPMD;
4423
4424 // Record that this kernel now runs SPMD so post-Attributor cleanup can drop
4425 // the now-dead parallel data-sharing wrapper without re-deriving the mode.
4426 OMPInfoCache.SPMDizedKernels.insert(Ptr: Kernel);
4427
4428 auto Remark = [&](OptimizationRemark OR) {
4429 return OR << "Transformed generic-mode kernel to SPMD-mode.";
4430 };
4431 A.emitRemark<OptimizationRemark>(I: KernelInitCB, RemarkName: "OMP120", RemarkCB&: Remark);
4432 return true;
4433 };
4434
4435 bool buildCustomStateMachine(Attributor &A, ChangeStatus &Changed) {
4436 // If we have disabled state machine rewrites, don't make a custom one
4437 if (DisableOpenMPOptStateMachineRewrite)
4438 return false;
4439
4440 // Don't rewrite the state machine if we are not in a valid state.
4441 if (!ReachedKnownParallelRegions.isValidState())
4442 return false;
4443
4444 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4445 if (!OMPInfoCache.runtimeFnsAvailable(Fns: {OMPRTL___kmpc_get_max_team_threads,
4446 OMPRTL___kmpc_barrier_simple_generic,
4447 OMPRTL___kmpc_kernel_parallel,
4448 OMPRTL___kmpc_kernel_end_parallel}))
4449 return false;
4450
4451 ConstantStruct *ExistingKernelEnvC =
4452 KernelInfo::getKernelEnvironementFromKernelInitCB(KernelInitCB);
4453
4454 // Check if the current configuration is non-SPMD and generic state machine.
4455 // If we already have SPMD mode or a custom state machine we do not need to
4456 // go any further. If it is anything but a constant something is weird and
4457 // we give up.
4458 ConstantInt *UseStateMachineC =
4459 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
4460 KernelEnvC: ExistingKernelEnvC);
4461 ConstantInt *ModeC =
4462 KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC: ExistingKernelEnvC);
4463
4464 // If we are stuck with generic mode, try to create a custom device (=GPU)
4465 // state machine which is specialized for the parallel regions that are
4466 // reachable by the kernel.
4467 if (UseStateMachineC->isZero() ||
4468 (ModeC->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD))
4469 return false;
4470
4471 Changed = ChangeStatus::CHANGED;
4472
4473 // If not SPMD mode, indicate we use a custom state machine now.
4474 setUseGenericStateMachineOfKernelEnvironment(
4475 ConstantInt::get(Ty: UseStateMachineC->getIntegerType(), V: false));
4476
4477 // If we don't actually need a state machine we are done here. This can
4478 // happen if there simply are no parallel regions. In the resulting kernel
4479 // all worker threads will simply exit right away, leaving the main thread
4480 // to do the work alone.
4481 if (!mayContainParallelRegion()) {
4482 ++NumOpenMPTargetRegionKernelsWithoutStateMachine;
4483
4484 auto Remark = [&](OptimizationRemark OR) {
4485 return OR << "Removing unused state machine from generic-mode kernel.";
4486 };
4487 A.emitRemark<OptimizationRemark>(I: KernelInitCB, RemarkName: "OMP130", RemarkCB&: Remark);
4488
4489 return true;
4490 }
4491
4492 // Keep track in the statistics of our new shiny custom state machine.
4493 if (ReachedUnknownParallelRegions.empty()) {
4494 ++NumOpenMPTargetRegionKernelsCustomStateMachineWithoutFallback;
4495
4496 auto Remark = [&](OptimizationRemark OR) {
4497 return OR << "Rewriting generic-mode kernel with a customized state "
4498 "machine.";
4499 };
4500 A.emitRemark<OptimizationRemark>(I: KernelInitCB, RemarkName: "OMP131", RemarkCB&: Remark);
4501 } else {
4502 ++NumOpenMPTargetRegionKernelsCustomStateMachineWithFallback;
4503
4504 auto Remark = [&](OptimizationRemarkAnalysis OR) {
4505 return OR << "Generic-mode kernel is executed with a customized state "
4506 "machine that requires a fallback.";
4507 };
4508 A.emitRemark<OptimizationRemarkAnalysis>(I: KernelInitCB, RemarkName: "OMP132", RemarkCB&: Remark);
4509
4510 // Tell the user why we ended up with a fallback.
4511 for (CallBase *UnknownParallelRegionCB : ReachedUnknownParallelRegions) {
4512 if (!UnknownParallelRegionCB)
4513 continue;
4514 auto Remark = [&](OptimizationRemarkAnalysis ORA) {
4515 return ORA << "Call may contain unknown parallel regions. Use "
4516 << "`[[omp::assume(\"omp_no_parallelism\")]]` to "
4517 "override.";
4518 };
4519 A.emitRemark<OptimizationRemarkAnalysis>(I: UnknownParallelRegionCB,
4520 RemarkName: "OMP133", RemarkCB&: Remark);
4521 }
4522 }
4523
4524 // Create all the blocks:
4525 //
4526 // InitCB = __kmpc_target_init(...)
4527 // MaxTeamThreads =
4528 // __kmpc_get_max_team_threads(/*IsSPMD=*/false);
4529 // IsWorkerCheckBB: bool IsWorker = InitCB != -1;
4530 // if (IsWorker) {
4531 // if (InitCB >= MaxTeamThreads) return;
4532 // SMBeginBB: __kmpc_barrier_simple_generic(...);
4533 // void *WorkFn;
4534 // bool Active = __kmpc_kernel_parallel(&WorkFn);
4535 // if (!WorkFn) return;
4536 // SMIsActiveCheckBB: if (Active) {
4537 // SMIfCascadeCurrentBB: if (WorkFn == <ParFn0>)
4538 // ParFn0(...);
4539 // SMIfCascadeCurrentBB: else if (WorkFn == <ParFn1>)
4540 // ParFn1(...);
4541 // ...
4542 // SMIfCascadeCurrentBB: else
4543 // ((WorkFnTy*)WorkFn)(...);
4544 // SMEndParallelBB: __kmpc_kernel_end_parallel(...);
4545 // }
4546 // SMDoneBB: __kmpc_barrier_simple_generic(...);
4547 // goto SMBeginBB;
4548 // }
4549 // UserCodeEntryBB: // user code
4550 // __kmpc_target_deinit(...)
4551 //
4552 auto &Ctx = getAnchorValue().getContext();
4553 Function *Kernel = getAssociatedFunction();
4554 assert(Kernel && "Expected an associated function!");
4555
4556 BasicBlock *InitBB = KernelInitCB->getParent();
4557 BasicBlock *UserCodeEntryBB = InitBB->splitBasicBlock(
4558 I: KernelInitCB->getNextNode(), BBName: "thread.user_code.check");
4559 BasicBlock *IsWorkerCheckBB =
4560 BasicBlock::Create(Context&: Ctx, Name: "is_worker_check", Parent: Kernel, InsertBefore: UserCodeEntryBB);
4561 BasicBlock *StateMachineBeginBB = BasicBlock::Create(
4562 Context&: Ctx, Name: "worker_state_machine.begin", Parent: Kernel, InsertBefore: UserCodeEntryBB);
4563 BasicBlock *StateMachineFinishedBB = BasicBlock::Create(
4564 Context&: Ctx, Name: "worker_state_machine.finished", Parent: Kernel, InsertBefore: UserCodeEntryBB);
4565 BasicBlock *StateMachineIsActiveCheckBB = BasicBlock::Create(
4566 Context&: Ctx, Name: "worker_state_machine.is_active.check", Parent: Kernel, InsertBefore: UserCodeEntryBB);
4567 BasicBlock *StateMachineIfCascadeCurrentBB =
4568 BasicBlock::Create(Context&: Ctx, Name: "worker_state_machine.parallel_region.check",
4569 Parent: Kernel, InsertBefore: UserCodeEntryBB);
4570 BasicBlock *StateMachineEndParallelBB =
4571 BasicBlock::Create(Context&: Ctx, Name: "worker_state_machine.parallel_region.end",
4572 Parent: Kernel, InsertBefore: UserCodeEntryBB);
4573 BasicBlock *StateMachineDoneBarrierBB = BasicBlock::Create(
4574 Context&: Ctx, Name: "worker_state_machine.done.barrier", Parent: Kernel, InsertBefore: UserCodeEntryBB);
4575 A.registerManifestAddedBasicBlock(BB&: *InitBB);
4576 A.registerManifestAddedBasicBlock(BB&: *UserCodeEntryBB);
4577 A.registerManifestAddedBasicBlock(BB&: *IsWorkerCheckBB);
4578 A.registerManifestAddedBasicBlock(BB&: *StateMachineBeginBB);
4579 A.registerManifestAddedBasicBlock(BB&: *StateMachineFinishedBB);
4580 A.registerManifestAddedBasicBlock(BB&: *StateMachineIsActiveCheckBB);
4581 A.registerManifestAddedBasicBlock(BB&: *StateMachineIfCascadeCurrentBB);
4582 A.registerManifestAddedBasicBlock(BB&: *StateMachineEndParallelBB);
4583 A.registerManifestAddedBasicBlock(BB&: *StateMachineDoneBarrierBB);
4584
4585 const DebugLoc &DLoc = KernelInitCB->getDebugLoc();
4586 ReturnInst::Create(C&: Ctx, InsertAtEnd: StateMachineFinishedBB)->setDebugLoc(DLoc);
4587 InitBB->getTerminator()->eraseFromParent();
4588
4589 Instruction *IsWorker =
4590 ICmpInst::Create(Op: ICmpInst::ICmp, Pred: llvm::CmpInst::ICMP_NE, S1: KernelInitCB,
4591 S2: ConstantInt::getAllOnesValue(Ty: KernelInitCB->getType()),
4592 Name: "thread.is_worker", InsertBefore: InitBB);
4593 IsWorker->setDebugLoc(DLoc);
4594 CondBrInst::Create(Cond: IsWorker, IfTrue: IsWorkerCheckBB, IfFalse: UserCodeEntryBB, InsertBefore: InitBB);
4595
4596 // How much of the block the main thread takes is the runtime's to know, so
4597 // ask it rather than subtracting a warp here. The mode is passed in because
4598 // this runs before the barrier that would make the shared one visible; it
4599 // is a constant, a custom state machine being built only for generic mode.
4600 Module &M = *Kernel->getParent();
4601 FunctionCallee MaxTeamThreadsFn =
4602 OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4603 M, FnID: OMPRTL___kmpc_get_max_team_threads);
4604 Constant *IsSPMDArg = ConstantInt::get(Ty: OMPInfoCache.OMPBuilder.Int32, V: 0);
4605 CallInst *MaxTeamThreads = CallInst::Create(
4606 Func: MaxTeamThreadsFn, Args: {IsSPMDArg}, NameStr: "max_team_threads", InsertBefore: IsWorkerCheckBB);
4607 OMPInfoCache.setCallingConvention(Callee: MaxTeamThreadsFn, CI: MaxTeamThreads);
4608 MaxTeamThreads->setDebugLoc(DLoc);
4609 Instruction *IsMainOrWorker = ICmpInst::Create(
4610 Op: ICmpInst::ICmp, Pred: llvm::CmpInst::ICMP_SLT, S1: KernelInitCB, S2: MaxTeamThreads,
4611 Name: "thread.is_main_or_worker", InsertBefore: IsWorkerCheckBB);
4612 IsMainOrWorker->setDebugLoc(DLoc);
4613 CondBrInst::Create(Cond: IsMainOrWorker, IfTrue: StateMachineBeginBB,
4614 IfFalse: StateMachineFinishedBB, InsertBefore: IsWorkerCheckBB);
4615
4616 // Create local storage for the work function pointer.
4617 const DataLayout &DL = M.getDataLayout();
4618 Type *VoidPtrTy = PointerType::getUnqual(C&: Ctx);
4619 Instruction *WorkFnAI =
4620 new AllocaInst(VoidPtrTy, DL.getAllocaAddrSpace(), nullptr,
4621 "worker.work_fn.addr", Kernel->getEntryBlock().begin());
4622 WorkFnAI->setDebugLoc(DLoc);
4623
4624 OMPInfoCache.OMPBuilder.updateToLocation(
4625 Loc: OpenMPIRBuilder::LocationDescription(
4626 IRBuilder<>::InsertPoint(StateMachineBeginBB,
4627 StateMachineBeginBB->end()),
4628 DLoc));
4629
4630 Value *Ident = KernelInfo::getIdentFromKernelEnvironment(KernelEnvC);
4631 Value *GTid = KernelInitCB;
4632
4633 FunctionCallee BarrierFn =
4634 OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4635 M, FnID: OMPRTL___kmpc_barrier_simple_generic);
4636 CallInst *Barrier =
4637 CallInst::Create(Func: BarrierFn, Args: {Ident, GTid}, NameStr: "", InsertBefore: StateMachineBeginBB);
4638 OMPInfoCache.setCallingConvention(Callee: BarrierFn, CI: Barrier);
4639 Barrier->setDebugLoc(DLoc);
4640
4641 if (WorkFnAI->getType()->getPointerAddressSpace() !=
4642 (unsigned int)AddressSpace::Generic) {
4643 WorkFnAI = new AddrSpaceCastInst(
4644 WorkFnAI, PointerType::get(C&: Ctx, AddressSpace: (unsigned int)AddressSpace::Generic),
4645 WorkFnAI->getName() + ".generic", StateMachineBeginBB);
4646 WorkFnAI->setDebugLoc(DLoc);
4647 }
4648
4649 FunctionCallee KernelParallelFn =
4650 OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4651 M, FnID: OMPRTL___kmpc_kernel_parallel);
4652 CallInst *IsActiveWorker = CallInst::Create(
4653 Func: KernelParallelFn, Args: {WorkFnAI}, NameStr: "worker.is_active", InsertBefore: StateMachineBeginBB);
4654 OMPInfoCache.setCallingConvention(Callee: KernelParallelFn, CI: IsActiveWorker);
4655 IsActiveWorker->setDebugLoc(DLoc);
4656 Instruction *WorkFn = new LoadInst(VoidPtrTy, WorkFnAI, "worker.work_fn",
4657 StateMachineBeginBB);
4658 WorkFn->setDebugLoc(DLoc);
4659
4660 FunctionType *ParallelRegionFnTy = FunctionType::get(
4661 Result: Type::getVoidTy(C&: Ctx), Params: {Type::getInt16Ty(C&: Ctx), Type::getInt32Ty(C&: Ctx)},
4662 isVarArg: false);
4663
4664 Instruction *IsDone =
4665 ICmpInst::Create(Op: ICmpInst::ICmp, Pred: llvm::CmpInst::ICMP_EQ, S1: WorkFn,
4666 S2: Constant::getNullValue(Ty: VoidPtrTy), Name: "worker.is_done",
4667 InsertBefore: StateMachineBeginBB);
4668 IsDone->setDebugLoc(DLoc);
4669 CondBrInst::Create(Cond: IsDone, IfTrue: StateMachineFinishedBB,
4670 IfFalse: StateMachineIsActiveCheckBB, InsertBefore: StateMachineBeginBB)
4671 ->setDebugLoc(DLoc);
4672
4673 CondBrInst::Create(Cond: IsActiveWorker, IfTrue: StateMachineIfCascadeCurrentBB,
4674 IfFalse: StateMachineDoneBarrierBB, InsertBefore: StateMachineIsActiveCheckBB)
4675 ->setDebugLoc(DLoc);
4676
4677 Value *ZeroArg =
4678 Constant::getNullValue(Ty: ParallelRegionFnTy->getParamType(i: 0));
4679
4680 const unsigned int WrapperFunctionArgNo = 6;
4681
4682 // Now that we have most of the CFG skeleton it is time for the if-cascade
4683 // that checks the function pointer we got from the runtime against the
4684 // parallel regions we expect, if there are any.
4685 for (int I = 0, E = ReachedKnownParallelRegions.size(); I < E; ++I) {
4686 auto *CB = ReachedKnownParallelRegions[I];
4687 auto *ParallelRegion = dyn_cast<Function>(
4688 Val: CB->getArgOperand(i: WrapperFunctionArgNo)->stripPointerCasts());
4689 BasicBlock *PRExecuteBB = BasicBlock::Create(
4690 Context&: Ctx, Name: "worker_state_machine.parallel_region.execute", Parent: Kernel,
4691 InsertBefore: StateMachineEndParallelBB);
4692 CallInst::Create(Func: ParallelRegion, Args: {ZeroArg, GTid}, NameStr: "", InsertBefore: PRExecuteBB)
4693 ->setDebugLoc(DLoc);
4694 UncondBrInst::Create(Target: StateMachineEndParallelBB, InsertBefore: PRExecuteBB)
4695 ->setDebugLoc(DLoc);
4696
4697 BasicBlock *PRNextBB =
4698 BasicBlock::Create(Context&: Ctx, Name: "worker_state_machine.parallel_region.check",
4699 Parent: Kernel, InsertBefore: StateMachineEndParallelBB);
4700 A.registerManifestAddedBasicBlock(BB&: *PRExecuteBB);
4701 A.registerManifestAddedBasicBlock(BB&: *PRNextBB);
4702
4703 // Check if we need to compare the pointer at all or if we can just
4704 // call the parallel region function.
4705 Value *IsPR;
4706 if (I + 1 < E || !ReachedUnknownParallelRegions.empty()) {
4707 Instruction *CmpI = ICmpInst::Create(
4708 Op: ICmpInst::ICmp, Pred: llvm::CmpInst::ICMP_EQ, S1: WorkFn, S2: ParallelRegion,
4709 Name: "worker.check_parallel_region", InsertBefore: StateMachineIfCascadeCurrentBB);
4710 CmpI->setDebugLoc(DLoc);
4711 IsPR = CmpI;
4712 } else {
4713 IsPR = ConstantInt::getTrue(Context&: Ctx);
4714 }
4715
4716 CondBrInst::Create(Cond: IsPR, IfTrue: PRExecuteBB, IfFalse: PRNextBB,
4717 InsertBefore: StateMachineIfCascadeCurrentBB)
4718 ->setDebugLoc(DLoc);
4719 StateMachineIfCascadeCurrentBB = PRNextBB;
4720 }
4721
4722 // At the end of the if-cascade we place the indirect function pointer call
4723 // in case we might need it, that is if there can be parallel regions we
4724 // have not handled in the if-cascade above.
4725 if (!ReachedUnknownParallelRegions.empty()) {
4726 StateMachineIfCascadeCurrentBB->setName(
4727 "worker_state_machine.parallel_region.fallback.execute");
4728 CallInst::Create(Ty: ParallelRegionFnTy, Func: WorkFn, Args: {ZeroArg, GTid}, NameStr: "",
4729 InsertBefore: StateMachineIfCascadeCurrentBB)
4730 ->setDebugLoc(DLoc);
4731 }
4732 UncondBrInst::Create(Target: StateMachineEndParallelBB,
4733 InsertBefore: StateMachineIfCascadeCurrentBB)
4734 ->setDebugLoc(DLoc);
4735
4736 FunctionCallee EndParallelFn =
4737 OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
4738 M, FnID: OMPRTL___kmpc_kernel_end_parallel);
4739 CallInst *EndParallel =
4740 CallInst::Create(Func: EndParallelFn, Args: {}, NameStr: "", InsertBefore: StateMachineEndParallelBB);
4741 OMPInfoCache.setCallingConvention(Callee: EndParallelFn, CI: EndParallel);
4742 EndParallel->setDebugLoc(DLoc);
4743 UncondBrInst::Create(Target: StateMachineDoneBarrierBB, InsertBefore: StateMachineEndParallelBB)
4744 ->setDebugLoc(DLoc);
4745
4746 CallInst::Create(Func: BarrierFn, Args: {Ident, GTid}, NameStr: "", InsertBefore: StateMachineDoneBarrierBB)
4747 ->setDebugLoc(DLoc);
4748 UncondBrInst::Create(Target: StateMachineBeginBB, InsertBefore: StateMachineDoneBarrierBB)
4749 ->setDebugLoc(DLoc);
4750
4751 return true;
4752 }
4753
4754 /// Fixpoint iteration update function. Will be called every time a dependence
4755 /// changed its state (and in the beginning).
4756 ChangeStatus updateImpl(Attributor &A) override {
4757 KernelInfoState StateBefore = getState();
4758
4759 // When we leave this function this RAII will make sure the member
4760 // KernelEnvC is updated properly depending on the state. That member is
4761 // used for simplification of values and needs to be up to date at all
4762 // times.
4763 struct UpdateKernelEnvCRAII {
4764 AAKernelInfoFunction &AA;
4765
4766 UpdateKernelEnvCRAII(AAKernelInfoFunction &AA) : AA(AA) {}
4767
4768 ~UpdateKernelEnvCRAII() {
4769 if (!AA.KernelEnvC)
4770 return;
4771
4772 ConstantStruct *ExistingKernelEnvC =
4773 KernelInfo::getKernelEnvironementFromKernelInitCB(KernelInitCB: AA.KernelInitCB);
4774
4775 if (!AA.isValidState()) {
4776 AA.KernelEnvC = ExistingKernelEnvC;
4777 return;
4778 }
4779
4780 if (!AA.ReachedKnownParallelRegions.isValidState())
4781 AA.setUseGenericStateMachineOfKernelEnvironment(
4782 KernelInfo::getUseGenericStateMachineFromKernelEnvironment(
4783 KernelEnvC: ExistingKernelEnvC));
4784
4785 if (!AA.SPMDCompatibilityTracker.isValidState())
4786 AA.setExecModeOfKernelEnvironment(
4787 KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC: ExistingKernelEnvC));
4788
4789 ConstantInt *MayUseNestedParallelismC =
4790 KernelInfo::getMayUseNestedParallelismFromKernelEnvironment(
4791 KernelEnvC: AA.KernelEnvC);
4792 ConstantInt *NewMayUseNestedParallelismC = ConstantInt::get(
4793 Ty: MayUseNestedParallelismC->getIntegerType(), V: AA.NestedParallelism);
4794 AA.setMayUseNestedParallelismOfKernelEnvironment(
4795 NewMayUseNestedParallelismC);
4796 }
4797 } RAII(*this);
4798
4799 // Callback to check a read/write instruction.
4800 auto CheckRWInst = [&](Instruction &I) {
4801 // We handle calls later.
4802 if (isa<CallBase>(Val: I))
4803 return true;
4804 // We only care about write effects.
4805 if (!I.mayWriteToMemory())
4806 return true;
4807 if (auto *SI = dyn_cast<StoreInst>(Val: &I)) {
4808 const auto *UnderlyingObjsAA = A.getAAFor<AAUnderlyingObjects>(
4809 QueryingAA: *this, IRP: IRPosition::value(V: *SI->getPointerOperand()),
4810 DepClass: DepClassTy::OPTIONAL);
4811 auto *HS = A.getAAFor<AAHeapToStack>(
4812 QueryingAA: *this, IRP: IRPosition::function(F: *I.getFunction()),
4813 DepClass: DepClassTy::OPTIONAL);
4814 if (UnderlyingObjsAA &&
4815 UnderlyingObjsAA->forallUnderlyingObjects(Pred: [&](Value &Obj) {
4816 if (AA::isAssumedThreadLocalObject(A, Obj, QueryingAA: *this))
4817 return true;
4818 // Check for AAHeapToStack moved objects which must not be
4819 // guarded.
4820 auto *CB = dyn_cast<CallBase>(Val: &Obj);
4821 return CB && HS && HS->isAssumedHeapToStack(CB: *CB);
4822 }))
4823 return true;
4824 }
4825
4826 // Insert instruction that needs guarding.
4827 SPMDCompatibilityTracker.insert(Elem: &I);
4828 return true;
4829 };
4830
4831 bool UsedAssumedInformationInCheckRWInst = false;
4832 if (!SPMDCompatibilityTracker.isAtFixpoint())
4833 if (!A.checkForAllReadWriteInstructions(
4834 Pred: CheckRWInst, QueryingAA&: *this, UsedAssumedInformation&: UsedAssumedInformationInCheckRWInst))
4835 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4836
4837 bool UsedAssumedInformationFromReachingKernels = false;
4838 if (!IsKernelEntry) {
4839 updateParallelLevels(A);
4840
4841 bool AllReachingKernelsKnown = true;
4842 updateReachingKernelEntries(A, AllReachingKernelsKnown);
4843 UsedAssumedInformationFromReachingKernels = !AllReachingKernelsKnown;
4844
4845 if (!SPMDCompatibilityTracker.empty()) {
4846 if (!ParallelLevels.isValidState())
4847 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4848 else if (!ReachingKernelEntries.isValidState())
4849 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4850 else {
4851 // Check if all reaching kernels agree on the mode as we can otherwise
4852 // not guard instructions. We might not be sure about the mode so we
4853 // we cannot fix the internal spmd-zation state either.
4854 int SPMD = 0, Generic = 0;
4855 for (auto *Kernel : ReachingKernelEntries) {
4856 auto *CBAA = A.getAAFor<AAKernelInfo>(
4857 QueryingAA: *this, IRP: IRPosition::function(F: *Kernel), DepClass: DepClassTy::OPTIONAL);
4858 if (CBAA && CBAA->SPMDCompatibilityTracker.isValidState() &&
4859 CBAA->SPMDCompatibilityTracker.isAssumed())
4860 ++SPMD;
4861 else
4862 ++Generic;
4863 if (!CBAA || !CBAA->SPMDCompatibilityTracker.isAtFixpoint())
4864 UsedAssumedInformationFromReachingKernels = true;
4865 }
4866 if (SPMD != 0 && Generic != 0)
4867 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
4868 }
4869 }
4870 }
4871
4872 // Callback to check a call instruction.
4873 bool AllParallelRegionStatesWereFixed = true;
4874 bool AllSPMDStatesWereFixed = true;
4875 auto CheckCallInst = [&](Instruction &I) {
4876 auto &CB = cast<CallBase>(Val&: I);
4877 // A runtime function that takes a callback runs the user's code inside
4878 // it, so whatever the callback reaches this kernel reaches too. Fold the
4879 // callback's state in; without this the call tells us nothing about the
4880 // parallel regions on the other side of it.
4881 if (Function *Callback = OMPInformationCache::getAnalyzableCallback(CB)) {
4882 LLVM_DEBUG(dbgs() << TAG << "folding in callback "
4883 << Callback->getName() << " of " << CB << "\n");
4884 if (auto *CallbackAA = A.getAAFor<AAKernelInfo>(
4885 QueryingAA: *this, IRP: IRPosition::function(F: *Callback), DepClass: DepClassTy::OPTIONAL)) {
4886 getState() ^= CallbackAA->getState();
4887 AllSPMDStatesWereFixed &=
4888 CallbackAA->SPMDCompatibilityTracker.isAtFixpoint();
4889 AllParallelRegionStatesWereFixed &=
4890 CallbackAA->ReachedKnownParallelRegions.isAtFixpoint();
4891 AllParallelRegionStatesWereFixed &=
4892 CallbackAA->ReachedUnknownParallelRegions.isAtFixpoint();
4893 }
4894 }
4895 auto *CBAA = A.getAAFor<AAKernelInfo>(
4896 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL);
4897 if (!CBAA)
4898 return false;
4899 getState() ^= CBAA->getState();
4900 AllSPMDStatesWereFixed &= CBAA->SPMDCompatibilityTracker.isAtFixpoint();
4901 AllParallelRegionStatesWereFixed &=
4902 CBAA->ReachedKnownParallelRegions.isAtFixpoint();
4903 AllParallelRegionStatesWereFixed &=
4904 CBAA->ReachedUnknownParallelRegions.isAtFixpoint();
4905 return true;
4906 };
4907
4908 bool UsedAssumedInformationInCheckCallInst = false;
4909 if (!A.checkForAllCallLikeInstructions(
4910 Pred: CheckCallInst, QueryingAA: *this, UsedAssumedInformation&: UsedAssumedInformationInCheckCallInst)) {
4911 LLVM_DEBUG(dbgs() << TAG
4912 << "Failed to visit all call-like instructions!\n";);
4913 return indicatePessimisticFixpoint();
4914 }
4915
4916 // If we haven't used any assumed information for the reached parallel
4917 // region states we can fix it.
4918 if (!UsedAssumedInformationInCheckCallInst &&
4919 AllParallelRegionStatesWereFixed) {
4920 ReachedKnownParallelRegions.indicateOptimisticFixpoint();
4921 ReachedUnknownParallelRegions.indicateOptimisticFixpoint();
4922 }
4923
4924 // If we haven't used any assumed information for the SPMD state we can fix
4925 // it.
4926 if (!UsedAssumedInformationInCheckRWInst &&
4927 !UsedAssumedInformationInCheckCallInst &&
4928 !UsedAssumedInformationFromReachingKernels && AllSPMDStatesWereFixed)
4929 SPMDCompatibilityTracker.indicateOptimisticFixpoint();
4930
4931 return StateBefore == getState() ? ChangeStatus::UNCHANGED
4932 : ChangeStatus::CHANGED;
4933 }
4934
4935private:
4936 /// Update info regarding reaching kernels.
4937 void updateReachingKernelEntries(Attributor &A,
4938 bool &AllReachingKernelsKnown) {
4939 auto PredCallSite = [&](AbstractCallSite ACS) {
4940 Function *Caller = ACS.getInstruction()->getFunction();
4941
4942 assert(Caller && "Caller is nullptr");
4943
4944 auto *CAA = A.getOrCreateAAFor<AAKernelInfo>(
4945 IRP: IRPosition::function(F: *Caller), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
4946 if (CAA && CAA->ReachingKernelEntries.isValidState()) {
4947 ReachingKernelEntries ^= CAA->ReachingKernelEntries;
4948 return true;
4949 }
4950
4951 // We lost track of the caller of the associated function, any kernel
4952 // could reach now.
4953 ReachingKernelEntries.indicatePessimisticFixpoint();
4954
4955 return true;
4956 };
4957
4958 if (!A.checkForAllCallSites(Pred: PredCallSite, QueryingAA: *this,
4959 RequireAllCallSites: true /* RequireAllCallSites */,
4960 UsedAssumedInformation&: AllReachingKernelsKnown))
4961 ReachingKernelEntries.indicatePessimisticFixpoint();
4962 }
4963
4964 /// Update info regarding parallel levels.
4965 void updateParallelLevels(Attributor &A) {
4966 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
4967 OMPInformationCache::RuntimeFunctionInfo &Parallel60RFI =
4968 OMPInfoCache.RFIs[OMPRTL___kmpc_parallel_60];
4969
4970 auto PredCallSite = [&](AbstractCallSite ACS) {
4971 Function *Caller = ACS.getInstruction()->getFunction();
4972
4973 assert(Caller && "Caller is nullptr");
4974
4975 auto *CAA =
4976 A.getOrCreateAAFor<AAKernelInfo>(IRP: IRPosition::function(F: *Caller));
4977 if (CAA && CAA->ParallelLevels.isValidState()) {
4978 // Any function that is called by `__kmpc_parallel_60` will not be
4979 // folded as the parallel level in the function is updated. In order to
4980 // get it right, all the analysis would depend on the implentation. That
4981 // said, if in the future any change to the implementation, the analysis
4982 // could be wrong. As a consequence, we are just conservative here.
4983 if (Caller == Parallel60RFI.Declaration) {
4984 ParallelLevels.indicatePessimisticFixpoint();
4985 return true;
4986 }
4987
4988 ParallelLevels ^= CAA->ParallelLevels;
4989
4990 return true;
4991 }
4992
4993 // We lost track of the caller of the associated function, any kernel
4994 // could reach now.
4995 ParallelLevels.indicatePessimisticFixpoint();
4996
4997 return true;
4998 };
4999
5000 bool AllCallSitesKnown = true;
5001 if (!A.checkForAllCallSites(Pred: PredCallSite, QueryingAA: *this,
5002 RequireAllCallSites: true /* RequireAllCallSites */,
5003 UsedAssumedInformation&: AllCallSitesKnown))
5004 ParallelLevels.indicatePessimisticFixpoint();
5005 }
5006};
5007
5008/// The call site kernel info abstract attribute, basically, what can we say
5009/// about a call site with regards to the KernelInfoState. For now this simply
5010/// forwards the information from the callee.
5011struct AAKernelInfoCallSite : AAKernelInfo {
5012 AAKernelInfoCallSite(const IRPosition &IRP, Attributor &A)
5013 : AAKernelInfo(IRP, A) {}
5014
5015 /// See AbstractAttribute::initialize(...).
5016 void initialize(Attributor &A) override {
5017 AAKernelInfo::initialize(A);
5018
5019 CallBase &CB = cast<CallBase>(Val&: getAssociatedValue());
5020 auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
5021 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL);
5022
5023 // Check for SPMD-mode assumptions.
5024 if (AssumptionAA && AssumptionAA->hasAssumption(Assumption: "ompx_spmd_amenable")) {
5025 indicateOptimisticFixpoint();
5026 return;
5027 }
5028
5029 // First weed out calls we do not care about, that is readonly/readnone
5030 // calls, intrinsics, and "no_openmp" calls. Neither of these can reach a
5031 // parallel region or anything else we are looking for.
5032 if (!CB.mayWriteToMemory() || isa<IntrinsicInst>(Val: CB)) {
5033 indicateOptimisticFixpoint();
5034 return;
5035 }
5036
5037 // Next we check if we know the callee. If it is a known OpenMP function
5038 // we will handle them explicitly in the switch below. If it is not, we
5039 // will use an AAKernelInfo object on the callee to gather information and
5040 // merge that into the current state. The latter happens in the updateImpl.
5041 auto CheckCallee = [&](Function *Callee, unsigned NumCallees) {
5042 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
5043 const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Val: Callee);
5044 if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
5045 // Unknown caller or declarations are not analyzable, we give up.
5046 if (!Callee || !A.isFunctionIPOAmendable(F: *Callee)) {
5047
5048 // Unknown callees might contain parallel regions, except if they have
5049 // an appropriate assumption attached.
5050 if (!AssumptionAA ||
5051 !(AssumptionAA->hasAssumption(Assumption: "omp_no_openmp") ||
5052 AssumptionAA->hasAssumption(Assumption: "omp_no_parallelism")))
5053 ReachedUnknownParallelRegions.insert(Elem: &CB);
5054
5055 // If SPMDCompatibilityTracker is not fixed, we need to give up on the
5056 // idea we can run something unknown in SPMD-mode.
5057 if (!SPMDCompatibilityTracker.isAtFixpoint()) {
5058 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5059 SPMDCompatibilityTracker.insert(Elem: &CB);
5060 }
5061
5062 // We have updated the state for this unknown call properly, there
5063 // won't be any change so we indicate a fixpoint.
5064 indicateOptimisticFixpoint();
5065 }
5066 // If the callee is known and can be used in IPO, we will update the
5067 // state based on the callee state in updateImpl.
5068 return;
5069 }
5070 // More than one callee normally means an indirect call we cannot resolve.
5071 // A runtime function carrying !callback is the exception: the extra edge
5072 // is the callback, which we analyze rather than give up on.
5073 if (NumCallees > 1 && !Callee->hasMetadata(KindID: LLVMContext::MD_callback)) {
5074 indicatePessimisticFixpoint();
5075 return;
5076 }
5077
5078 RuntimeFunction RF = It->getSecond();
5079 switch (RF) {
5080 // All the functions we know are compatible with SPMD mode.
5081 case OMPRTL___kmpc_is_spmd_exec_mode:
5082 case OMPRTL___kmpc_distribute_static_fini:
5083 case OMPRTL___kmpc_for_static_fini:
5084 case OMPRTL___kmpc_global_thread_num:
5085 case OMPRTL___kmpc_get_hardware_num_threads_in_block:
5086 case OMPRTL___kmpc_get_hardware_num_blocks:
5087 case OMPRTL___kmpc_single:
5088 case OMPRTL___kmpc_end_single:
5089 case OMPRTL___kmpc_master:
5090 case OMPRTL___kmpc_end_master:
5091 case OMPRTL___kmpc_barrier:
5092 case OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2:
5093 case OMPRTL___kmpc_gpu_xteam_reduce_nowait:
5094 case OMPRTL___kmpc_error:
5095 case OMPRTL___kmpc_flush:
5096 case OMPRTL___kmpc_get_hardware_thread_id_in_block:
5097 case OMPRTL___kmpc_get_warp_size:
5098 case OMPRTL_omp_get_thread_num:
5099 case OMPRTL_omp_get_num_threads:
5100 case OMPRTL_omp_get_max_threads:
5101 case OMPRTL_omp_in_parallel:
5102 case OMPRTL_omp_get_dynamic:
5103 case OMPRTL_omp_get_cancellation:
5104 case OMPRTL_omp_get_nested:
5105 case OMPRTL_omp_get_schedule:
5106 case OMPRTL_omp_get_thread_limit:
5107 case OMPRTL_omp_get_supported_active_levels:
5108 case OMPRTL_omp_get_max_active_levels:
5109 case OMPRTL_omp_get_level:
5110 case OMPRTL_omp_get_ancestor_thread_num:
5111 case OMPRTL_omp_get_team_size:
5112 case OMPRTL_omp_get_active_level:
5113 case OMPRTL_omp_in_final:
5114 case OMPRTL_omp_get_proc_bind:
5115 case OMPRTL_omp_get_num_places:
5116 case OMPRTL_omp_get_num_procs:
5117 case OMPRTL_omp_get_place_proc_ids:
5118 case OMPRTL_omp_get_place_num:
5119 case OMPRTL_omp_get_partition_num_places:
5120 case OMPRTL_omp_get_partition_place_nums:
5121 case OMPRTL_omp_get_wtime:
5122 break;
5123 case OMPRTL___kmpc_distribute_static_init_4:
5124 case OMPRTL___kmpc_distribute_static_init_4u:
5125 case OMPRTL___kmpc_distribute_static_init_8:
5126 case OMPRTL___kmpc_distribute_static_init_8u:
5127 case OMPRTL___kmpc_for_static_init_4:
5128 case OMPRTL___kmpc_for_static_init_4u:
5129 case OMPRTL___kmpc_for_static_init_8:
5130 case OMPRTL___kmpc_for_static_init_8u: {
5131 // Check the schedule and allow static schedule in SPMD mode.
5132 unsigned ScheduleArgOpNo = 2;
5133 auto *ScheduleTypeCI =
5134 dyn_cast<ConstantInt>(Val: CB.getArgOperand(i: ScheduleArgOpNo));
5135 unsigned ScheduleTypeVal =
5136 ScheduleTypeCI ? ScheduleTypeCI->getZExtValue() : 0;
5137 switch (OMPScheduleType(ScheduleTypeVal)) {
5138 case OMPScheduleType::UnorderedStatic:
5139 case OMPScheduleType::UnorderedStaticChunked:
5140 case OMPScheduleType::OrderedDistribute:
5141 case OMPScheduleType::OrderedDistributeChunked:
5142 break;
5143 default:
5144 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5145 SPMDCompatibilityTracker.insert(Elem: &CB);
5146 break;
5147 };
5148 } break;
5149 case OMPRTL___kmpc_target_init:
5150 KernelInitCB = &CB;
5151 break;
5152 case OMPRTL___kmpc_target_deinit:
5153 KernelDeinitCB = &CB;
5154 break;
5155 case OMPRTL___kmpc_parallel_60:
5156 if (!handleParallel60(A, CB))
5157 indicatePessimisticFixpoint();
5158 return;
5159 case OMPRTL___kmpc_omp_task:
5160 // We do not look into tasks right now, just give up.
5161 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5162 SPMDCompatibilityTracker.insert(Elem: &CB);
5163 ReachedUnknownParallelRegions.insert(Elem: &CB);
5164 break;
5165 case OMPRTL___kmpc_alloc_shared:
5166 case OMPRTL___kmpc_free_shared:
5167 // Return without setting a fixpoint, to be resolved in updateImpl.
5168 return;
5169 // The twelve static-loop entry points split into the two groups below.
5170 // Both come out SPMD-incompatible, but for different reasons: the first
5171 // because the call is single-threaded by construction, the second only
5172 // because SPMD-ization cannot yet guard per iteration. They are kept
5173 // apart so the second can be relaxed on its own once it can.
5174 case OMPRTL___kmpc_distribute_static_loop_4:
5175 case OMPRTL___kmpc_distribute_static_loop_4u:
5176 case OMPRTL___kmpc_distribute_static_loop_8:
5177 case OMPRTL___kmpc_distribute_static_loop_8u:
5178 // A plain `distribute` spreads its iterations over the teams, not over
5179 // the threads of a team: the runtime runs it with TId 0 and a team size
5180 // of one, and asserts the kernel is at parallel level 0. One thread per
5181 // block calls it, which is what generic mode gives it. In SPMD mode
5182 // every thread would call it, each running the whole of its block's
5183 // share of the loop body, so the kernel cannot be SPMD-ized however
5184 // analyzable the body is.
5185 if (!OMPInformationCache::getAnalyzableCallback(CB))
5186 ReachedUnknownParallelRegions.insert(Elem: &CB);
5187 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5188 SPMDCompatibilityTracker.insert(Elem: &CB);
5189 break;
5190 case OMPRTL___kmpc_distribute_for_static_loop_4:
5191 case OMPRTL___kmpc_distribute_for_static_loop_4u:
5192 case OMPRTL___kmpc_distribute_for_static_loop_8:
5193 case OMPRTL___kmpc_distribute_for_static_loop_8u:
5194 case OMPRTL___kmpc_for_static_loop_4:
5195 case OMPRTL___kmpc_for_static_loop_4u:
5196 case OMPRTL___kmpc_for_static_loop_8:
5197 case OMPRTL___kmpc_for_static_loop_8u:
5198 // These index by the thread's own id, so unlike a plain distribute they
5199 // are meant to be called by every thread of the block, and a kernel
5200 // reaching one is not SPMD-incompatible for that reason alone. What
5201 // stops us is the transform rather than the analysis: SPMD-ization
5202 // guards whatever has to stay single-threaded with a block-wide
5203 // barrier, and a barrier placed inside a loop body only some threads
5204 // run is divergent. Until guarding can express "the thread that owns
5205 // this iteration", stay conservative here too.
5206 if (!OMPInformationCache::getAnalyzableCallback(CB))
5207 ReachedUnknownParallelRegions.insert(Elem: &CB);
5208 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5209 SPMDCompatibilityTracker.insert(Elem: &CB);
5210 break;
5211 default:
5212 // Unknown OpenMP runtime calls cannot be executed in SPMD-mode,
5213 // generally. However, they do not hide parallel regions.
5214 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5215 SPMDCompatibilityTracker.insert(Elem: &CB);
5216 break;
5217 }
5218 // All other OpenMP runtime calls will not reach parallel regions so they
5219 // can be safely ignored for now. Since it is a known OpenMP runtime call
5220 // we have now modeled all effects and there is no need for any update.
5221 indicateOptimisticFixpoint();
5222 };
5223
5224 const auto *AACE =
5225 A.getAAFor<AACallEdges>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
5226 if (!AACE || !AACE->getState().isValidState() || AACE->hasUnknownCallee()) {
5227 CheckCallee(getAssociatedFunction(), 1);
5228 return;
5229 }
5230 const auto &OptimisticEdges = AACE->getOptimisticEdges();
5231 for (auto *Callee : OptimisticEdges) {
5232 CheckCallee(Callee, OptimisticEdges.size());
5233 if (isAtFixpoint())
5234 break;
5235 }
5236 }
5237
5238 ChangeStatus updateImpl(Attributor &A) override {
5239 // TODO: Once we have call site specific value information we can provide
5240 // call site specific liveness information and then it makes
5241 // sense to specialize attributes for call sites arguments instead of
5242 // redirecting requests to the callee argument.
5243 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
5244 KernelInfoState StateBefore = getState();
5245
5246 auto CheckCallee = [&](Function *F, int NumCallees) {
5247 const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Val: F);
5248
5249 // If F is not a runtime function, propagate the AAKernelInfo of the
5250 // callee.
5251 if (It == OMPInfoCache.RuntimeFunctionIDMap.end()) {
5252 const IRPosition &FnPos = IRPosition::function(F: *F);
5253 auto *FnAA =
5254 A.getAAFor<AAKernelInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
5255 if (!FnAA)
5256 return indicatePessimisticFixpoint();
5257 if (getState() == FnAA->getState())
5258 return ChangeStatus::UNCHANGED;
5259 getState() = FnAA->getState();
5260 return ChangeStatus::CHANGED;
5261 }
5262 // See the matching check in initialize: a !callback runtime function has
5263 // a second call edge by construction, and it is one we can analyze.
5264 if (NumCallees > 1 && !F->hasMetadata(KindID: LLVMContext::MD_callback))
5265 return indicatePessimisticFixpoint();
5266
5267 CallBase &CB = cast<CallBase>(Val&: getAssociatedValue());
5268 if (It->getSecond() == OMPRTL___kmpc_parallel_60) {
5269 if (!handleParallel60(A, CB))
5270 return indicatePessimisticFixpoint();
5271 return StateBefore == getState() ? ChangeStatus::UNCHANGED
5272 : ChangeStatus::CHANGED;
5273 }
5274
5275 // F is a runtime function that allocates or frees memory, check
5276 // AAHeapToStack and AAHeapToShared.
5277 assert(
5278 (It->getSecond() == OMPRTL___kmpc_alloc_shared ||
5279 It->getSecond() == OMPRTL___kmpc_free_shared) &&
5280 "Expected a __kmpc_alloc_shared or __kmpc_free_shared runtime call");
5281
5282 auto *HeapToStackAA = A.getAAFor<AAHeapToStack>(
5283 QueryingAA: *this, IRP: IRPosition::function(F: *CB.getCaller()), DepClass: DepClassTy::OPTIONAL);
5284 auto *HeapToSharedAA = A.getAAFor<AAHeapToShared>(
5285 QueryingAA: *this, IRP: IRPosition::function(F: *CB.getCaller()), DepClass: DepClassTy::OPTIONAL);
5286
5287 RuntimeFunction RF = It->getSecond();
5288
5289 switch (RF) {
5290 // If neither HeapToStack nor HeapToShared assume the call is removed,
5291 // assume SPMD incompatibility.
5292 case OMPRTL___kmpc_alloc_shared:
5293 if ((!HeapToStackAA || !HeapToStackAA->isAssumedHeapToStack(CB)) &&
5294 (!HeapToSharedAA || !HeapToSharedAA->isAssumedHeapToShared(CB)))
5295 SPMDCompatibilityTracker.insert(Elem: &CB);
5296 break;
5297 case OMPRTL___kmpc_free_shared:
5298 if ((!HeapToStackAA ||
5299 !HeapToStackAA->isAssumedHeapToStackRemovedFree(CB)) &&
5300 (!HeapToSharedAA ||
5301 !HeapToSharedAA->isAssumedHeapToSharedRemovedFree(CB)))
5302 SPMDCompatibilityTracker.insert(Elem: &CB);
5303 break;
5304 default:
5305 SPMDCompatibilityTracker.indicatePessimisticFixpoint();
5306 SPMDCompatibilityTracker.insert(Elem: &CB);
5307 }
5308 return ChangeStatus::CHANGED;
5309 };
5310
5311 const auto *AACE =
5312 A.getAAFor<AACallEdges>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
5313 if (!AACE || !AACE->getState().isValidState() || AACE->hasUnknownCallee()) {
5314 if (Function *F = getAssociatedFunction())
5315 CheckCallee(F, /*NumCallees=*/1);
5316 } else {
5317 const auto &OptimisticEdges = AACE->getOptimisticEdges();
5318 for (auto *Callee : OptimisticEdges) {
5319 CheckCallee(Callee, OptimisticEdges.size());
5320 if (isAtFixpoint())
5321 break;
5322 }
5323 }
5324
5325 return StateBefore == getState() ? ChangeStatus::UNCHANGED
5326 : ChangeStatus::CHANGED;
5327 }
5328
5329 /// Deal with a __kmpc_parallel_60 call (\p CB). Returns true if the call was
5330 /// handled, if a problem occurred, false is returned.
5331 bool handleParallel60(Attributor &A, CallBase &CB) {
5332 const unsigned int NonWrapperFunctionArgNo = 5;
5333 const unsigned int WrapperFunctionArgNo = 6;
5334 auto ParallelRegionOpArgNo = SPMDCompatibilityTracker.isAssumed()
5335 ? NonWrapperFunctionArgNo
5336 : WrapperFunctionArgNo;
5337
5338 auto *ParallelRegion = dyn_cast<Function>(
5339 Val: CB.getArgOperand(i: ParallelRegionOpArgNo)->stripPointerCasts());
5340 if (!ParallelRegion)
5341 return false;
5342
5343 ReachedKnownParallelRegions.insert(Elem: &CB);
5344 /// Check nested parallelism
5345 auto *FnAA = A.getAAFor<AAKernelInfo>(
5346 QueryingAA: *this, IRP: IRPosition::function(F: *ParallelRegion), DepClass: DepClassTy::OPTIONAL);
5347 NestedParallelism |= !FnAA || !FnAA->getState().isValidState() ||
5348 !FnAA->ReachedKnownParallelRegions.empty() ||
5349 !FnAA->ReachedKnownParallelRegions.isValidState() ||
5350 !FnAA->ReachedUnknownParallelRegions.isValidState() ||
5351 !FnAA->ReachedUnknownParallelRegions.empty();
5352 return true;
5353 }
5354};
5355
5356struct AAFoldRuntimeCall
5357 : public StateWrapper<BooleanState, AbstractAttribute> {
5358 using Base = StateWrapper<BooleanState, AbstractAttribute>;
5359
5360 AAFoldRuntimeCall(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
5361
5362 /// Statistics are tracked as part of manifest for now.
5363 void trackStatistics() const override {}
5364
5365 /// Create an abstract attribute biew for the position \p IRP.
5366 static AAFoldRuntimeCall &createForPosition(const IRPosition &IRP,
5367 Attributor &A);
5368
5369 /// See AbstractAttribute::getName()
5370 StringRef getName() const override { return "AAFoldRuntimeCall"; }
5371
5372 /// See AbstractAttribute::getIdAddr()
5373 const char *getIdAddr() const override { return &ID; }
5374
5375 /// This function should return true if the type of the \p AA is
5376 /// AAFoldRuntimeCall
5377 static bool classof(const AbstractAttribute *AA) {
5378 return (AA->getIdAddr() == &ID);
5379 }
5380
5381 static const char ID;
5382};
5383
5384struct AAFoldRuntimeCallCallSiteReturned : AAFoldRuntimeCall {
5385 AAFoldRuntimeCallCallSiteReturned(const IRPosition &IRP, Attributor &A)
5386 : AAFoldRuntimeCall(IRP, A) {}
5387
5388 /// See AbstractAttribute::getAsStr()
5389 const std::string getAsStr(Attributor *) const override {
5390 if (!isValidState())
5391 return "<invalid>";
5392
5393 std::string Str("simplified value: ");
5394
5395 if (!SimplifiedValue)
5396 return Str + std::string("none");
5397
5398 if (!*SimplifiedValue)
5399 return Str + std::string("nullptr");
5400
5401 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: *SimplifiedValue))
5402 return Str + std::to_string(val: CI->getSExtValue());
5403
5404 return Str + std::string("unknown");
5405 }
5406
5407 void initialize(Attributor &A) override {
5408 if (DisableOpenMPOptFolding)
5409 indicatePessimisticFixpoint();
5410
5411 Function *Callee = getAssociatedFunction();
5412
5413 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
5414 const auto &It = OMPInfoCache.RuntimeFunctionIDMap.find(Val: Callee);
5415 assert(It != OMPInfoCache.RuntimeFunctionIDMap.end() &&
5416 "Expected a known OpenMP runtime function");
5417
5418 RFKind = It->getSecond();
5419
5420 CallBase &CB = cast<CallBase>(Val&: getAssociatedValue());
5421 A.registerSimplificationCallback(
5422 IRP: IRPosition::callsite_returned(CB),
5423 CB: [&](const IRPosition &IRP, const AbstractAttribute *AA,
5424 bool &UsedAssumedInformation) -> std::optional<Value *> {
5425 assert((isValidState() || SimplifiedValue == nullptr) &&
5426 "Unexpected invalid state!");
5427
5428 if (!isAtFixpoint()) {
5429 UsedAssumedInformation = true;
5430 if (AA)
5431 A.recordDependence(FromAA: *this, ToAA: *AA, DepClass: DepClassTy::OPTIONAL);
5432 }
5433 return SimplifiedValue;
5434 });
5435 }
5436
5437 ChangeStatus updateImpl(Attributor &A) override {
5438 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5439 switch (RFKind) {
5440 case OMPRTL___kmpc_is_spmd_exec_mode:
5441 Changed |= foldIsSPMDExecMode(A);
5442 break;
5443 case OMPRTL___kmpc_parallel_level:
5444 Changed |= foldParallelLevel(A);
5445 break;
5446 case OMPRTL___kmpc_get_hardware_num_threads_in_block:
5447 Changed = Changed | foldKernelFnAttribute(A, Attr: "omp_target_thread_limit");
5448 break;
5449 case OMPRTL___kmpc_get_hardware_num_blocks:
5450 Changed = Changed | foldKernelFnAttribute(A, Attr: "omp_target_num_teams");
5451 break;
5452 default:
5453 llvm_unreachable("Unhandled OpenMP runtime function!");
5454 }
5455
5456 return Changed;
5457 }
5458
5459 ChangeStatus manifest(Attributor &A) override {
5460 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5461
5462 if (SimplifiedValue && *SimplifiedValue) {
5463 Instruction &I = *getCtxI();
5464 A.changeAfterManifest(IRP: IRPosition::inst(I), NV&: **SimplifiedValue);
5465 A.deleteAfterManifest(I);
5466
5467 CallBase *CB = dyn_cast<CallBase>(Val: &I);
5468 auto Remark = [&](OptimizationRemark OR) {
5469 if (auto *C = dyn_cast<ConstantInt>(Val: *SimplifiedValue))
5470 return OR << "Replacing OpenMP runtime call "
5471 << CB->getCalledFunction()->getName() << " with "
5472 << ore::NV("FoldedValue", C->getZExtValue()) << ".";
5473 return OR << "Replacing OpenMP runtime call "
5474 << CB->getCalledFunction()->getName() << ".";
5475 };
5476
5477 if (CB && EnableVerboseRemarks)
5478 A.emitRemark<OptimizationRemark>(I: CB, RemarkName: "OMP180", RemarkCB&: Remark);
5479
5480 LLVM_DEBUG(dbgs() << TAG << "Replacing runtime call: " << I << " with "
5481 << **SimplifiedValue << "\n");
5482
5483 Changed = ChangeStatus::CHANGED;
5484 }
5485
5486 return Changed;
5487 }
5488
5489 ChangeStatus indicatePessimisticFixpoint() override {
5490 SimplifiedValue = nullptr;
5491 return AAFoldRuntimeCall::indicatePessimisticFixpoint();
5492 }
5493
5494private:
5495 /// Fold __kmpc_is_spmd_exec_mode into a constant if possible.
5496 ChangeStatus foldIsSPMDExecMode(Attributor &A) {
5497 std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
5498
5499 unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
5500 unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
5501 auto *CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
5502 QueryingAA: *this, IRP: IRPosition::function(F: *getAnchorScope()), DepClass: DepClassTy::REQUIRED);
5503
5504 if (!CallerKernelInfoAA ||
5505 !CallerKernelInfoAA->ReachingKernelEntries.isValidState())
5506 return indicatePessimisticFixpoint();
5507
5508 for (Kernel K : CallerKernelInfoAA->ReachingKernelEntries) {
5509 auto *AA = A.getAAFor<AAKernelInfo>(QueryingAA: *this, IRP: IRPosition::function(F: *K),
5510 DepClass: DepClassTy::REQUIRED);
5511
5512 if (!AA || !AA->isValidState()) {
5513 SimplifiedValue = nullptr;
5514 return indicatePessimisticFixpoint();
5515 }
5516
5517 if (AA->SPMDCompatibilityTracker.isAssumed()) {
5518 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5519 ++KnownSPMDCount;
5520 else
5521 ++AssumedSPMDCount;
5522 } else {
5523 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5524 ++KnownNonSPMDCount;
5525 else
5526 ++AssumedNonSPMDCount;
5527 }
5528 }
5529
5530 if ((AssumedSPMDCount + KnownSPMDCount) &&
5531 (AssumedNonSPMDCount + KnownNonSPMDCount))
5532 return indicatePessimisticFixpoint();
5533
5534 auto &Ctx = getAnchorValue().getContext();
5535 if (KnownSPMDCount || AssumedSPMDCount) {
5536 assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
5537 "Expected only SPMD kernels!");
5538 // All reaching kernels are in SPMD mode. Update all function calls to
5539 // __kmpc_is_spmd_exec_mode to 1.
5540 SimplifiedValue = ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: true);
5541 } else if (KnownNonSPMDCount || AssumedNonSPMDCount) {
5542 assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
5543 "Expected only non-SPMD kernels!");
5544 // All reaching kernels are in non-SPMD mode. Update all function
5545 // calls to __kmpc_is_spmd_exec_mode to 0.
5546 SimplifiedValue = ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: false);
5547 } else {
5548 // We have empty reaching kernels, therefore we cannot tell if the
5549 // associated call site can be folded. At this moment, SimplifiedValue
5550 // must be none.
5551 assert(!SimplifiedValue && "SimplifiedValue should be none");
5552 }
5553
5554 return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
5555 : ChangeStatus::CHANGED;
5556 }
5557
5558 /// Fold __kmpc_parallel_level into a constant if possible.
5559 ChangeStatus foldParallelLevel(Attributor &A) {
5560 std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
5561
5562 auto *CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
5563 QueryingAA: *this, IRP: IRPosition::function(F: *getAnchorScope()), DepClass: DepClassTy::REQUIRED);
5564
5565 if (!CallerKernelInfoAA ||
5566 !CallerKernelInfoAA->ParallelLevels.isValidState())
5567 return indicatePessimisticFixpoint();
5568
5569 if (!CallerKernelInfoAA->ReachingKernelEntries.isValidState())
5570 return indicatePessimisticFixpoint();
5571
5572 if (CallerKernelInfoAA->ReachingKernelEntries.empty()) {
5573 assert(!SimplifiedValue &&
5574 "SimplifiedValue should keep none at this point");
5575 return ChangeStatus::UNCHANGED;
5576 }
5577
5578 unsigned AssumedSPMDCount = 0, KnownSPMDCount = 0;
5579 unsigned AssumedNonSPMDCount = 0, KnownNonSPMDCount = 0;
5580 for (Kernel K : CallerKernelInfoAA->ReachingKernelEntries) {
5581 auto *AA = A.getAAFor<AAKernelInfo>(QueryingAA: *this, IRP: IRPosition::function(F: *K),
5582 DepClass: DepClassTy::REQUIRED);
5583 if (!AA || !AA->SPMDCompatibilityTracker.isValidState())
5584 return indicatePessimisticFixpoint();
5585
5586 if (AA->SPMDCompatibilityTracker.isAssumed()) {
5587 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5588 ++KnownSPMDCount;
5589 else
5590 ++AssumedSPMDCount;
5591 } else {
5592 if (AA->SPMDCompatibilityTracker.isAtFixpoint())
5593 ++KnownNonSPMDCount;
5594 else
5595 ++AssumedNonSPMDCount;
5596 }
5597 }
5598
5599 if ((AssumedSPMDCount + KnownSPMDCount) &&
5600 (AssumedNonSPMDCount + KnownNonSPMDCount))
5601 return indicatePessimisticFixpoint();
5602
5603 auto &Ctx = getAnchorValue().getContext();
5604 // If the caller can only be reached by SPMD kernel entries, the parallel
5605 // level is 1. Similarly, if the caller can only be reached by non-SPMD
5606 // kernel entries, it is 0.
5607 if (AssumedSPMDCount || KnownSPMDCount) {
5608 assert(KnownNonSPMDCount == 0 && AssumedNonSPMDCount == 0 &&
5609 "Expected only SPMD kernels!");
5610 SimplifiedValue = ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: 1);
5611 } else {
5612 assert(KnownSPMDCount == 0 && AssumedSPMDCount == 0 &&
5613 "Expected only non-SPMD kernels!");
5614 SimplifiedValue = ConstantInt::get(Ty: Type::getInt8Ty(C&: Ctx), V: 0);
5615 }
5616 return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
5617 : ChangeStatus::CHANGED;
5618 }
5619
5620 ChangeStatus foldKernelFnAttribute(Attributor &A, llvm::StringRef Attr) {
5621 // Specialize only if all the calls agree with the attribute constant value
5622 int32_t CurrentAttrValue = -1;
5623 std::optional<Value *> SimplifiedValueBefore = SimplifiedValue;
5624
5625 auto *CallerKernelInfoAA = A.getAAFor<AAKernelInfo>(
5626 QueryingAA: *this, IRP: IRPosition::function(F: *getAnchorScope()), DepClass: DepClassTy::REQUIRED);
5627
5628 if (!CallerKernelInfoAA ||
5629 !CallerKernelInfoAA->ReachingKernelEntries.isValidState())
5630 return indicatePessimisticFixpoint();
5631
5632 // Iterate over the kernels that reach this function
5633 for (Kernel K : CallerKernelInfoAA->ReachingKernelEntries) {
5634 int32_t NextAttrVal = K->getFnAttributeAsParsedInteger(Kind: Attr, Default: -1);
5635
5636 if (NextAttrVal == -1 ||
5637 (CurrentAttrValue != -1 && CurrentAttrValue != NextAttrVal))
5638 return indicatePessimisticFixpoint();
5639 CurrentAttrValue = NextAttrVal;
5640 }
5641
5642 if (CurrentAttrValue != -1) {
5643 auto &Ctx = getAnchorValue().getContext();
5644 SimplifiedValue =
5645 ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: CurrentAttrValue);
5646 }
5647 return SimplifiedValue == SimplifiedValueBefore ? ChangeStatus::UNCHANGED
5648 : ChangeStatus::CHANGED;
5649 }
5650
5651 /// An optional value the associated value is assumed to fold to. That is, we
5652 /// assume the associated value (which is a call) can be replaced by this
5653 /// simplified value.
5654 std::optional<Value *> SimplifiedValue;
5655
5656 /// The runtime function kind of the callee of the associated call site.
5657 RuntimeFunction RFKind;
5658};
5659
5660} // namespace
5661
5662/// Register folding callsite
5663void OpenMPOpt::registerFoldRuntimeCall(RuntimeFunction RF) {
5664 auto &RFI = OMPInfoCache.RFIs[RF];
5665 RFI.foreachUse(SCC, CB: [&](Use &U, Function &F) {
5666 CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, RFI: &RFI);
5667 if (!CI)
5668 return false;
5669 A.getOrCreateAAFor<AAFoldRuntimeCall>(
5670 IRP: IRPosition::callsite_returned(CB: *CI), /* QueryingAA */ nullptr,
5671 DepClass: DepClassTy::NONE, /* ForceUpdate */ false,
5672 /* UpdateAfterInit */ false);
5673 return false;
5674 });
5675}
5676
5677void OpenMPOpt::registerAAs(bool IsModulePass) {
5678 if (SCC.empty())
5679 return;
5680
5681 if (IsModulePass) {
5682 // Ensure we create the AAKernelInfo AAs first and without triggering an
5683 // update. This will make sure we register all value simplification
5684 // callbacks before any other AA has the chance to create an AAValueSimplify
5685 // or similar.
5686 auto CreateKernelInfoCB = [&](Use &, Function &Kernel) {
5687 A.getOrCreateAAFor<AAKernelInfo>(
5688 IRP: IRPosition::function(F: Kernel), /* QueryingAA */ nullptr,
5689 DepClass: DepClassTy::NONE, /* ForceUpdate */ false,
5690 /* UpdateAfterInit */ false);
5691 return false;
5692 };
5693 OMPInformationCache::RuntimeFunctionInfo &InitRFI =
5694 OMPInfoCache.RFIs[OMPRTL___kmpc_target_init];
5695 InitRFI.foreachUse(SCC, CB: CreateKernelInfoCB);
5696
5697 registerFoldRuntimeCall(RF: OMPRTL___kmpc_is_spmd_exec_mode);
5698 registerFoldRuntimeCall(RF: OMPRTL___kmpc_parallel_level);
5699 registerFoldRuntimeCall(RF: OMPRTL___kmpc_get_hardware_num_threads_in_block);
5700 registerFoldRuntimeCall(RF: OMPRTL___kmpc_get_hardware_num_blocks);
5701 }
5702
5703 // Create CallSite AA for all Getters.
5704 if (DeduceICVValues) {
5705 for (int Idx = 0; Idx < OMPInfoCache.ICVs.size() - 1; ++Idx) {
5706 auto ICVInfo = OMPInfoCache.ICVs[static_cast<InternalControlVar>(Idx)];
5707
5708 auto &GetterRFI = OMPInfoCache.RFIs[ICVInfo.Getter];
5709
5710 auto CreateAA = [&](Use &U, Function &Caller) {
5711 CallInst *CI = OpenMPOpt::getCallIfRegularCall(U, RFI: &GetterRFI);
5712 if (!CI)
5713 return false;
5714
5715 auto &CB = cast<CallBase>(Val&: *CI);
5716
5717 IRPosition CBPos = IRPosition::callsite_function(CB);
5718 A.getOrCreateAAFor<AAICVTracker>(IRP: CBPos);
5719 return false;
5720 };
5721
5722 GetterRFI.foreachUse(SCC, CB: CreateAA);
5723 }
5724 }
5725
5726 // Create an ExecutionDomain AA for every function and a HeapToStack AA for
5727 // every function if there is a device kernel.
5728 if (!isOpenMPDevice(M))
5729 return;
5730
5731 for (auto *F : SCC) {
5732 if (F->isDeclaration())
5733 continue;
5734
5735 // We look at internal functions only on-demand but if any use is not a
5736 // direct call or outside the current set of analyzed functions, we have
5737 // to do it eagerly.
5738 if (F->hasLocalLinkage()) {
5739 if (llvm::all_of(Range: F->uses(), P: [this](const Use &U) {
5740 const auto *CB = dyn_cast<CallBase>(Val: U.getUser());
5741 return CB && CB->isCallee(U: &U) &&
5742 A.isRunOn(Fn: const_cast<Function *>(CB->getCaller()));
5743 }))
5744 continue;
5745 }
5746 registerAAsForFunction(A, F: *F);
5747 }
5748}
5749
5750void OpenMPOpt::registerAAsForFunction(Attributor &A, const Function &F) {
5751 auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
5752
5753 IRPosition FPos = IRPosition::function(F);
5754 A.getOrCreateAAFor<AAExecutionDomain>(IRP: FPos);
5755 if (F.hasFnAttribute(Kind: Attribute::Convergent))
5756 A.getOrCreateAAFor<AANonConvergent>(IRP: FPos);
5757
5758 bool FunctionUsesSharedAlloc = false;
5759 if (!DisableOpenMPOptDeglobalization) {
5760 const OMPInformationCache::RuntimeFunctionInfo::UseVector *SharedAllocUses =
5761 OMPInfoCache.RFIs[OMPRTL___kmpc_alloc_shared].getUseVector(
5762 F&: const_cast<Function &>(F));
5763 FunctionUsesSharedAlloc = SharedAllocUses && !SharedAllocUses->empty();
5764 }
5765 bool HasHeapToStackCandidate = false;
5766 const TargetLibraryInfo *TLI = nullptr;
5767
5768 for (auto &I : instructions(F)) {
5769 if (auto *LI = dyn_cast<LoadInst>(Val: &I)) {
5770 bool UsedAssumedInformation = false;
5771 A.getAssumedSimplified(V: IRPosition::value(V: *LI), /* AA */ nullptr,
5772 UsedAssumedInformation, S: AA::Interprocedural);
5773 A.getOrCreateAAFor<AAAddressSpace>(
5774 IRP: IRPosition::value(V: *LI->getPointerOperand()));
5775 continue;
5776 }
5777 if (auto *CI = dyn_cast<CallBase>(Val: &I)) {
5778 if (!DisableOpenMPOptDeglobalization && !HasHeapToStackCandidate) {
5779 if (!TLI)
5780 TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F);
5781 HasHeapToStackCandidate =
5782 isRemovableAlloc(V: CI, TLI) || getFreedOperand(CB: CI, TLI);
5783 }
5784 if (CI->isIndirectCall())
5785 A.getOrCreateAAFor<AAIndirectCallInfo>(
5786 IRP: IRPosition::callsite_function(CB: *CI));
5787 }
5788 if (auto *SI = dyn_cast<StoreInst>(Val: &I)) {
5789 A.getOrCreateAAFor<AAIsDead>(IRP: IRPosition::value(V: *SI));
5790 A.getOrCreateAAFor<AAAddressSpace>(
5791 IRP: IRPosition::value(V: *SI->getPointerOperand()));
5792 continue;
5793 }
5794 if (auto *FI = dyn_cast<FenceInst>(Val: &I)) {
5795 A.getOrCreateAAFor<AAIsDead>(IRP: IRPosition::value(V: *FI));
5796 continue;
5797 }
5798 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
5799 if (II->getIntrinsicID() == Intrinsic::assume) {
5800 A.getOrCreateAAFor<AAPotentialValues>(
5801 IRP: IRPosition::value(V: *II->getArgOperand(i: 0)));
5802 continue;
5803 }
5804 }
5805 }
5806
5807 if (FunctionUsesSharedAlloc)
5808 A.getOrCreateAAFor<AAHeapToShared>(IRP: FPos);
5809 if (HasHeapToStackCandidate)
5810 A.getOrCreateAAFor<AAHeapToStack>(IRP: FPos);
5811}
5812
5813const char AAICVTracker::ID = 0;
5814const char AAKernelInfo::ID = 0;
5815const char AAExecutionDomain::ID = 0;
5816const char AAHeapToShared::ID = 0;
5817const char AAFoldRuntimeCall::ID = 0;
5818
5819AAICVTracker &AAICVTracker::createForPosition(const IRPosition &IRP,
5820 Attributor &A) {
5821 AAICVTracker *AA = nullptr;
5822 switch (IRP.getPositionKind()) {
5823 case IRPosition::IRP_INVALID:
5824 case IRPosition::IRP_FLOAT:
5825 case IRPosition::IRP_ARGUMENT:
5826 case IRPosition::IRP_CALL_SITE_ARGUMENT:
5827 llvm_unreachable("ICVTracker can only be created for function position!");
5828 case IRPosition::IRP_RETURNED:
5829 AA = new (A.Allocator) AAICVTrackerFunctionReturned(IRP, A);
5830 break;
5831 case IRPosition::IRP_CALL_SITE_RETURNED:
5832 AA = new (A.Allocator) AAICVTrackerCallSiteReturned(IRP, A);
5833 break;
5834 case IRPosition::IRP_CALL_SITE:
5835 AA = new (A.Allocator) AAICVTrackerCallSite(IRP, A);
5836 break;
5837 case IRPosition::IRP_FUNCTION:
5838 AA = new (A.Allocator) AAICVTrackerFunction(IRP, A);
5839 break;
5840 }
5841
5842 return *AA;
5843}
5844
5845AAExecutionDomain &AAExecutionDomain::createForPosition(const IRPosition &IRP,
5846 Attributor &A) {
5847 AAExecutionDomainFunction *AA = nullptr;
5848 switch (IRP.getPositionKind()) {
5849 case IRPosition::IRP_INVALID:
5850 case IRPosition::IRP_FLOAT:
5851 case IRPosition::IRP_ARGUMENT:
5852 case IRPosition::IRP_CALL_SITE_ARGUMENT:
5853 case IRPosition::IRP_RETURNED:
5854 case IRPosition::IRP_CALL_SITE_RETURNED:
5855 case IRPosition::IRP_CALL_SITE:
5856 llvm_unreachable(
5857 "AAExecutionDomain can only be created for function position!");
5858 case IRPosition::IRP_FUNCTION:
5859 AA = new (A.Allocator) AAExecutionDomainFunction(IRP, A);
5860 break;
5861 }
5862
5863 return *AA;
5864}
5865
5866AAHeapToShared &AAHeapToShared::createForPosition(const IRPosition &IRP,
5867 Attributor &A) {
5868 AAHeapToSharedFunction *AA = nullptr;
5869 switch (IRP.getPositionKind()) {
5870 case IRPosition::IRP_INVALID:
5871 case IRPosition::IRP_FLOAT:
5872 case IRPosition::IRP_ARGUMENT:
5873 case IRPosition::IRP_CALL_SITE_ARGUMENT:
5874 case IRPosition::IRP_RETURNED:
5875 case IRPosition::IRP_CALL_SITE_RETURNED:
5876 case IRPosition::IRP_CALL_SITE:
5877 llvm_unreachable(
5878 "AAHeapToShared can only be created for function position!");
5879 case IRPosition::IRP_FUNCTION:
5880 AA = new (A.Allocator) AAHeapToSharedFunction(IRP, A);
5881 break;
5882 }
5883
5884 return *AA;
5885}
5886
5887AAKernelInfo &AAKernelInfo::createForPosition(const IRPosition &IRP,
5888 Attributor &A) {
5889 AAKernelInfo *AA = nullptr;
5890 switch (IRP.getPositionKind()) {
5891 case IRPosition::IRP_INVALID:
5892 case IRPosition::IRP_FLOAT:
5893 case IRPosition::IRP_ARGUMENT:
5894 case IRPosition::IRP_RETURNED:
5895 case IRPosition::IRP_CALL_SITE_RETURNED:
5896 case IRPosition::IRP_CALL_SITE_ARGUMENT:
5897 llvm_unreachable("KernelInfo can only be created for function position!");
5898 case IRPosition::IRP_CALL_SITE:
5899 AA = new (A.Allocator) AAKernelInfoCallSite(IRP, A);
5900 break;
5901 case IRPosition::IRP_FUNCTION:
5902 AA = new (A.Allocator) AAKernelInfoFunction(IRP, A);
5903 break;
5904 }
5905
5906 return *AA;
5907}
5908
5909AAFoldRuntimeCall &AAFoldRuntimeCall::createForPosition(const IRPosition &IRP,
5910 Attributor &A) {
5911 AAFoldRuntimeCall *AA = nullptr;
5912 switch (IRP.getPositionKind()) {
5913 case IRPosition::IRP_INVALID:
5914 case IRPosition::IRP_FLOAT:
5915 case IRPosition::IRP_ARGUMENT:
5916 case IRPosition::IRP_RETURNED:
5917 case IRPosition::IRP_FUNCTION:
5918 case IRPosition::IRP_CALL_SITE:
5919 case IRPosition::IRP_CALL_SITE_ARGUMENT:
5920 llvm_unreachable("KernelInfo can only be created for call site position!");
5921 case IRPosition::IRP_CALL_SITE_RETURNED:
5922 AA = new (A.Allocator) AAFoldRuntimeCallCallSiteReturned(IRP, A);
5923 break;
5924 }
5925
5926 return *AA;
5927}
5928
5929/// Bound the if-cascade AAIndirectCallInfo builds for an indirect call. Device
5930/// code reaches its callees through function-pointer tables and virtual
5931/// dispatch, so a call site can see every address-taken candidate in the
5932/// module; specializing all of them costs more in code size and compile time
5933/// than the direct calls are worth.
5934///
5935/// This is a threshold on the call site rather than a limit on how many callees
5936/// get specialized: the Attributor asks about each callee with the same total,
5937/// so a site above the threshold keeps its indirect call instead of getting
5938/// this many direct ones plus a fallback.
5939static bool shouldSpecializeIndirectCallee(Attributor &,
5940 const AbstractAttribute &,
5941 CallBase &, Function &,
5942 unsigned NumAssumedCallees) {
5943 return NumAssumedCallees <= MaxCalleesForSpecialization;
5944}
5945
5946PreservedAnalyses OpenMPOptPass::run(Module &M, ModuleAnalysisManager &AM) {
5947 if (!containsOpenMP(M))
5948 return PreservedAnalyses::all();
5949 if (DisableOpenMPOptimizations)
5950 return PreservedAnalyses::all();
5951
5952 FunctionAnalysisManager &FAM =
5953 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
5954 KernelSet Kernels = getDeviceKernels(M);
5955
5956 if (PrintModuleBeforeOptimizations)
5957 LLVM_DEBUG(dbgs() << TAG << "Module before OpenMPOpt Module Pass:\n" << M);
5958
5959 auto IsCalled = [&](Function &F) {
5960 if (Kernels.contains(key: &F))
5961 return true;
5962 return !F.use_empty();
5963 };
5964
5965 auto EmitRemark = [&](Function &F) {
5966 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
5967 ORE.emit(RemarkBuilder: [&]() {
5968 OptimizationRemarkAnalysis ORA(DEBUG_TYPE, "OMP140", &F);
5969 return ORA << "Could not internalize function. "
5970 << "Some optimizations may not be possible. [OMP140]";
5971 });
5972 };
5973
5974 bool Changed = false;
5975
5976 // Create internal copies of each function if this is a kernel Module. This
5977 // allows iterprocedural passes to see every call edge.
5978 DenseMap<Function *, Function *> InternalizedMap;
5979 if (isOpenMPDevice(M)) {
5980 SmallPtrSet<Function *, 16> InternalizeFns;
5981 for (Function &F : M)
5982 if (!F.isDeclaration() && !Kernels.contains(key: &F) && IsCalled(F) &&
5983 !DisableInternalization) {
5984 if (Attributor::isInternalizable(F)) {
5985 InternalizeFns.insert(Ptr: &F);
5986 } else if (!F.hasLocalLinkage() && !F.hasFnAttribute(Kind: Attribute::Cold)) {
5987 EmitRemark(F);
5988 }
5989 }
5990
5991 Changed |=
5992 Attributor::internalizeFunctions(FnSet&: InternalizeFns, FnMap&: InternalizedMap);
5993 }
5994
5995 // Look at every function in the Module unless it was internalized.
5996 SetVector<Function *> Functions;
5997 SmallVector<Function *, 16> SCC;
5998 for (Function &F : M)
5999 if (!F.isDeclaration() && !InternalizedMap.lookup(Val: &F)) {
6000 SCC.push_back(Elt: &F);
6001 Functions.insert(X: &F);
6002 }
6003
6004 if (SCC.empty())
6005 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
6006
6007 AnalysisGetter AG(FAM);
6008
6009 auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
6010 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *F);
6011 };
6012
6013 BumpPtrAllocator Allocator;
6014 CallGraphUpdater CGUpdater;
6015
6016 bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
6017 LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink ||
6018 LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
6019 OMPInformationCache InfoCache(M, AG, Allocator, /*CGSCC*/ nullptr, PostLink);
6020
6021 unsigned MaxFixpointIterations =
6022 (isOpenMPDevice(M)) ? SetFixpointIterations : 32;
6023
6024 AttributorConfig AC(CGUpdater);
6025 AC.DefaultInitializeLiveInternals = false;
6026 AC.IsModulePass = true;
6027 AC.RewriteSignatures = false;
6028 AC.MaxFixpointIterations = MaxFixpointIterations;
6029 AC.OREGetter = OREGetter;
6030 AC.PassName = DEBUG_TYPE;
6031 AC.InitializationCallback = OpenMPOpt::registerAAsForFunction;
6032 AC.IndirectCalleeSpecializationCallback = shouldSpecializeIndirectCallee;
6033 AC.IPOAmendableCB = [](const Function &F) {
6034 return F.hasFnAttribute(Kind: "kernel");
6035 };
6036
6037 Attributor A(Functions, InfoCache, AC);
6038
6039 OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
6040 Changed |= OMPOpt.run(IsModulePass: true);
6041
6042 // Optionally inline device functions for potentially better performance.
6043 if (AlwaysInlineDeviceFunctions && isOpenMPDevice(M))
6044 for (Function &F : M)
6045 if (!F.isDeclaration() && !Kernels.contains(key: &F) &&
6046 !F.hasFnAttribute(Kind: Attribute::NoInline))
6047 F.addFnAttr(Kind: Attribute::AlwaysInline);
6048
6049 if (PrintModuleAfterOptimizations)
6050 LLVM_DEBUG(dbgs() << TAG << "Module after OpenMPOpt Module Pass:\n" << M);
6051
6052 if (Changed)
6053 return PreservedAnalyses::none();
6054
6055 return PreservedAnalyses::all();
6056}
6057
6058PreservedAnalyses OpenMPOptCGSCCPass::run(LazyCallGraph::SCC &C,
6059 CGSCCAnalysisManager &AM,
6060 LazyCallGraph &CG,
6061 CGSCCUpdateResult &UR) {
6062 if (!containsOpenMP(M&: *C.begin()->getFunction().getParent()))
6063 return PreservedAnalyses::all();
6064 if (DisableOpenMPOptimizations)
6065 return PreservedAnalyses::all();
6066
6067 SmallVector<Function *, 16> SCC;
6068 // If there are kernels in the module, we have to run on all SCC's.
6069 for (LazyCallGraph::Node &N : C) {
6070 Function *Fn = &N.getFunction();
6071 SCC.push_back(Elt: Fn);
6072 }
6073
6074 if (SCC.empty())
6075 return PreservedAnalyses::all();
6076
6077 Module &M = *C.begin()->getFunction().getParent();
6078
6079 if (PrintModuleBeforeOptimizations)
6080 LLVM_DEBUG(dbgs() << TAG << "Module before OpenMPOpt CGSCC Pass:\n" << M);
6081
6082 FunctionAnalysisManager &FAM =
6083 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(IR&: C, ExtraArgs&: CG).getManager();
6084
6085 AnalysisGetter AG(FAM);
6086
6087 auto OREGetter = [&FAM](Function *F) -> OptimizationRemarkEmitter & {
6088 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: *F);
6089 };
6090
6091 BumpPtrAllocator Allocator;
6092 CallGraphUpdater CGUpdater;
6093 CGUpdater.initialize(LCG&: CG, SCC&: C, AM, UR);
6094
6095 bool PostLink = LTOPhase == ThinOrFullLTOPhase::FullLTOPostLink ||
6096 LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink ||
6097 LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink;
6098 SetVector<Function *> Functions(llvm::from_range, SCC);
6099 OMPInformationCache InfoCache(*(Functions.back()->getParent()), AG, Allocator,
6100 /*CGSCC*/ &Functions, PostLink);
6101
6102 unsigned MaxFixpointIterations =
6103 (isOpenMPDevice(M)) ? SetFixpointIterations : 32;
6104
6105 AttributorConfig AC(CGUpdater);
6106 AC.DefaultInitializeLiveInternals = false;
6107 AC.IsModulePass = false;
6108 AC.RewriteSignatures = false;
6109 AC.MaxFixpointIterations = MaxFixpointIterations;
6110 AC.OREGetter = OREGetter;
6111 AC.PassName = DEBUG_TYPE;
6112 AC.InitializationCallback = OpenMPOpt::registerAAsForFunction;
6113 AC.IndirectCalleeSpecializationCallback = shouldSpecializeIndirectCallee;
6114
6115 Attributor A(Functions, InfoCache, AC);
6116
6117 OpenMPOpt OMPOpt(SCC, CGUpdater, OREGetter, InfoCache, A);
6118 bool Changed = OMPOpt.run(IsModulePass: false);
6119
6120 if (PrintModuleAfterOptimizations)
6121 LLVM_DEBUG(dbgs() << TAG << "Module after OpenMPOpt CGSCC Pass:\n" << M);
6122
6123 if (Changed)
6124 return PreservedAnalyses::none();
6125
6126 return PreservedAnalyses::all();
6127}
6128
6129bool llvm::omp::isOpenMPKernel(Function &Fn) {
6130 return Fn.hasFnAttribute(Kind: "kernel");
6131}
6132
6133KernelSet llvm::omp::getDeviceKernels(Module &M) {
6134 KernelSet Kernels;
6135
6136 for (Function &F : M)
6137 if (F.hasKernelCallingConv()) {
6138 // We are only interested in OpenMP target regions. Others, such as
6139 // kernels generated by CUDA but linked together, are not interesting to
6140 // this pass.
6141 if (isOpenMPKernel(Fn&: F)) {
6142 ++NumOpenMPTargetRegionKernels;
6143 Kernels.insert(X: &F);
6144 } else
6145 ++NumNonOpenMPTargetRegionKernels;
6146 }
6147
6148 return Kernels;
6149}
6150
6151bool llvm::omp::containsOpenMP(Module &M) {
6152 Metadata *MD = M.getModuleFlag(Key: "openmp");
6153 if (!MD)
6154 return false;
6155
6156 return true;
6157}
6158
6159bool llvm::omp::isOpenMPDevice(Module &M) {
6160 Metadata *MD = M.getModuleFlag(Key: "openmp-device");
6161 if (!MD)
6162 return false;
6163
6164 return true;
6165}
6166