1//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file implements the OpenMPIRBuilder class, which is used as a
11/// convenient way to create LLVM instructions for OpenMP directives.
12///
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16#include "llvm/ADT/SmallBitVector.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/ADT/SmallVectorExtras.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/Analysis/AssumptionCache.h"
22#include "llvm/Analysis/CodeMetrics.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/Analysis/OptimizationRemarkEmitter.h"
25#include "llvm/Analysis/PostDominators.h"
26#include "llvm/Analysis/ScalarEvolution.h"
27#include "llvm/Analysis/TargetLibraryInfo.h"
28#include "llvm/Bitcode/BitcodeReader.h"
29#include "llvm/Frontend/Offloading/Utility.h"
30#include "llvm/Frontend/OpenMP/OMPGridValues.h"
31#include "llvm/IR/Attributes.h"
32#include "llvm/IR/BasicBlock.h"
33#include "llvm/IR/CFG.h"
34#include "llvm/IR/CallingConv.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DIBuilder.h"
38#include "llvm/IR/DebugInfoMetadata.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalVariable.h"
42#include "llvm/IR/IRBuilder.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/LLVMContext.h"
45#include "llvm/IR/MDBuilder.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/PassInstrumentation.h"
48#include "llvm/IR/PassManager.h"
49#include "llvm/IR/ReplaceConstant.h"
50#include "llvm/IR/Value.h"
51#include "llvm/MC/TargetRegistry.h"
52#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Error.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/NVVMAttributes.h"
56#include "llvm/Support/VirtualFileSystem.h"
57#include "llvm/Target/TargetMachine.h"
58#include "llvm/Target/TargetOptions.h"
59#include "llvm/Transforms/Utils/BasicBlockUtils.h"
60#include "llvm/Transforms/Utils/Cloning.h"
61#include "llvm/Transforms/Utils/CodeExtractor.h"
62#include "llvm/Transforms/Utils/LoopPeel.h"
63#include "llvm/Transforms/Utils/UnrollLoop.h"
64
65#include <cstdint>
66#include <optional>
67
68#define DEBUG_TYPE "openmp-ir-builder"
69
70using namespace llvm;
71using namespace omp;
72
73static cl::opt<bool>
74 OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
75 cl::desc("Use optimistic attributes describing "
76 "'as-if' properties of runtime calls."),
77 cl::init(Val: false));
78
79static cl::opt<double> UnrollThresholdFactor(
80 "openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
81 cl::desc("Factor for the unroll threshold to account for code "
82 "simplifications still taking place"),
83 cl::init(Val: 1.5));
84
85static cl::opt<bool> UseDefaultMaxThreads(
86 "openmp-ir-builder-use-default-max-threads", cl::Hidden,
87 cl::desc("Use a default max threads if none is provided."), cl::init(Val: true));
88
89#ifndef NDEBUG
90/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
91/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
92/// an InsertPoint stores the instruction before something is inserted. For
93/// instance, if both point to the same instruction, two IRBuilders alternating
94/// creating instruction will cause the instructions to be interleaved.
95static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
96 IRBuilder<>::InsertPoint IP2) {
97 if (!IP1.isValid() || !IP2.isValid())
98 return false;
99 return IP1 == IP2;
100}
101
102static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
103 // Valid ordered/unordered and base algorithm combinations.
104 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
105 case OMPScheduleType::UnorderedStaticChunked:
106 case OMPScheduleType::UnorderedStatic:
107 case OMPScheduleType::UnorderedDynamicChunked:
108 case OMPScheduleType::UnorderedGuidedChunked:
109 case OMPScheduleType::UnorderedRuntime:
110 case OMPScheduleType::UnorderedAuto:
111 case OMPScheduleType::UnorderedTrapezoidal:
112 case OMPScheduleType::UnorderedGreedy:
113 case OMPScheduleType::UnorderedBalanced:
114 case OMPScheduleType::UnorderedGuidedIterativeChunked:
115 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
116 case OMPScheduleType::UnorderedSteal:
117 case OMPScheduleType::UnorderedStaticBalancedChunked:
118 case OMPScheduleType::UnorderedGuidedSimd:
119 case OMPScheduleType::UnorderedRuntimeSimd:
120 case OMPScheduleType::OrderedStaticChunked:
121 case OMPScheduleType::OrderedStatic:
122 case OMPScheduleType::OrderedDynamicChunked:
123 case OMPScheduleType::OrderedGuidedChunked:
124 case OMPScheduleType::OrderedRuntime:
125 case OMPScheduleType::OrderedAuto:
126 case OMPScheduleType::OrderdTrapezoidal:
127 case OMPScheduleType::NomergeUnorderedStaticChunked:
128 case OMPScheduleType::NomergeUnorderedStatic:
129 case OMPScheduleType::NomergeUnorderedDynamicChunked:
130 case OMPScheduleType::NomergeUnorderedGuidedChunked:
131 case OMPScheduleType::NomergeUnorderedRuntime:
132 case OMPScheduleType::NomergeUnorderedAuto:
133 case OMPScheduleType::NomergeUnorderedTrapezoidal:
134 case OMPScheduleType::NomergeUnorderedGreedy:
135 case OMPScheduleType::NomergeUnorderedBalanced:
136 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
137 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
138 case OMPScheduleType::NomergeUnorderedSteal:
139 case OMPScheduleType::NomergeOrderedStaticChunked:
140 case OMPScheduleType::NomergeOrderedStatic:
141 case OMPScheduleType::NomergeOrderedDynamicChunked:
142 case OMPScheduleType::NomergeOrderedGuidedChunked:
143 case OMPScheduleType::NomergeOrderedRuntime:
144 case OMPScheduleType::NomergeOrderedAuto:
145 case OMPScheduleType::NomergeOrderedTrapezoidal:
146 case OMPScheduleType::OrderedDistributeChunked:
147 case OMPScheduleType::OrderedDistribute:
148 break;
149 default:
150 return false;
151 }
152
153 // Must not set both monotonicity modifiers at the same time.
154 OMPScheduleType MonotonicityFlags =
155 SchedType & OMPScheduleType::MonotonicityMask;
156 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
157 return false;
158
159 return true;
160}
161#endif
162
163/// This is a wrapper over IRBuilderBase::restoreIP that also restores a current
164/// debug location when the insert point is at the end of a block. It picks a
165/// location scoped to the current function: the block's last instruction
166/// location if the block is non-empty, otherwise a location synthesized from
167/// the function's subprogram (when the function has debug info).
168static void restoreIPandDebugLoc(llvm::IRBuilderBase &Builder,
169 llvm::IRBuilderBase::InsertPoint IP) {
170 Builder.restoreIP(IP);
171 // When IP points at a real instruction, restoreIP (SetInsertPoint) already
172 // set the debug location from that instruction, so leave it alone.
173 llvm::BasicBlock *BB = Builder.GetInsertBlock();
174 if (Builder.GetInsertPoint() != BB->end())
175 return;
176
177 // At the end of a block, pick a location guaranteed to belong to the current
178 // insertion function's subprogram. Prefer the block's own last instruction;
179 // otherwise synthesize a location from the function's subprogram.
180 if (!BB->empty())
181 Builder.SetCurrentDebugLocation(BB->back().getDebugLoc());
182 else if (llvm::DISubprogram *FSP =
183 BB->getParent() ? BB->getParent()->getSubprogram() : nullptr) {
184 unsigned Line = FSP->getScopeLine() ? FSP->getScopeLine() : FSP->getLine();
185 Builder.SetCurrentDebugLocation(
186 llvm::DILocation::get(Context&: FSP->getContext(), Line, /*Column=*/0, Scope: FSP));
187 }
188}
189
190static bool hasGridValue(const Triple &T) {
191 return T.isAMDGPU() || T.isNVPTX() || T.isSPIRV();
192}
193
194static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
195 if (T.isAMDGPU()) {
196 StringRef Features =
197 Kernel->getFnAttribute(Kind: "target-features").getValueAsString();
198 if (Features.count(Str: "+wavefrontsize64"))
199 return omp::getAMDGPUGridValues<64>();
200 return omp::getAMDGPUGridValues<32>();
201 }
202 if (T.isNVPTX())
203 return omp::NVPTXGridValues;
204 if (T.isSPIRV())
205 return omp::SPIRVGridValues;
206 llvm_unreachable("No grid value available for this architecture!");
207}
208
209/// Determine which scheduling algorithm to use, determined from schedule clause
210/// arguments.
211static OMPScheduleType
212getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
213 bool HasSimdModifier, bool HasDistScheduleChunks) {
214 // Currently, the default schedule it static.
215 switch (ClauseKind) {
216 case OMP_SCHEDULE_Default:
217 case OMP_SCHEDULE_Static:
218 return HasChunks ? OMPScheduleType::BaseStaticChunked
219 : OMPScheduleType::BaseStatic;
220 case OMP_SCHEDULE_Dynamic:
221 return OMPScheduleType::BaseDynamicChunked;
222 case OMP_SCHEDULE_Guided:
223 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
224 : OMPScheduleType::BaseGuidedChunked;
225 case OMP_SCHEDULE_Auto:
226 return llvm::omp::OMPScheduleType::BaseAuto;
227 case OMP_SCHEDULE_Runtime:
228 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
229 : OMPScheduleType::BaseRuntime;
230 case OMP_SCHEDULE_Distribute:
231 return HasDistScheduleChunks ? OMPScheduleType::BaseDistributeChunked
232 : OMPScheduleType::BaseDistribute;
233 }
234 llvm_unreachable("unhandled schedule clause argument");
235}
236
237/// Adds ordering modifier flags to schedule type.
238static OMPScheduleType
239getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
240 bool HasOrderedClause) {
241 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
242 OMPScheduleType::None &&
243 "Must not have ordering nor monotonicity flags already set");
244
245 OMPScheduleType OrderingModifier = HasOrderedClause
246 ? OMPScheduleType::ModifierOrdered
247 : OMPScheduleType::ModifierUnordered;
248 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
249
250 // Unsupported combinations
251 if (OrderingScheduleType ==
252 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
253 return OMPScheduleType::OrderedGuidedChunked;
254 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
255 OMPScheduleType::ModifierOrdered))
256 return OMPScheduleType::OrderedRuntime;
257
258 return OrderingScheduleType;
259}
260
261/// Adds monotonicity modifier flags to schedule type.
262static OMPScheduleType
263getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
264 bool HasSimdModifier, bool HasMonotonic,
265 bool HasNonmonotonic, bool HasOrderedClause) {
266 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
267 OMPScheduleType::None &&
268 "Must not have monotonicity flags already set");
269 assert((!HasMonotonic || !HasNonmonotonic) &&
270 "Monotonic and Nonmonotonic are contradicting each other");
271
272 if (HasMonotonic) {
273 return ScheduleType | OMPScheduleType::ModifierMonotonic;
274 } else if (HasNonmonotonic) {
275 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
276 } else {
277 // OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
278 // If the static schedule kind is specified or if the ordered clause is
279 // specified, and if the nonmonotonic modifier is not specified, the
280 // effect is as if the monotonic modifier is specified. Otherwise, unless
281 // the monotonic modifier is specified, the effect is as if the
282 // nonmonotonic modifier is specified.
283 OMPScheduleType BaseScheduleType =
284 ScheduleType & ~OMPScheduleType::ModifierMask;
285 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
286 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
287 HasOrderedClause) {
288 // The monotonic is used by default in openmp runtime library, so no need
289 // to set it.
290 return ScheduleType;
291 } else {
292 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
293 }
294 }
295}
296
297/// Determine the schedule type using schedule and ordering clause arguments.
298static OMPScheduleType
299computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
300 bool HasSimdModifier, bool HasMonotonicModifier,
301 bool HasNonmonotonicModifier, bool HasOrderedClause,
302 bool HasDistScheduleChunks) {
303 OMPScheduleType BaseSchedule = getOpenMPBaseScheduleType(
304 ClauseKind, HasChunks, HasSimdModifier, HasDistScheduleChunks);
305 OMPScheduleType OrderedSchedule =
306 getOpenMPOrderingScheduleType(BaseScheduleType: BaseSchedule, HasOrderedClause);
307 OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
308 ScheduleType: OrderedSchedule, HasSimdModifier, HasMonotonic: HasMonotonicModifier,
309 HasNonmonotonic: HasNonmonotonicModifier, HasOrderedClause);
310
311 assert(isValidWorkshareLoopScheduleType(Result));
312 return Result;
313}
314
315/// Given a function, if it represents the entry point of a target kernel, this
316/// returns the execution mode flags associated with that kernel.
317static std::optional<omp::OMPTgtExecModeFlags>
318getTargetKernelExecMode(Function &Kernel) {
319 CallInst *TargetInitCall = nullptr;
320 for (Instruction &Inst : Kernel.getEntryBlock()) {
321 if (auto *Call = dyn_cast<CallInst>(Val: &Inst)) {
322 if (Call->getCalledFunction()->getName() == "__kmpc_target_init") {
323 TargetInitCall = Call;
324 break;
325 }
326 }
327 }
328
329 if (!TargetInitCall)
330 return std::nullopt;
331
332 // Get the kernel mode information from the global variable associated to the
333 // first argument to the call to __kmpc_target_init. Refer to
334 // createTargetInit() to see how this is initialized.
335 Value *InitOperand = TargetInitCall->getArgOperand(i: 0);
336 GlobalVariable *KernelEnv = nullptr;
337 if (auto *Cast = dyn_cast<ConstantExpr>(Val: InitOperand))
338 KernelEnv = cast<GlobalVariable>(Val: Cast->getOperand(i_nocapture: 0));
339 else
340 KernelEnv = cast<GlobalVariable>(Val: InitOperand);
341 auto *KernelEnvInit = cast<ConstantStruct>(Val: KernelEnv->getInitializer());
342 auto *ConfigEnv = cast<ConstantStruct>(Val: KernelEnvInit->getOperand(i_nocapture: 0));
343 auto *KernelMode = cast<ConstantInt>(Val: ConfigEnv->getOperand(i_nocapture: 2));
344 return static_cast<OMPTgtExecModeFlags>(KernelMode->getZExtValue());
345}
346
347static bool isGenericKernel(Function &Fn) {
348 std::optional<omp::OMPTgtExecModeFlags> ExecMode =
349 getTargetKernelExecMode(Kernel&: Fn);
350 return !ExecMode || (*ExecMode & OMP_TGT_EXEC_MODE_GENERIC);
351}
352
353/// Make \p Source branch to \p Target.
354///
355/// Handles two situations:
356/// * \p Source already has an unconditional branch.
357/// * \p Source is a degenerate block (no terminator because the BB is
358/// the current head of the IR construction).
359static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
360 if (Instruction *Term = Source->getTerminatorOrNull()) {
361 auto *Br = cast<UncondBrInst>(Val: Term);
362 BasicBlock *Succ = Br->getSuccessor();
363 Succ->removePredecessor(Pred: Source, /*KeepOneInputPHIs=*/true);
364 Br->setSuccessor(Target);
365 return;
366 }
367
368 auto *NewBr = UncondBrInst::Create(Target, InsertBefore: Source);
369 NewBr->setDebugLoc(DL);
370}
371
372void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
373 bool CreateBranch, DebugLoc DL) {
374 assert(New->getFirstInsertionPt() == New->begin() &&
375 "Target BB must not have PHI nodes");
376
377 // Move instructions to new block.
378 BasicBlock *Old = IP.getNodeParent();
379 // If the `Old` block is empty then there are no instructions to move. But in
380 // the new debug scheme, it could have trailing debug records which will be
381 // moved to `New` in `spliceDebugInfoEmptyBlock`. We dont want that for 2
382 // reasons:
383 // 1. If `New` is also empty, `BasicBlock::splice` crashes.
384 // 2. Even if `New` is not empty, the rationale to move those records to `New`
385 // (in `spliceDebugInfoEmptyBlock`) does not apply here. That function
386 // assumes that `Old` is optimized out and is going away. This is not the case
387 // here. The `Old` block is still being used e.g. a branch instruction is
388 // added to it later in this function.
389 // So we call `BasicBlock::splice` only when `Old` is not empty.
390 if (!Old->empty())
391 New->splice(ToIt: New->begin(), FromBB: Old, FromBeginIt: IP, FromEndIt: Old->end());
392
393 if (CreateBranch) {
394 auto *NewBr = UncondBrInst::Create(Target: New, InsertBefore: Old);
395 NewBr->setDebugLoc(DL);
396 }
397}
398
399void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
400 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
401 BasicBlock *Old = Builder.GetInsertBlock();
402
403 spliceBB(IP: Builder.saveIP(), New, CreateBranch, DL: DebugLoc);
404 if (CreateBranch)
405 Builder.SetInsertPoint(Old->getTerminator());
406 else
407 Builder.SetInsertPoint(Old);
408
409 // SetInsertPoint also updates the Builder's debug location, but we want to
410 // keep the one the Builder was configured to use.
411 Builder.SetCurrentDebugLocation(DebugLoc);
412}
413
414BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
415 DebugLoc DL, llvm::Twine Name) {
416 BasicBlock *Old = IP.getNodeParent();
417 BasicBlock *New = BasicBlock::Create(
418 Context&: Old->getContext(), Name: Name.isTriviallyEmpty() ? Old->getName() : Name,
419 Parent: Old->getParent(), InsertBefore: Old->getNextNode());
420 spliceBB(IP, New, CreateBranch, DL);
421 New->replaceSuccessorsPhiUsesWith(Old, New);
422 return New;
423}
424
425BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
426 llvm::Twine Name) {
427 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
428 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
429 if (CreateBranch)
430 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
431 else
432 Builder.SetInsertPoint(Builder.GetInsertBlock());
433 // SetInsertPoint also updates the Builder's debug location, but we want to
434 // keep the one the Builder was configured to use.
435 Builder.SetCurrentDebugLocation(DebugLoc);
436 return New;
437}
438
439BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
440 llvm::Twine Name) {
441 DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
442 BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, DL: DebugLoc, Name);
443 if (CreateBranch)
444 Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
445 else
446 Builder.SetInsertPoint(Builder.GetInsertBlock());
447 // SetInsertPoint also updates the Builder's debug location, but we want to
448 // keep the one the Builder was configured to use.
449 Builder.SetCurrentDebugLocation(DebugLoc);
450 return New;
451}
452
453BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
454 llvm::Twine Suffix) {
455 BasicBlock *Old = Builder.GetInsertBlock();
456 return splitBB(Builder, CreateBranch, Name: Old->getName() + Suffix);
457}
458
459// This function creates a fake integer value and a fake use for the integer
460// value. It returns the fake value created. This is useful in modeling the
461// extra arguments to the outlined functions.
462Value *createFakeIntVal(IRBuilderBase &Builder,
463 OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
464 llvm::SmallVectorImpl<Instruction *> &ToBeDeleted,
465 OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
466 const Twine &Name = "", bool AsPtr = true,
467 bool Is64Bit = false) {
468 Builder.restoreIP(IP: OuterAllocaIP);
469 IntegerType *IntTy = Is64Bit ? Builder.getInt64Ty() : Builder.getInt32Ty();
470 Instruction *FakeVal;
471 AllocaInst *FakeValAddr =
472 Builder.CreateAlloca(Ty: IntTy, ArraySize: nullptr, Name: Name + ".addr");
473 ToBeDeleted.push_back(Elt: FakeValAddr);
474
475 if (AsPtr) {
476 FakeVal = FakeValAddr;
477 // The runtime passes these extra arguments to the outlined function as
478 // generic pointers, so cast away a non-zero alloca address space.
479 if (FakeValAddr->getAddressSpace() != 0) {
480 FakeVal = cast<Instruction>(Val: Builder.CreateAddrSpaceCast(
481 V: FakeValAddr, DestTy: Builder.getPtrTy(), Name: Name + ".ascast"));
482 ToBeDeleted.push_back(Elt: FakeVal);
483 }
484 } else {
485 FakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeValAddr, Name: Name + ".val");
486 ToBeDeleted.push_back(Elt: FakeVal);
487 }
488
489 // Generate a fake use of this value
490 Builder.restoreIP(IP: InnerAllocaIP);
491 Instruction *UseFakeVal;
492 if (AsPtr) {
493 UseFakeVal = Builder.CreateLoad(Ty: IntTy, Ptr: FakeVal, Name: Name + ".use");
494 } else {
495 UseFakeVal = cast<BinaryOperator>(Val: Builder.CreateAdd(
496 LHS: FakeVal, RHS: Is64Bit ? Builder.getInt64(C: 10) : Builder.getInt32(C: 10)));
497 }
498 ToBeDeleted.push_back(Elt: UseFakeVal);
499 return FakeVal;
500}
501
502//===----------------------------------------------------------------------===//
503// OpenMPIRBuilderConfig
504//===----------------------------------------------------------------------===//
505
506namespace {
507LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
508/// Values for bit flags for marking which requires clauses have been used.
509enum OpenMPOffloadingRequiresDirFlags {
510 /// flag undefined.
511 OMP_REQ_UNDEFINED = 0x000,
512 /// no requires directive present.
513 OMP_REQ_NONE = 0x001,
514 /// reverse_offload clause.
515 OMP_REQ_REVERSE_OFFLOAD = 0x002,
516 /// unified_address clause.
517 OMP_REQ_UNIFIED_ADDRESS = 0x004,
518 /// unified_shared_memory clause.
519 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
520 /// dynamic_allocators clause.
521 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
522 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
523};
524
525class OMPCodeExtractor : public CodeExtractor {
526public:
527 OMPCodeExtractor(OpenMPIRBuilder &OMPBuilder, ArrayRef<BasicBlock *> BBs,
528 DominatorTree *DT = nullptr, bool AggregateArgs = false,
529 BlockFrequencyInfo *BFI = nullptr,
530 BranchProbabilityInfo *BPI = nullptr,
531 AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
532 bool AllowAlloca = false,
533 BasicBlock *AllocationBlock = nullptr,
534 ArrayRef<BasicBlock *> DeallocationBlocks = {},
535 std::string Suffix = "", bool ArgsInZeroAddressSpace = false)
536 : CodeExtractor(BBs, DT, AggregateArgs, BFI, BPI, AC, AllowVarArgs,
537 AllowAlloca, AllocationBlock, DeallocationBlocks, Suffix,
538 ArgsInZeroAddressSpace),
539 OMPBuilder(OMPBuilder) {}
540
541 virtual ~OMPCodeExtractor() = default;
542
543protected:
544 OpenMPIRBuilder &OMPBuilder;
545};
546
547class DeviceSharedMemCodeExtractor : public OMPCodeExtractor {
548public:
549 using OMPCodeExtractor::OMPCodeExtractor;
550 virtual ~DeviceSharedMemCodeExtractor() = default;
551
552protected:
553 virtual Instruction *
554 allocateVar(IRBuilder<>::InsertPoint AllocaIP, DebugLoc DL, Type *VarType,
555 const Twine &Name = Twine(""),
556 AddrSpaceCastInst **CastedAlloc = nullptr) override {
557 return OMPBuilder.createOMPAllocShared(Loc: {AllocaIP, DL}, VarType, Name);
558 }
559
560 virtual Instruction *deallocateVar(IRBuilder<>::InsertPoint DeallocIP,
561 DebugLoc DL, Value *Var,
562 Type *VarType) override {
563 return OMPBuilder.createOMPFreeShared(Loc: {DeallocIP, DL}, Addr: Var, VarType);
564 }
565};
566
567/// Helper storing information about regions to outline using device shared
568/// memory for intermediate allocations.
569struct DeviceSharedMemOutlineInfo : public OpenMPIRBuilder::OutlineInfo {
570 OpenMPIRBuilder &OMPBuilder;
571
572 DeviceSharedMemOutlineInfo(OpenMPIRBuilder &OMPBuilder)
573 : OMPBuilder(OMPBuilder) {}
574 virtual ~DeviceSharedMemOutlineInfo() = default;
575
576 virtual std::unique_ptr<CodeExtractor>
577 createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
578 bool ArgsInZeroAddressSpace,
579 Twine Suffix = Twine("")) override;
580};
581
582} // anonymous namespace
583
584OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()
585 : RequiresFlags(OMP_REQ_UNDEFINED) {}
586
587OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(
588 bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,
589 bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,
590 bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)
591 : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
592 OpenMPOffloadMandatory(OpenMPOffloadMandatory),
593 RequiresFlags(OMP_REQ_UNDEFINED) {
594 if (HasRequiresReverseOffload)
595 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
596 if (HasRequiresUnifiedAddress)
597 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
598 if (HasRequiresUnifiedSharedMemory)
599 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
600 if (HasRequiresDynamicAllocators)
601 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
602}
603
604bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {
605 return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
606}
607
608bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {
609 return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
610}
611
612bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {
613 return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
614}
615
616bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {
617 return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
618}
619
620int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {
621 return hasRequiresFlags() ? RequiresFlags
622 : static_cast<int64_t>(OMP_REQ_NONE);
623}
624
625void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {
626 if (Value)
627 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
628 else
629 RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
630}
631
632void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {
633 if (Value)
634 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
635 else
636 RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
637}
638
639void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {
640 if (Value)
641 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
642 else
643 RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
644}
645
646void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {
647 if (Value)
648 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
649 else
650 RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
651}
652
653//===----------------------------------------------------------------------===//
654// OpenMPIRBuilder
655//===----------------------------------------------------------------------===//
656
657void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
658 IRBuilderBase &Builder,
659 SmallVector<Value *> &ArgsVector) {
660 Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
661 Value *PointerNum = Builder.getInt32(C: KernelArgs.NumTargetItems);
662 auto Int32Ty = Type::getInt32Ty(C&: Builder.getContext());
663 constexpr size_t MaxDim = 3;
664 Value *ZeroArray = Constant::getNullValue(Ty: ArrayType::get(ElementType: Int32Ty, NumElements: MaxDim));
665
666 Value *HasNoWaitFlag = Builder.getInt64(C: KernelArgs.HasNoWait);
667
668 Value *DynCGroupMemFallbackFlag =
669 Builder.getInt64(C: static_cast<uint64_t>(KernelArgs.DynCGroupMemFallback));
670 DynCGroupMemFallbackFlag = Builder.CreateShl(LHS: DynCGroupMemFallbackFlag, RHS: 2);
671
672 Value *StrictBlocksFlag = Builder.getInt64(C: KernelArgs.StrictBlocks);
673 Value *StrictThreadsFlag = Builder.getInt64(C: KernelArgs.StrictThreads);
674
675 StrictBlocksFlag = Builder.CreateShl(LHS: StrictBlocksFlag, RHS: 6);
676 StrictThreadsFlag = Builder.CreateShl(LHS: StrictThreadsFlag, RHS: 7);
677
678 Value *Flags = Builder.CreateOr(LHS: HasNoWaitFlag, RHS: DynCGroupMemFallbackFlag);
679 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictBlocksFlag);
680 Flags = Builder.CreateOr(LHS: Flags, RHS: StrictThreadsFlag);
681
682 assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());
683
684 Value *NumTeams3D =
685 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumTeams[0], Idxs: {0});
686 Value *NumThreads3D =
687 Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumThreads[0], Idxs: {0});
688 for (unsigned I :
689 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumTeams.size(), b: MaxDim)))
690 NumTeams3D =
691 Builder.CreateInsertValue(Agg: NumTeams3D, Val: KernelArgs.NumTeams[I], Idxs: {I});
692 for (unsigned I :
693 seq<unsigned>(Begin: 1, End: std::min(a: KernelArgs.NumThreads.size(), b: MaxDim)))
694 NumThreads3D =
695 Builder.CreateInsertValue(Agg: NumThreads3D, Val: KernelArgs.NumThreads[I], Idxs: {I});
696
697 ArgsVector = {Version,
698 PointerNum,
699 KernelArgs.RTArgs.BasePointersArray,
700 KernelArgs.RTArgs.PointersArray,
701 KernelArgs.RTArgs.SizesArray,
702 KernelArgs.RTArgs.MapTypesArray,
703 KernelArgs.RTArgs.MapNamesArray,
704 KernelArgs.RTArgs.MappersArray,
705 KernelArgs.NumIterations,
706 Flags,
707 NumTeams3D,
708 NumThreads3D,
709 KernelArgs.DynCGroupMem};
710}
711
712void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
713 LLVMContext &Ctx = Fn.getContext();
714
715 // Get the function's current attributes.
716 auto Attrs = Fn.getAttributes();
717 auto FnAttrs = Attrs.getFnAttrs();
718 auto RetAttrs = Attrs.getRetAttrs();
719 SmallVector<AttributeSet, 4> ArgAttrs;
720 for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
721 ArgAttrs.emplace_back(Args: Attrs.getParamAttrs(ArgNo));
722
723 // Add AS to FnAS while taking special care with integer extensions.
724 auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,
725 bool Param = true) -> void {
726 bool HasSignExt = AS.hasAttribute(Kind: Attribute::SExt);
727 bool HasZeroExt = AS.hasAttribute(Kind: Attribute::ZExt);
728 if (HasSignExt || HasZeroExt) {
729 assert(AS.getNumAttributes() == 1 &&
730 "Currently not handling extension attr combined with others.");
731 if (Param) {
732 if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, Signed: HasSignExt))
733 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
734 } else if (auto AK =
735 TargetLibraryInfo::getExtAttrForI32Return(T, Signed: HasSignExt))
736 FnAS = FnAS.addAttribute(C&: Ctx, Kind: AK);
737 } else {
738 FnAS = FnAS.addAttributes(C&: Ctx, AS);
739 }
740 };
741
742#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
743#include "llvm/Frontend/OpenMP/OMPKinds.def"
744
745 // Add attributes to the function declaration.
746 switch (FnID) {
747#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
748 case Enum: \
749 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
750 addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false); \
751 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
752 addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
753 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
754 break;
755#include "llvm/Frontend/OpenMP/OMPKinds.def"
756 default:
757 // Attributes are optional.
758 break;
759 }
760}
761
762FunctionCallee
763OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
764 FunctionType *FnTy = nullptr;
765 Function *Fn = nullptr;
766
767 // Try to find the declation in the module first.
768 switch (FnID) {
769#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
770 case Enum: \
771 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
772 IsVarArg); \
773 Fn = M.getFunction(Str); \
774 break;
775#include "llvm/Frontend/OpenMP/OMPKinds.def"
776 }
777
778 if (!Fn) {
779 // Create a new declaration if we need one.
780 switch (FnID) {
781#define OMP_RTL(Enum, Str, ...) \
782 case Enum: \
783 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
784 break;
785#include "llvm/Frontend/OpenMP/OMPKinds.def"
786 }
787 Fn->setCallingConv(Config.getRuntimeCC());
788 // Add information if the runtime function takes a callback function
789 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
790 if (!Fn->hasMetadata(KindID: LLVMContext::MD_callback)) {
791 LLVMContext &Ctx = Fn->getContext();
792 MDBuilder MDB(Ctx);
793 // Annotate the callback behavior of the runtime function:
794 // - The callback callee is argument number 2 (microtask).
795 // - The first two arguments of the callback callee are unknown (-1).
796 // - All variadic arguments to the runtime function are passed to the
797 // callback callee.
798 Fn->addMetadata(
799 KindID: LLVMContext::MD_callback,
800 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
801 CalleeArgNo: 2, Arguments: {-1, -1}, /* VarArgsArePassed */ true)}));
802 }
803 }
804
805 LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()
806 << " with type " << *Fn->getFunctionType() << "\n");
807 addAttributes(FnID, Fn&: *Fn);
808
809 } else {
810 LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()
811 << " with type " << *Fn->getFunctionType() << "\n");
812 }
813
814 assert(Fn && "Failed to create OpenMP runtime function");
815
816 return {FnTy, Fn};
817}
818
819Expected<BasicBlock *>
820OpenMPIRBuilder::FinalizationInfo::getFiniBB(IRBuilderBase &Builder) {
821 if (!FiniBB) {
822 Function *ParentFunc = Builder.GetInsertBlock()->getParent();
823 IRBuilderBase::InsertPointGuard Guard(Builder);
824 FiniBB = BasicBlock::Create(Context&: Builder.getContext(), Name: ".fini", Parent: ParentFunc);
825 Builder.SetInsertPoint(FiniBB);
826 // FiniCB adds the branch to the exit stub.
827 if (Error Err = FiniCB(Builder.saveIP()))
828 return Err;
829 }
830 return FiniBB;
831}
832
833Error OpenMPIRBuilder::FinalizationInfo::mergeFiniBB(IRBuilderBase &Builder,
834 BasicBlock *OtherFiniBB) {
835 // Simple case: FiniBB does not exist yet: re-use OtherFiniBB.
836 if (!FiniBB) {
837 FiniBB = OtherFiniBB;
838
839 Builder.SetInsertPoint(FiniBB->getFirstNonPHIIt());
840 if (Error Err = FiniCB(Builder.saveIP()))
841 return Err;
842
843 return Error::success();
844 }
845
846 // Move instructions from FiniBB to the start of OtherFiniBB.
847 auto EndIt = FiniBB->end();
848 if (FiniBB->size() >= 1)
849 if (auto Prev = std::prev(x: EndIt); Prev->isTerminator())
850 EndIt = Prev;
851 OtherFiniBB->splice(ToIt: OtherFiniBB->getFirstNonPHIIt(), FromBB: FiniBB, FromBeginIt: FiniBB->begin(),
852 FromEndIt: EndIt);
853
854 FiniBB->replaceAllUsesWith(V: OtherFiniBB);
855 FiniBB->eraseFromParent();
856 FiniBB = OtherFiniBB;
857 return Error::success();
858}
859
860Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
861 FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
862 auto *Fn = dyn_cast<llvm::Function>(Val: RTLFn.getCallee());
863 assert(Fn && "Failed to create OpenMP runtime function pointer");
864 return Fn;
865}
866
867CallInst *OpenMPIRBuilder::createRuntimeFunctionCall(FunctionCallee Callee,
868 ArrayRef<Value *> Args,
869 StringRef Name) {
870 CallInst *Call = Builder.CreateCall(Callee, Args, Name);
871 Call->setCallingConv(Config.getRuntimeCC());
872 return Call;
873}
874
875void OpenMPIRBuilder::initialize() { initializeTypes(M); }
876
877static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder,
878 Function *Function) {
879 BasicBlock &EntryBlock = Function->getEntryBlock();
880 BasicBlock::iterator MoveLocInst = EntryBlock.getFirstNonPHIIt();
881
882 // Loop over blocks looking for constant allocas, skipping the entry block
883 // as any allocas there are already in the desired location.
884 for (auto Block = std::next(x: Function->begin(), n: 1); Block != Function->end();
885 Block++) {
886 for (auto Inst = Block->getReverseIterator()->begin();
887 Inst != Block->getReverseIterator()->end();) {
888 if (auto *AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Val&: Inst)) {
889 Inst++;
890 if (!isa<ConstantData>(Val: AllocaInst->getArraySize()))
891 continue;
892 AllocaInst->moveBeforePreserving(MovePos: MoveLocInst);
893 } else {
894 Inst++;
895 }
896 }
897 }
898}
899
900static void hoistNonEntryAllocasToEntryBlock(llvm::BasicBlock &Block) {
901 llvm::SmallVector<llvm::Instruction *> AllocasToMove;
902
903 auto ShouldHoistAlloca = [](const llvm::AllocaInst &AllocaInst) {
904 // TODO: For now, we support simple static allocations, we might need to
905 // move non-static ones as well. However, this will need further analysis to
906 // move the lenght arguments as well.
907 return !AllocaInst.isArrayAllocation();
908 };
909
910 for (llvm::Instruction &Inst : Block)
911 if (auto *AllocaInst = llvm::dyn_cast<llvm::AllocaInst>(Val: &Inst))
912 if (ShouldHoistAlloca(*AllocaInst))
913 AllocasToMove.push_back(Elt: AllocaInst);
914
915 auto InsertPoint =
916 Block.getParent()->getEntryBlock().getTerminator()->getIterator();
917
918 for (llvm::Instruction *AllocaInst : AllocasToMove)
919 AllocaInst->moveBefore(InsertPos: InsertPoint);
920}
921
922static void hoistNonEntryAllocasToEntryBlock(llvm::Function *Func) {
923 PostDominatorTree PostDomTree(*Func);
924 for (llvm::BasicBlock &BB : *Func)
925 if (PostDomTree.properlyDominates(A: &BB, B: &Func->getEntryBlock()))
926 hoistNonEntryAllocasToEntryBlock(Block&: BB);
927}
928
929void OpenMPIRBuilder::finalize(Function *Fn) {
930 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
931 SmallVector<BasicBlock *, 32> Blocks;
932 SmallVector<std::unique_ptr<OutlineInfo>, 16> DeferredOutlines;
933 for (std::unique_ptr<OutlineInfo> &OI : OutlineInfos) {
934 // Skip functions that have not finalized yet; may happen with nested
935 // function generation.
936 if (Fn && OI->getFunction() != Fn) {
937 DeferredOutlines.push_back(Elt: std::move(OI));
938 continue;
939 }
940
941 ParallelRegionBlockSet.clear();
942 Blocks.clear();
943 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
944
945 Function *OuterFn = OI->getFunction();
946 CodeExtractorAnalysisCache CEAC(*OuterFn);
947 // If we generate code for the target device, we need to allocate
948 // struct for aggregate params in the device default alloca address space.
949 // OpenMP runtime requires that the params of the extracted functions are
950 // passed as zero address space pointers. This flag ensures that
951 // CodeExtractor generates correct code for extracted functions
952 // which are used by OpenMP runtime.
953 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
954 std::unique_ptr<CodeExtractor> Extractor =
955 OI->createCodeExtractor(Blocks, ArgsInZeroAddressSpace, Suffix: ".omp_par");
956
957 LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n");
958 LLVM_DEBUG(dbgs() << "Entry " << OI->EntryBB->getName()
959 << " Exit: " << OI->ExitBB->getName() << "\n");
960 assert(Extractor->isEligible() &&
961 "Expected OpenMP outlining to be possible!");
962
963 for (auto *V : OI->ExcludeArgsFromAggregate)
964 Extractor->excludeArgFromAggregate(Arg: V);
965
966 Function *OutlinedFn =
967 Extractor->extractCodeRegion(CEAC, Inputs&: OI->Inputs, Outputs&: OI->Outputs);
968
969 // Forward target-cpu, target-features attributes to the outlined function.
970 auto TargetCpuAttr = OuterFn->getFnAttribute(Kind: "target-cpu");
971 if (TargetCpuAttr.isStringAttribute())
972 OutlinedFn->addFnAttr(Attr: TargetCpuAttr);
973
974 auto TargetFeaturesAttr = OuterFn->getFnAttribute(Kind: "target-features");
975 if (TargetFeaturesAttr.isStringAttribute())
976 OutlinedFn->addFnAttr(Attr: TargetFeaturesAttr);
977
978 LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n");
979 LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n");
980 assert(OutlinedFn->getReturnType()->isVoidTy() &&
981 "OpenMP outlined functions should not return a value!");
982
983 // For compability with the clang CG we move the outlined function after the
984 // one with the parallel region.
985 OutlinedFn->removeFromParent();
986 M.getFunctionList().insertAfter(where: OuterFn->getIterator(), New: OutlinedFn);
987
988 // Remove the artificial entry introduced by the extractor right away, we
989 // made our own entry block after all.
990 {
991 BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
992 assert(ArtificialEntry.getUniqueSuccessor() == OI->EntryBB);
993 assert(OI->EntryBB->getUniquePredecessor() == &ArtificialEntry);
994 // Move instructions from the to-be-deleted ArtificialEntry to the entry
995 // basic block of the parallel region. CodeExtractor generates
996 // instructions to unwrap the aggregate argument and may sink
997 // allocas/bitcasts for values that are solely used in the outlined region
998 // and do not escape.
999 assert(!ArtificialEntry.empty() &&
1000 "Expected instructions to add in the outlined region entry");
1001 for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
1002 End = ArtificialEntry.rend();
1003 It != End;) {
1004 Instruction &I = *It;
1005 It++;
1006
1007 if (I.isTerminator()) {
1008 // Absorb any debug value that terminator may have
1009 if (Instruction *TI = OI->EntryBB->getTerminatorOrNull())
1010 TI->adoptDbgRecords(BB: &ArtificialEntry, It: I.getIterator(), InsertAtHead: false);
1011 continue;
1012 }
1013
1014 I.moveBeforePreserving(BB&: *OI->EntryBB,
1015 I: OI->EntryBB->getFirstInsertionPt());
1016 }
1017
1018 OI->EntryBB->moveBefore(MovePos: &ArtificialEntry);
1019 ArtificialEntry.eraseFromParent();
1020 }
1021 assert(&OutlinedFn->getEntryBlock() == OI->EntryBB);
1022 assert(OutlinedFn && OutlinedFn->hasNUses(1));
1023
1024 // Run a user callback, e.g. to add attributes.
1025 if (OI->PostOutlineCB)
1026 OI->PostOutlineCB(*OutlinedFn);
1027
1028 if (OI->FixUpNonEntryAllocas)
1029 hoistNonEntryAllocasToEntryBlock(Func: OutlinedFn);
1030 }
1031
1032 // Remove work items that have been completed.
1033 OutlineInfos = std::move(DeferredOutlines);
1034
1035 // The createTarget functions embeds user written code into
1036 // the target region which may inject allocas which need to
1037 // be moved to the entry block of our target or risk malformed
1038 // optimisations by later passes (however, we do not block on
1039 // that here, it's up to the inserter to the list to do so).
1040 // This notbaly has to occur after the OutlinedInfo candidates
1041 // have been extracted so we have an end product that will not
1042 // be implicitly adversely affected by any raises unless
1043 // intentionally appended to the list.
1044 // NOTE: This only does so for ConstantData, it could be extended
1045 // to ConstantExpr's with further effort, however, they should
1046 // largely be folded when they get here. Extending it to runtime
1047 // defined/read+writeable allocation sizes would be non-trivial
1048 // (need to factor in movement of any stores to variables the
1049 // allocation size depends on, as well as the usual loads,
1050 // otherwise it'll yield the wrong result after movement) and
1051 // likely be more suitable as an LLVM optimisation pass.
1052 for (Function *F : ConstantAllocaRaiseCandidates)
1053 raiseUserConstantDataAllocasToEntryBlock(Builder, Function: F);
1054
1055 EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
1056 [](EmitMetadataErrorKind Kind,
1057 const TargetRegionEntryInfo &EntryInfo) -> void {
1058 errs() << "Error of kind: " << Kind
1059 << " when emitting offload entries and metadata during "
1060 "OMPIRBuilder finalization \n";
1061 };
1062
1063 if (!OffloadInfoManager.empty())
1064 createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
1065
1066 // Rewrite uses of globals to their replacement declare target globals if
1067 // we are processing a device module.
1068 if (Config.isTargetDevice())
1069 applyDeclareTargetGlobalReplacements();
1070
1071 if (Config.EmitLLVMUsedMetaInfo.value_or(u: false)) {
1072 std::vector<WeakTrackingVH> LLVMCompilerUsed = {
1073 M.getGlobalVariable(Name: "__openmp_nvptx_data_transfer_temporary_storage")};
1074 emitUsed(Name: "llvm.compiler.used", List: LLVMCompilerUsed);
1075 }
1076
1077 IsFinalized = true;
1078}
1079
1080bool OpenMPIRBuilder::isFinalized() { return IsFinalized; }
1081
1082void OpenMPIRBuilder::registerDeclareTargetGlobalReplacement(
1083 GlobalValue *Original, GlobalValue *Replacement) {
1084 assert(Original && Replacement &&
1085 "Null values provided to registerDeclareTargetGlobalReplacement");
1086 DeclareTargetGlobalReplacements.push_back(Elt: {.Original: Original, .Replacement: Replacement});
1087}
1088
1089void OpenMPIRBuilder::applyDeclareTargetGlobalReplacements() {
1090 for (DeclareTargetGlobalReplacement &R : DeclareTargetGlobalReplacements) {
1091 GlobalValue *OldGV = R.Original;
1092 GlobalValue *NewGV = R.Replacement;
1093
1094 assert(OldGV && NewGV &&
1095 "A null value was inserted into DeclareTargetGlobalReplacements");
1096
1097 // The assert above should catch this case, but this is kept to attempt
1098 // to proceed without issue when asserts are off.
1099 if (!OldGV || !NewGV)
1100 continue;
1101
1102 // The replacement global is a reference pointer that holds the
1103 // address of the device-resident storage. Every use must load the
1104 // reference pointer first and use the loaded address.
1105 //
1106 // Constant expression users (e.g. a constant GEP embedded in another
1107 // global's initializer or in an instruction) cannot have a load inserted
1108 // in place, so first expand any constant-expression users that live inside
1109 // functions into instructions. Any remaining constant users are handled
1110 // via a direct constant rewrite below as we cannot materialize a load
1111 // there.
1112 //
1113 // NOTE: We extend the constant rewrite to module scope, as we replace all
1114 // usages.
1115 if (auto *OldConst = dyn_cast<Constant>(Val: OldGV))
1116 convertUsersOfConstantsToInstructions(Consts: OldConst,
1117 /*RestrictToFunc=*/nullptr,
1118 /*RemoveDeadConstants=*/false);
1119
1120 IRBuilderBase::InsertPointGuard Guard(Builder);
1121 SmallVector<User *, 16> Users(OldGV->users());
1122 for (User *U : Users) {
1123 auto *Insn = dyn_cast<Instruction>(Val: U);
1124 if (!Insn)
1125 continue;
1126
1127 // A PHI node cannot have a load inserted immediately before it, as PHIs
1128 // must remain grouped at the top of their basic block. So we need to
1129 // make sure any loads we emit are generated in the preceding edge, a
1130 // PHI may reference the global on more than one edge, so every matching
1131 // slot must be handled.
1132 if (auto *PHI = dyn_cast<PHINode>(Val: Insn)) {
1133 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
1134 if (PHI->getIncomingValue(i: I) != OldGV)
1135 continue;
1136
1137 BasicBlock *IncomingBB = PHI->getIncomingBlock(i: I);
1138 Builder.SetInsertPoint(IncomingBB->getTerminator());
1139 Builder.SetCurrentDebugLocation(PHI->getDebugLoc());
1140 LoadInst *EdgeLoad = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1141 PHI->setIncomingValue(i: I, V: EdgeLoad);
1142 }
1143 continue;
1144 }
1145
1146 Builder.SetInsertPoint(Insn);
1147 Builder.SetCurrentDebugLocation(Insn->getDebugLoc());
1148 LoadInst *Load = Builder.CreateLoad(Ty: NewGV->getType(), Ptr: NewGV);
1149
1150 // The replacement declare target global lives in the default address
1151 // space, whereas the original global may reside in a non-default
1152 // address space. In that case the initial lowering may have
1153 // emitted an addrspacecast that is no longer valid. Replace the
1154 // whole addrspacecast with the load and erase it rather than
1155 // feeding the load back into the (now pointless) cast.
1156 // NOTE: If we end up with replacement declare target globals in
1157 // non-zero AS's the below will need some minor extensions to have the
1158 // option to alter the address space cast to the new address space where
1159 // required rather than just replacing it.
1160 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: Insn)) {
1161 unsigned NewGVAS = NewGV->getType()->getPointerAddressSpace();
1162 assert(NewGVAS == 0 &&
1163 "Non-default address space declare target global");
1164 unsigned OldGVAS = OldGV->getType()->getPointerAddressSpace();
1165 unsigned DestAS = ASC->getType()->getPointerAddressSpace();
1166 if (DestAS == 0 && NewGVAS != OldGVAS) {
1167 ASC->replaceAllUsesWith(V: Load);
1168 ASC->eraseFromParent();
1169 continue;
1170 }
1171 }
1172
1173 Insn->replaceUsesOfWith(From: OldGV, To: Load);
1174 }
1175 }
1176
1177 DeclareTargetGlobalReplacements.clear();
1178}
1179
1180OpenMPIRBuilder::~OpenMPIRBuilder() {
1181 assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
1182}
1183
1184GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
1185 IntegerType *I32Ty = Type::getInt32Ty(C&: M.getContext());
1186 auto *GV =
1187 new GlobalVariable(M, I32Ty,
1188 /* isConstant = */ true, GlobalValue::WeakODRLinkage,
1189 ConstantInt::get(Ty: I32Ty, V: Value), Name);
1190 GV->setVisibility(GlobalValue::HiddenVisibility);
1191
1192 return GV;
1193}
1194
1195void OpenMPIRBuilder::emitUsed(StringRef Name, ArrayRef<WeakTrackingVH> List) {
1196 if (List.empty())
1197 return;
1198
1199 // Convert List to what ConstantArray needs.
1200 SmallVector<Constant *, 8> UsedArray;
1201 UsedArray.resize(N: List.size());
1202 for (unsigned I = 0, E = List.size(); I != E; ++I)
1203 UsedArray[I] = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1204 C: cast<Constant>(Val: &*List[I]), Ty: Builder.getPtrTy());
1205
1206 if (UsedArray.empty())
1207 return;
1208 ArrayType *ATy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: UsedArray.size());
1209
1210 auto *GV = new GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage,
1211 ConstantArray::get(T: ATy, V: UsedArray), Name);
1212
1213 GV->setSection("llvm.metadata");
1214}
1215
1216GlobalVariable *
1217OpenMPIRBuilder::emitKernelExecutionMode(StringRef KernelName,
1218 OMPTgtExecModeFlags Mode) {
1219 auto *Int8Ty = Builder.getInt8Ty();
1220 auto *GVMode = new GlobalVariable(
1221 M, Int8Ty, /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
1222 ConstantInt::get(Ty: Int8Ty, V: Mode), Twine(KernelName, "_exec_mode"));
1223 GVMode->setVisibility(GlobalVariable::ProtectedVisibility);
1224 return GVMode;
1225}
1226
1227Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
1228 uint32_t SrcLocStrSize,
1229 IdentFlag LocFlags,
1230 unsigned Reserve2Flags) {
1231 // Enable "C-mode".
1232 LocFlags |= OMP_IDENT_FLAG_KMPC;
1233
1234 Constant *&Ident =
1235 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
1236 if (!Ident) {
1237 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
1238 Constant *IdentData[] = {I32Null,
1239 ConstantInt::get(Ty: Int32, V: uint32_t(LocFlags)),
1240 ConstantInt::get(Ty: Int32, V: Reserve2Flags),
1241 ConstantInt::get(Ty: Int32, V: SrcLocStrSize), SrcLocStr};
1242
1243 size_t SrcLocStrArgIdx = 4;
1244 if (OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx)
1245 ->getPointerAddressSpace() !=
1246 IdentData[SrcLocStrArgIdx]->getType()->getPointerAddressSpace())
1247 IdentData[SrcLocStrArgIdx] = ConstantExpr::getAddrSpaceCast(
1248 C: SrcLocStr, Ty: OpenMPIRBuilder::Ident->getElementType(N: SrcLocStrArgIdx));
1249 Constant *Initializer =
1250 ConstantStruct::get(T: OpenMPIRBuilder::Ident, V: IdentData);
1251
1252 // Look for existing encoding of the location + flags, not needed but
1253 // minimizes the difference to the existing solution while we transition.
1254 for (GlobalVariable &GV : M.globals())
1255 if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
1256 if (GV.getInitializer() == Initializer)
1257 Ident = &GV;
1258
1259 if (!Ident) {
1260 auto *GV = new GlobalVariable(
1261 M, OpenMPIRBuilder::Ident,
1262 /* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
1263 nullptr, GlobalValue::NotThreadLocal,
1264 M.getDataLayout().getDefaultGlobalsAddressSpace());
1265 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1266 GV->setAlignment(Align(8));
1267 Ident = GV;
1268 }
1269 }
1270
1271 return ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Ident, Ty: IdentPtr);
1272}
1273
1274Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
1275 uint32_t &SrcLocStrSize) {
1276 SrcLocStrSize = LocStr.size();
1277 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
1278 if (!SrcLocStr) {
1279 Constant *Initializer =
1280 ConstantDataArray::getString(Context&: M.getContext(), Initializer: LocStr);
1281
1282 // Look for existing encoding of the location, not needed but minimizes the
1283 // difference to the existing solution while we transition.
1284 for (GlobalVariable &GV : M.globals())
1285 if (GV.isConstant() && GV.hasInitializer() &&
1286 GV.getInitializer() == Initializer)
1287 return SrcLocStr = ConstantExpr::getPointerCast(C: &GV, Ty: Int8Ptr);
1288
1289 SrcLocStr = Builder.CreateGlobalString(
1290 Str: LocStr, /*Name=*/"", AddressSpace: M.getDataLayout().getDefaultGlobalsAddressSpace(),
1291 M: &M);
1292 }
1293 return SrcLocStr;
1294}
1295
1296Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
1297 StringRef FileName,
1298 unsigned Line, unsigned Column,
1299 uint32_t &SrcLocStrSize) {
1300 SmallString<128> Buffer;
1301 Buffer.push_back(Elt: ';');
1302 Buffer.append(RHS: FileName);
1303 Buffer.push_back(Elt: ';');
1304 Buffer.append(RHS: FunctionName);
1305 Buffer.push_back(Elt: ';');
1306 Buffer.append(RHS: std::to_string(val: Line));
1307 Buffer.push_back(Elt: ';');
1308 Buffer.append(RHS: std::to_string(val: Column));
1309 Buffer.push_back(Elt: ';');
1310 Buffer.push_back(Elt: ';');
1311 return getOrCreateSrcLocStr(LocStr: Buffer.str(), SrcLocStrSize);
1312}
1313
1314Constant *
1315OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
1316 StringRef UnknownLoc = ";unknown;unknown;0;0;;";
1317 return getOrCreateSrcLocStr(LocStr: UnknownLoc, SrcLocStrSize);
1318}
1319
1320Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
1321 uint32_t &SrcLocStrSize,
1322 const Function *F) {
1323 DILocation *DIL = DL.get();
1324 if (!DIL)
1325 return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1326 StringRef FileName =
1327 !DIL->getFilename().empty() ? DIL->getFilename() : M.getName();
1328 StringRef Function = DIL->getScope()->getSubprogram()->getName();
1329 if (Function.empty() && F)
1330 Function = F->getName();
1331 return getOrCreateSrcLocStr(FunctionName: Function, FileName, Line: DIL->getLine(),
1332 Column: DIL->getColumn(), SrcLocStrSize);
1333}
1334
1335Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
1336 uint32_t &SrcLocStrSize) {
1337 return getOrCreateSrcLocStr(DL: Loc.DL, SrcLocStrSize,
1338 F: Loc.IP.getNodeParent()->getParent());
1339}
1340
1341Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
1342 return createRuntimeFunctionCall(
1343 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num), Args: Ident,
1344 Name: "omp_global_thread_num");
1345}
1346
1347OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInReduction(
1348 const LocationDescription &Loc, ArrayRef<Value *> OrigPtrs,
1349 ArrayRef<Type *> ResultPtrTys,
1350 function_ref<void(unsigned, Value *)> MapPrivateCB) {
1351 assert(OrigPtrs.size() == ResultPtrTys.size() &&
1352 "expected one result pointer type per in_reduction item");
1353 if (!updateToLocation(Loc))
1354 return Loc.IP;
1355 if (OrigPtrs.empty())
1356 return Builder.saveIP();
1357
1358 // Compute the executing thread's gtid once for the whole target body and
1359 // reuse it for every in_reduction lookup, so a target with several
1360 // in_reduction items does not emit a redundant __kmpc_global_thread_num per
1361 // item.
1362 uint32_t SrcLocStrSize;
1363 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1364 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1365 Value *Gtid = getOrCreateThreadID(Ident);
1366
1367 // The runtime entry point takes (and returns) a generic, default-address-
1368 // space `ptr`. A NULL descriptor makes the runtime walk the enclosing
1369 // taskgroups to find the matching task_reduction registration for the item.
1370 Type *PtrTy = PointerType::getUnqual(C&: M.getContext());
1371 Value *NullDesc = ConstantPointerNull::get(T: PtrTy);
1372 FunctionCallee GetThData =
1373 getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_task_reduction_get_th_data);
1374
1375 for (unsigned Idx = 0; Idx < OrigPtrs.size(); ++Idx) {
1376 // Normalize a non-default-address-space original pointer to the generic
1377 // address space before the call.
1378 Value *OrigPtr = OrigPtrs[Idx];
1379 if (auto *OrigPtrTy = dyn_cast<PointerType>(Val: OrigPtr->getType());
1380 OrigPtrTy && OrigPtrTy->getAddressSpace() != 0)
1381 OrigPtr = Builder.CreateAddrSpaceCast(V: OrigPtr, DestTy: PtrTy);
1382
1383 Value *Priv = Builder.CreateCall(Callee: GetThData, Args: {Gtid, NullDesc, OrigPtr},
1384 Name: "omp.inred.priv");
1385
1386 // Cast the returned private pointer back to the requested address space
1387 // when it differs.
1388 if (auto *ResPtrTy = dyn_cast<PointerType>(Val: ResultPtrTys[Idx]);
1389 ResPtrTy && ResPtrTy->getAddressSpace() != 0)
1390 Priv = Builder.CreateAddrSpaceCast(V: Priv, DestTy: ResultPtrTys[Idx]);
1391
1392 MapPrivateCB(Idx, Priv);
1393 }
1394 return Builder.saveIP();
1395}
1396
1397OpenMPIRBuilder::InsertPointOrErrorTy
1398OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive Kind,
1399 bool ForceSimpleCall, bool CheckCancelFlag) {
1400 if (!updateToLocation(Loc))
1401 return Loc.IP;
1402
1403 // Build call __kmpc_cancel_barrier(loc, thread_id) or
1404 // __kmpc_barrier(loc, thread_id);
1405
1406 IdentFlag BarrierLocFlags;
1407 switch (Kind) {
1408 case OMPD_for:
1409 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
1410 break;
1411 case OMPD_sections:
1412 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
1413 break;
1414 case OMPD_single:
1415 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
1416 break;
1417 case OMPD_barrier:
1418 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
1419 break;
1420 default:
1421 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
1422 break;
1423 }
1424
1425 uint32_t SrcLocStrSize;
1426 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1427 Value *Args[] = {
1428 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: BarrierLocFlags),
1429 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
1430
1431 // If we are in a cancellable parallel region, barriers are cancellation
1432 // points.
1433 // TODO: Check why we would force simple calls or to ignore the cancel flag.
1434 bool UseCancelBarrier =
1435 !ForceSimpleCall && isLastFinalizationInfoCancellable(DK: OMPD_parallel);
1436
1437 Value *Result = createRuntimeFunctionCall(
1438 Callee: getOrCreateRuntimeFunctionPtr(FnID: UseCancelBarrier
1439 ? OMPRTL___kmpc_cancel_barrier
1440 : OMPRTL___kmpc_barrier),
1441 Args);
1442
1443 if (UseCancelBarrier && CheckCancelFlag)
1444 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective: OMPD_parallel))
1445 return Err;
1446
1447 return Builder.saveIP();
1448}
1449
1450OpenMPIRBuilder::InsertPointOrErrorTy
1451OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
1452 Value *IfCondition,
1453 omp::Directive CanceledDirective) {
1454 if (!updateToLocation(Loc))
1455 return Loc.IP;
1456
1457 // LLVM utilities like blocks with terminators.
1458 auto *UI = Builder.CreateUnreachable();
1459
1460 Instruction *ThenTI = UI, *ElseTI = nullptr;
1461 if (IfCondition) {
1462 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: UI, ThenTerm: &ThenTI, ElseTerm: &ElseTI);
1463
1464 // Even if the if condition evaluates to false, this should count as a
1465 // cancellation point
1466 Builder.SetInsertPoint(ElseTI);
1467 auto ElseIP = Builder.saveIP();
1468
1469 InsertPointOrErrorTy IPOrErr = createCancellationPoint(
1470 Loc: LocationDescription{ElseIP, Loc.DL}, CanceledDirective);
1471 if (!IPOrErr)
1472 return IPOrErr;
1473 }
1474
1475 Builder.SetInsertPoint(ThenTI);
1476
1477 Value *CancelKind = nullptr;
1478 switch (CanceledDirective) {
1479#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1480 case DirectiveEnum: \
1481 CancelKind = Builder.getInt32(Value); \
1482 break;
1483#include "llvm/Frontend/OpenMP/OMPKinds.def"
1484 default:
1485 llvm_unreachable("Unknown cancel kind!");
1486 }
1487
1488 uint32_t SrcLocStrSize;
1489 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1490 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1491 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1492 Value *Result = createRuntimeFunctionCall(
1493 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancel), Args);
1494
1495 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1496 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1497 return Err;
1498
1499 // Update the insertion point and remove the terminator we introduced.
1500 Builder.SetInsertPoint(UI->getParent());
1501 UI->eraseFromParent();
1502
1503 return Builder.saveIP();
1504}
1505
1506OpenMPIRBuilder::InsertPointOrErrorTy
1507OpenMPIRBuilder::createCancellationPoint(const LocationDescription &Loc,
1508 omp::Directive CanceledDirective) {
1509 if (!updateToLocation(Loc))
1510 return Loc.IP;
1511
1512 // LLVM utilities like blocks with terminators.
1513 auto *UI = Builder.CreateUnreachable();
1514 Builder.SetInsertPoint(UI);
1515
1516 Value *CancelKind = nullptr;
1517 switch (CanceledDirective) {
1518#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1519 case DirectiveEnum: \
1520 CancelKind = Builder.getInt32(Value); \
1521 break;
1522#include "llvm/Frontend/OpenMP/OMPKinds.def"
1523 default:
1524 llvm_unreachable("Unknown cancel kind!");
1525 }
1526
1527 uint32_t SrcLocStrSize;
1528 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1529 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1530 Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
1531 Value *Result = createRuntimeFunctionCall(
1532 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancellationpoint), Args);
1533
1534 // The actual cancel logic is shared with others, e.g., cancel_barriers.
1535 if (Error Err = emitCancelationCheckImpl(CancelFlag: Result, CanceledDirective))
1536 return Err;
1537
1538 // Update the insertion point and remove the terminator we introduced.
1539 Builder.SetInsertPoint(UI->getParent());
1540 UI->eraseFromParent();
1541
1542 return Builder.saveIP();
1543}
1544
1545OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
1546 const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
1547 Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
1548 Value *HostPtr, ArrayRef<Value *> KernelArgs) {
1549 if (!updateToLocation(Loc))
1550 return Loc.IP;
1551
1552 Builder.restoreIP(IP: AllocaIP);
1553 auto *KernelArgsPtr =
1554 Builder.CreateAlloca(Ty: OpenMPIRBuilder::KernelArgs, ArraySize: nullptr, Name: "kernel_args");
1555 updateToLocation(Loc);
1556
1557 for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
1558 llvm::Value *Arg =
1559 Builder.CreateStructGEP(Ty: OpenMPIRBuilder::KernelArgs, Ptr: KernelArgsPtr, Idx: I);
1560 Builder.CreateAlignedStore(
1561 Val: KernelArgs[I], Ptr: Arg,
1562 Align: M.getDataLayout().getPrefTypeAlign(Ty: KernelArgs[I]->getType()));
1563 }
1564
1565 SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
1566 NumThreads, HostPtr, KernelArgsPtr};
1567
1568 Return = createRuntimeFunctionCall(
1569 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_target_kernel),
1570 Args: OffloadingArgs);
1571
1572 return Builder.saveIP();
1573}
1574
1575OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitKernelLaunch(
1576 const LocationDescription &Loc, Value *OutlinedFnID,
1577 EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args,
1578 Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {
1579
1580 if (!updateToLocation(Loc))
1581 return Loc.IP;
1582
1583 // On top of the arrays that were filled up, the target offloading call
1584 // takes as arguments the device id as well as the host pointer. The host
1585 // pointer is used by the runtime library to identify the current target
1586 // region, so it only has to be unique and not necessarily point to
1587 // anything. It could be the pointer to the outlined function that
1588 // implements the target region, but we aren't using that so that the
1589 // compiler doesn't need to keep that, and could therefore inline the host
1590 // function if proven worthwhile during optimization.
1591
1592 // From this point on, we need to have an ID of the target region defined.
1593 assert(OutlinedFnID && "Invalid outlined function ID!");
1594 (void)OutlinedFnID;
1595
1596 // Return value of the runtime offloading call.
1597 Value *Return = nullptr;
1598
1599 // Arguments for the target kernel.
1600 SmallVector<Value *> ArgsVector;
1601 getKernelArgsVector(KernelArgs&: Args, Builder, ArgsVector);
1602
1603 // The target region is an outlined function launched by the runtime
1604 // via calls to __tgt_target_kernel().
1605 //
1606 // Note that on the host and CPU targets, the runtime implementation of
1607 // these calls simply call the outlined function without forking threads.
1608 // The outlined functions themselves have runtime calls to
1609 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
1610 // the compiler in emitTeamsCall() and emitParallelCall().
1611 //
1612 // In contrast, on the NVPTX target, the implementation of
1613 // __tgt_target_teams() launches a GPU kernel with the requested number
1614 // of teams and threads so no additional calls to the runtime are required.
1615 // Check the error code and execute the host version if required.
1616 Builder.restoreIP(IP: emitTargetKernel(
1617 Loc: Builder, AllocaIP, Return, Ident: RTLoc, DeviceID, NumTeams: Args.NumTeams.front(),
1618 NumThreads: Args.NumThreads.front(), HostPtr: OutlinedFnID, KernelArgs: ArgsVector));
1619
1620 BasicBlock *OffloadFailedBlock =
1621 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.failed");
1622 BasicBlock *OffloadContBlock =
1623 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
1624 Value *Failed = Builder.CreateIsNotNull(Arg: Return);
1625 Builder.CreateCondBr(Cond: Failed, True: OffloadFailedBlock, False: OffloadContBlock);
1626
1627 auto CurFn = Builder.GetInsertBlock()->getParent();
1628 emitBlock(BB: OffloadFailedBlock, CurFn);
1629 InsertPointOrErrorTy AfterIP = EmitTargetCallFallbackCB(Builder.saveIP());
1630 if (!AfterIP)
1631 return AfterIP.takeError();
1632 Builder.restoreIP(IP: *AfterIP);
1633 emitBranch(Target: OffloadContBlock);
1634 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
1635 return Builder.saveIP();
1636}
1637
1638Error OpenMPIRBuilder::emitCancelationCheckImpl(
1639 Value *CancelFlag, omp::Directive CanceledDirective) {
1640 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
1641 "Unexpected cancellation!");
1642
1643 // For a cancel barrier we create two new blocks.
1644 BasicBlock *BB = Builder.GetInsertBlock();
1645 BasicBlock *NonCancellationBlock;
1646 if (Builder.GetInsertPoint() == BB->end()) {
1647 // TODO: This branch will not be needed once we moved to the
1648 // OpenMPIRBuilder codegen completely.
1649 NonCancellationBlock = BasicBlock::Create(
1650 Context&: BB->getContext(), Name: BB->getName() + ".cont", Parent: BB->getParent());
1651 } else {
1652 NonCancellationBlock = SplitBlock(Old: BB, SplitPt: &*Builder.GetInsertPoint());
1653 BB->getTerminator()->eraseFromParent();
1654 Builder.SetInsertPoint(BB);
1655 }
1656 BasicBlock *CancellationBlock = BasicBlock::Create(
1657 Context&: BB->getContext(), Name: BB->getName() + ".cncl", Parent: BB->getParent());
1658
1659 // Jump to them based on the return value.
1660 Value *Cmp = Builder.CreateIsNull(Arg: CancelFlag);
1661 Builder.CreateCondBr(Cond: Cmp, True: NonCancellationBlock, False: CancellationBlock,
1662 /* TODO weight */ BranchWeights: nullptr, Unpredictable: nullptr);
1663
1664 // From the cancellation block we finalize all variables and go to the
1665 // post finalization block that is known to the FiniCB callback.
1666 auto &FI = FinalizationStack.back();
1667 Expected<BasicBlock *> FiniBBOrErr = FI.getFiniBB(Builder);
1668 if (!FiniBBOrErr)
1669 return FiniBBOrErr.takeError();
1670 Builder.SetInsertPoint(CancellationBlock);
1671 Builder.CreateBr(Dest: *FiniBBOrErr);
1672
1673 // The continuation block is where code generation continues.
1674 Builder.SetInsertPoint(NonCancellationBlock->begin());
1675 return Error::success();
1676}
1677
1678/// Create wrapper function used to gather the outlined function's argument
1679/// structure from a shared buffer and to forward them to it when running in
1680/// Generic mode.
1681///
1682/// The outlined function is expected to receive 2 integer arguments followed by
1683/// an optional pointer argument to an argument structure holding the rest.
1684static Function *createTargetParallelWrapper(OpenMPIRBuilder *OMPIRBuilder,
1685 Function &OutlinedFn) {
1686 size_t NumArgs = OutlinedFn.arg_size();
1687 assert((NumArgs == 2 || NumArgs == 3) &&
1688 "expected a 2-3 argument parallel outlined function");
1689 bool UseArgStruct = NumArgs == 3;
1690
1691 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1692 IRBuilder<>::InsertPointGuard IPG(Builder);
1693 auto *FnTy = FunctionType::get(Result: Builder.getVoidTy(),
1694 Params: {Builder.getInt16Ty(), Builder.getInt32Ty()},
1695 /*isVarArg=*/false);
1696 auto *WrapperFn =
1697 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage,
1698 N: OutlinedFn.getName() + ".wrapper", M&: OMPIRBuilder->M);
1699
1700 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1701 WrapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::ZExt);
1702 WrapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1703
1704 BasicBlock *EntryBB =
1705 BasicBlock::Create(Context&: OMPIRBuilder->M.getContext(), Name: "entry", Parent: WrapperFn);
1706 Builder.SetInsertPoint(EntryBB);
1707
1708 // Allocation.
1709 Value *AddrAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1710 /*ArraySize=*/nullptr, Name: "addr");
1711 AddrAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1712 V: AddrAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1713 Name: AddrAlloca->getName() + ".ascast");
1714
1715 Value *ZeroAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(),
1716 /*ArraySize=*/nullptr, Name: "zero");
1717 ZeroAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1718 V: ZeroAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1719 Name: ZeroAlloca->getName() + ".ascast");
1720
1721 Value *ArgsAlloca = nullptr;
1722 if (UseArgStruct) {
1723 ArgsAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(),
1724 /*ArraySize=*/nullptr, Name: "global_args");
1725 ArgsAlloca = Builder.CreatePointerBitCastOrAddrSpaceCast(
1726 V: ArgsAlloca, DestTy: Builder.getPtrTy(/*AddrSpace=*/0),
1727 Name: ArgsAlloca->getName() + ".ascast");
1728 }
1729
1730 // Initialization.
1731 Builder.CreateStore(Val: WrapperFn->getArg(i: 1), Ptr: AddrAlloca);
1732 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: ZeroAlloca);
1733 if (UseArgStruct) {
1734 Builder.CreateCall(
1735 Callee: OMPIRBuilder->getOrCreateRuntimeFunctionPtr(
1736 FnID: llvm::omp::RuntimeFunction::OMPRTL___kmpc_get_shared_variables),
1737 Args: {ArgsAlloca});
1738 }
1739
1740 SmallVector<Value *, 3> Args{AddrAlloca, ZeroAlloca};
1741
1742 // Load structArg from global_args.
1743 if (UseArgStruct) {
1744 Value *StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ArgsAlloca);
1745 StructArg = Builder.CreateInBoundsGEP(Ty: Builder.getPtrTy(), Ptr: StructArg,
1746 IdxList: {Builder.getInt64(C: 0)});
1747 StructArg = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: StructArg, Name: "structArg");
1748 Args.push_back(Elt: StructArg);
1749 }
1750
1751 // Call the outlined function holding the parallel body.
1752 Builder.CreateCall(Callee: &OutlinedFn, Args);
1753 Builder.CreateRetVoid();
1754
1755 return WrapperFn;
1756}
1757
1758// Callback used to create OpenMP runtime calls to support
1759// omp parallel clause for the device.
1760// We need to use this callback to replace call to the OutlinedFn in OuterFn
1761// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_60)
1762static void targetParallelCallback(
1763 OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,
1764 BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,
1765 Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1766 Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {
1767 assert(OutlinedFn.arg_size() >= 2 &&
1768 "Expected at least tid and bounded tid as arguments");
1769 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1770
1771 // Add some known attributes.
1772 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1773 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1774 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1775 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
1776 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
1777 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1778
1779 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1780 assert(CI && "Expected call instruction to outlined function");
1781 CI->getParent()->setName("omp_parallel");
1782
1783 Builder.SetInsertPoint(CI);
1784 Type *PtrTy = OMPIRBuilder->VoidPtr;
1785
1786 // Add alloca for kernel args
1787 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
1788 Builder.SetInsertPoint(OuterAllocaBB->getFirstInsertionPt());
1789 AllocaInst *ArgsAlloca =
1790 Builder.CreateAlloca(Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars));
1791 Value *Args = ArgsAlloca;
1792 // Add address space cast if array for storing arguments is not allocated
1793 // in address space 0
1794 if (ArgsAlloca->getAddressSpace())
1795 Args = Builder.CreatePointerCast(V: ArgsAlloca, DestTy: PtrTy);
1796 Builder.restoreIP(IP: CurrentIP);
1797
1798 // Store captured vars which are used by kmpc_parallel_60
1799 for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {
1800 Value *V = *(CI->arg_begin() + 2 + Idx);
1801 Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
1802 Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars), Ptr: Args, Idx0: 0, Idx1: Idx);
1803 Builder.CreateStore(Val: V, Ptr: StoreAddress);
1804 }
1805
1806 Value *Cond =
1807 IfCondition ? Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32)
1808 : Builder.getInt32(C: 1);
1809 Value *NumThreadsArg =
1810 NumThreads ? Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: OMPIRBuilder->Int32)
1811 : Builder.getInt32(C: -1);
1812
1813 // If this is not a Generic kernel, we can skip generating the wrapper.
1814 Value *WrapperFn;
1815 if (isGenericKernel(Fn&: *OuterFn))
1816 WrapperFn = createTargetParallelWrapper(OMPIRBuilder, OutlinedFn);
1817 else
1818 WrapperFn = Constant::getNullValue(Ty: PtrTy);
1819
1820 // Build kmpc_parallel_60 call
1821 Value *Parallel60CallArgs[] = {
1822 /* identifier*/ Ident,
1823 /* global thread num*/ ThreadID,
1824 /* if expression */ Cond,
1825 /* number of threads */ NumThreadsArg,
1826 /* Proc bind */ Builder.getInt32(C: -1),
1827 /* outlined function */ &OutlinedFn,
1828 /* wrapper function */ WrapperFn,
1829 /* arguments of the outlined funciton*/ Args,
1830 /* number of arguments */ Builder.getInt64(C: NumCapturedVars),
1831 /* strict for number of threads */ Builder.getInt32(C: 0)};
1832
1833 FunctionCallee RTLFn =
1834 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_parallel_60);
1835
1836 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: Parallel60CallArgs);
1837
1838 LLVM_DEBUG(dbgs() << "With kmpc_parallel_60 placed: "
1839 << *Builder.GetInsertBlock()->getParent() << "\n");
1840
1841 // Initialize the local TID stack location with the argument value.
1842 Builder.SetInsertPoint(PrivTID);
1843 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1844 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1845 Ptr: PrivTIDAddr);
1846
1847 // Remove redundant call to the outlined function.
1848 CI->eraseFromParent();
1849
1850 for (Instruction *I : ToBeDeleted) {
1851 I->eraseFromParent();
1852 }
1853}
1854
1855// Callback used to create OpenMP runtime calls to support
1856// omp parallel clause for the host.
1857// We need to use this callback to replace call to the OutlinedFn in OuterFn
1858// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])
1859static void
1860hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
1861 Function *OuterFn, Value *Ident, Value *IfCondition,
1862 Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1863 const SmallVector<Instruction *, 4> &ToBeDeleted) {
1864 IRBuilder<> &Builder = OMPIRBuilder->Builder;
1865 FunctionCallee RTLFn;
1866 if (IfCondition) {
1867 RTLFn =
1868 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call_if);
1869 } else {
1870 RTLFn =
1871 OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call);
1872 }
1873 if (auto *F = dyn_cast<Function>(Val: RTLFn.getCallee())) {
1874 if (!F->hasMetadata(KindID: LLVMContext::MD_callback)) {
1875 LLVMContext &Ctx = F->getContext();
1876 MDBuilder MDB(Ctx);
1877 // Annotate the callback behavior of the __kmpc_fork_call:
1878 // - The callback callee is argument number 2 (microtask).
1879 // - The first two arguments of the callback callee are unknown (-1).
1880 // - All variadic arguments to the __kmpc_fork_call are passed to the
1881 // callback callee.
1882 F->addMetadata(KindID: LLVMContext::MD_callback,
1883 MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
1884 CalleeArgNo: 2, Arguments: {-1, -1},
1885 /* VarArgsArePassed */ true)}));
1886 }
1887 }
1888 // Add some known attributes.
1889 OutlinedFn.addParamAttr(ArgNo: 0, Kind: Attribute::NoAlias);
1890 OutlinedFn.addParamAttr(ArgNo: 1, Kind: Attribute::NoAlias);
1891 OutlinedFn.addFnAttr(Kind: Attribute::NoUnwind);
1892
1893 assert(OutlinedFn.arg_size() >= 2 &&
1894 "Expected at least tid and bounded tid as arguments");
1895 unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1896
1897 CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1898
1899 // CodeExtractor creates an alloca aggregate struct containing pointers to all
1900 // the captured values. The address of this struct is then passed as a
1901 // parameter to the outlined function. This parameter of the outlined function
1902 // can be marked noalias because CodeExractor + this host OpenMP lowering
1903 // maintains the following invariants:
1904 // - The caller initializes the fields before starting any callbacks.
1905 // - Neither the caller, OpenMP runtime, nor callbacks modify the structure
1906 // while callbacks are executing.
1907 // - The callbacks only load captured values from the structure; they do not
1908 // use it to write results back to the caller. This is generated code; the
1909 // user has no way to modify the structure directly.
1910 //
1911 // Multiple callbacks may read the same structure. This is allowed by noalias,
1912 // which permits aliasing accesses to memory that is not modified during the
1913 // call.
1914 //
1915 // The synchronous fork keeps the structure alive until all callbacks return,
1916 // which justifies nofreeobj. Neither attribute extends to the objects pointed
1917 // to by the captured fields.
1918 if (NumCapturedVars) {
1919 assert(NumCapturedVars == 1 && "Expected an aggregate capture argument");
1920 assert(isa<AllocaInst>(CI->getArgOperand(2)) &&
1921 cast<AllocaInst>(CI->getArgOperand(2))
1922 ->getAllocatedType()
1923 ->isStructTy() &&
1924 "Expected a stack-allocated capture structure");
1925
1926 OutlinedFn.addParamAttr(ArgNo: 2, Kind: Attribute::NoAlias);
1927 OutlinedFn.addParamAttr(ArgNo: 2, Kind: Attribute::NoFreeObj);
1928 }
1929
1930 CI->getParent()->setName("omp_parallel");
1931 Builder.SetInsertPoint(CI);
1932
1933 // Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);
1934 Value *ForkCallArgs[] = {Ident, Builder.getInt32(C: NumCapturedVars),
1935 &OutlinedFn};
1936
1937 SmallVector<Value *, 16> RealArgs;
1938 RealArgs.append(in_start: std::begin(arr&: ForkCallArgs), in_end: std::end(arr&: ForkCallArgs));
1939 if (IfCondition) {
1940 Value *Cond = Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32);
1941 RealArgs.push_back(Elt: Cond);
1942 }
1943 RealArgs.append(in_start: CI->arg_begin() + /* tid & bound tid */ 2, in_end: CI->arg_end());
1944
1945 // __kmpc_fork_call_if always expects a void ptr as the last argument
1946 // If there are no arguments, pass a null pointer.
1947 auto PtrTy = OMPIRBuilder->VoidPtr;
1948 if (IfCondition && NumCapturedVars == 0) {
1949 Value *NullPtrValue = Constant::getNullValue(Ty: PtrTy);
1950 RealArgs.push_back(Elt: NullPtrValue);
1951 }
1952
1953 OMPIRBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
1954
1955 LLVM_DEBUG(dbgs() << "With fork_call placed: "
1956 << *Builder.GetInsertBlock()->getParent() << "\n");
1957
1958 // Initialize the local TID stack location with the argument value.
1959 Builder.SetInsertPoint(PrivTID);
1960 Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1961 Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1962 Ptr: PrivTIDAddr);
1963
1964 // Remove redundant call to the outlined function.
1965 CI->eraseFromParent();
1966
1967 for (Instruction *I : ToBeDeleted) {
1968 I->eraseFromParent();
1969 }
1970}
1971
1972OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
1973 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
1974 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB,
1975 PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition,
1976 Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable) {
1977 assert(!isConflictIP(Loc.IP, OuterAllocIP) && "IPs must not be ambiguous");
1978
1979 if (!updateToLocation(Loc))
1980 return Loc.IP;
1981
1982 uint32_t SrcLocStrSize;
1983 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1984 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1985 const bool NeedThreadID = NumThreads || Config.isTargetDevice() ||
1986 (ProcBind != OMP_PROC_BIND_default);
1987 Value *ThreadID = NeedThreadID ? getOrCreateThreadID(Ident) : nullptr;
1988 // If we generate code for the target device, we need to allocate
1989 // struct for aggregate params in the device default alloca address space.
1990 // OpenMP runtime requires that the params of the extracted functions are
1991 // passed as zero address space pointers. This flag ensures that extracted
1992 // function arguments are declared in zero address space
1993 bool ArgsInZeroAddressSpace = Config.isTargetDevice();
1994
1995 // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
1996 // only if we compile for host side.
1997 if (NumThreads && !Config.isTargetDevice()) {
1998 Value *Args[] = {
1999 Ident, ThreadID,
2000 Builder.CreateIntCast(V: NumThreads, DestTy: Int32, /*isSigned*/ false)};
2001 createRuntimeFunctionCall(
2002 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_threads), Args);
2003 }
2004
2005 if (ProcBind != OMP_PROC_BIND_default) {
2006 // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
2007 Value *Args[] = {
2008 Ident, ThreadID,
2009 ConstantInt::get(Ty: Int32, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
2010 createRuntimeFunctionCall(
2011 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_proc_bind), Args);
2012 }
2013
2014 BasicBlock *InsertBB = Builder.GetInsertBlock();
2015 Function *OuterFn = InsertBB->getParent();
2016
2017 // Save the outer alloca block because the insertion iterator may get
2018 // invalidated and we still need this later.
2019 BasicBlock *OuterAllocaBlock = OuterAllocIP.getNodeParent();
2020
2021 // Vector to remember instructions we used only during the modeling but which
2022 // we want to delete at the end.
2023 SmallVector<Instruction *, 4> ToBeDeleted;
2024
2025 // Change the location to the outer alloca insertion point to create and
2026 // initialize the allocas we pass into the parallel region.
2027 InsertPointTy NewOuter(OuterAllocaBlock->begin());
2028 Builder.restoreIP(IP: NewOuter);
2029 AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr");
2030 AllocaInst *ZeroAddrAlloca =
2031 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "zero.addr");
2032 Instruction *TIDAddr = TIDAddrAlloca;
2033 Instruction *ZeroAddr = ZeroAddrAlloca;
2034 if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {
2035 // Add additional casts to enforce pointers in zero address space
2036 TIDAddr = new AddrSpaceCastInst(
2037 TIDAddrAlloca, PointerType ::get(C&: M.getContext(), AddressSpace: 0), "tid.addr.ascast");
2038 TIDAddr->insertAfter(InsertPos: TIDAddrAlloca->getIterator());
2039 ToBeDeleted.push_back(Elt: TIDAddr);
2040 ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,
2041 PointerType ::get(C&: M.getContext(), AddressSpace: 0),
2042 "zero.addr.ascast");
2043 ZeroAddr->insertAfter(InsertPos: ZeroAddrAlloca->getIterator());
2044 ToBeDeleted.push_back(Elt: ZeroAddr);
2045 }
2046
2047 // We only need TIDAddr and ZeroAddr for modeling purposes to get the
2048 // associated arguments in the outlined function, so we delete them later.
2049 ToBeDeleted.push_back(Elt: TIDAddrAlloca);
2050 ToBeDeleted.push_back(Elt: ZeroAddrAlloca);
2051
2052 // Create an artificial insertion point that will also ensure the blocks we
2053 // are about to split are not degenerated.
2054 auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
2055
2056 BasicBlock *EntryBB = UI->getParent();
2057 BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(I: UI, BBName: "omp.par.entry");
2058 BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(I: UI, BBName: "omp.par.region");
2059 BasicBlock *PRegPreFiniBB =
2060 PRegBodyBB->splitBasicBlock(I: UI, BBName: "omp.par.pre_finalize");
2061 BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(I: UI, BBName: "omp.par.exit");
2062
2063 auto FiniCBWrapper = [&](InsertPointTy IP) {
2064 // Hide "open-ended" blocks from the given FiniCB by setting the right jump
2065 // target to the region exit block.
2066 if (IP == IP.getNodeParent()->end()) {
2067 IRBuilder<>::InsertPointGuard IPG(Builder);
2068 Builder.restoreIP(IP);
2069 Instruction *I = Builder.CreateBr(Dest: PRegExitBB);
2070 IP = I->getIterator();
2071 }
2072 assert(IP.getNodeParent()->getTerminator()->getNumSuccessors() == 1 &&
2073 IP.getNodeParent()->getTerminator()->getSuccessor(0) == PRegExitBB &&
2074 "Unexpected insertion point for finalization call!");
2075 return FiniCB(IP);
2076 };
2077
2078 FinalizationStack.push_back(Elt: {FiniCBWrapper, OMPD_parallel, IsCancellable});
2079
2080 // Generate the privatization allocas in the block that will become the entry
2081 // of the outlined function.
2082 Builder.SetInsertPoint(PRegEntryBB->getTerminator());
2083 InsertPointTy InnerAllocaIP = Builder.saveIP();
2084
2085 AllocaInst *PrivTIDAddr =
2086 Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr.local");
2087 Instruction *PrivTID = Builder.CreateLoad(Ty: Int32, Ptr: PrivTIDAddr, Name: "tid");
2088
2089 // Add some fake uses for OpenMP provided arguments.
2090 ToBeDeleted.push_back(Elt: Builder.CreateLoad(Ty: Int32, Ptr: TIDAddr, Name: "tid.addr.use"));
2091 Instruction *ZeroAddrUse =
2092 Builder.CreateLoad(Ty: Int32, Ptr: ZeroAddr, Name: "zero.addr.use");
2093 ToBeDeleted.push_back(Elt: ZeroAddrUse);
2094
2095 // EntryBB
2096 // |
2097 // V
2098 // PRegionEntryBB <- Privatization allocas are placed here.
2099 // |
2100 // V
2101 // PRegionBodyBB <- BodeGen is invoked here.
2102 // |
2103 // V
2104 // PRegPreFiniBB <- The block we will start finalization from.
2105 // |
2106 // V
2107 // PRegionExitBB <- A common exit to simplify block collection.
2108 //
2109
2110 LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");
2111
2112 // Let the caller create the body.
2113 assert(BodyGenCB && "Expected body generation callback!");
2114 InsertPointTy CodeGenIP(PRegBodyBB->begin());
2115 if (Error Err = BodyGenCB(InnerAllocaIP, CodeGenIP, PRegExitBB))
2116 return Err;
2117
2118 LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
2119
2120 // If OuterFn is a Generic kernel, we need to use device shared memory to
2121 // allocate argument structures. Otherwise, we use stack allocations as usual.
2122 bool UsesDeviceSharedMemory =
2123 Config.isTargetDevice() && isGenericKernel(Fn&: *OuterFn);
2124 std::unique_ptr<OutlineInfo> OI =
2125 UsesDeviceSharedMemory
2126 ? std::make_unique<DeviceSharedMemOutlineInfo>(args&: *this)
2127 : std::make_unique<OutlineInfo>();
2128
2129 if (Config.isTargetDevice()) {
2130 // Generate OpenMP target specific runtime call
2131 OI->PostOutlineCB = [=, ToBeDeletedVec =
2132 std::move(ToBeDeleted)](Function &OutlinedFn) {
2133 targetParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, OuterAllocaBB: OuterAllocaBlock, Ident,
2134 IfCondition, NumThreads, PrivTID, PrivTIDAddr,
2135 ThreadID, ToBeDeleted: ToBeDeletedVec);
2136 };
2137 } else {
2138 // Generate OpenMP host runtime call
2139 OI->PostOutlineCB = [=, ToBeDeletedVec =
2140 std::move(ToBeDeleted)](Function &OutlinedFn) {
2141 hostParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, Ident, IfCondition,
2142 PrivTID, PrivTIDAddr, ToBeDeleted: ToBeDeletedVec);
2143 };
2144 }
2145
2146 OI->FixUpNonEntryAllocas = true;
2147 OI->OuterAllocBB = OuterAllocaBlock;
2148 OI->EntryBB = PRegEntryBB;
2149 OI->ExitBB = PRegExitBB;
2150 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
2151 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
2152
2153 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
2154 SmallVector<BasicBlock *, 32> Blocks;
2155 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
2156
2157 CodeExtractorAnalysisCache CEAC(*OuterFn);
2158 CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
2159 /* AggregateArgs */ false,
2160 /* BlockFrequencyInfo */ nullptr,
2161 /* BranchProbabilityInfo */ nullptr,
2162 /* AssumptionCache */ nullptr,
2163 /* AllowVarArgs */ true,
2164 /* AllowAlloca */ true,
2165 /* AllocationBlock */ OuterAllocaBlock,
2166 /* DeallocationBlocks */ {},
2167 /* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
2168
2169 // Find inputs to, outputs from the code region.
2170 BasicBlock *CommonExit = nullptr;
2171 SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
2172 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
2173
2174 Extractor.findInputsOutputs(Inputs, Outputs, Allocas: SinkingCands,
2175 /*CollectGlobalInputs=*/true);
2176
2177 Inputs.remove_if(P: [&](Value *I) {
2178 if (auto *GV = dyn_cast_if_present<GlobalVariable>(Val: I))
2179 return GV->getValueType() == OpenMPIRBuilder::Ident;
2180
2181 return false;
2182 });
2183
2184 LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");
2185
2186 FunctionCallee TIDRTLFn =
2187 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num);
2188
2189 auto PrivHelper = [&](Value &V) -> Error {
2190 if (&V == TIDAddr || &V == ZeroAddr) {
2191 OI->ExcludeArgsFromAggregate.push_back(Elt: &V);
2192 return Error::success();
2193 }
2194
2195 SetVector<Use *> Uses;
2196 for (Use &U : V.uses())
2197 if (auto *UserI = dyn_cast<Instruction>(Val: U.getUser()))
2198 if (ParallelRegionBlockSet.count(Ptr: UserI->getParent()))
2199 Uses.insert(X: &U);
2200
2201 // __kmpc_fork_call expects extra arguments as pointers. If the input
2202 // already has a pointer type, everything is fine. Otherwise, store the
2203 // value onto stack and load it back inside the to-be-outlined region. This
2204 // will ensure only the pointer will be passed to the function.
2205 // FIXME: if there are more than 15 trailing arguments, they must be
2206 // additionally packed in a struct.
2207 Value *Inner = &V;
2208 if (!V.getType()->isPointerTy()) {
2209 IRBuilder<>::InsertPointGuard Guard(Builder);
2210 LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
2211
2212 Builder.restoreIP(IP: OuterAllocIP);
2213 Value *Ptr;
2214 if (UsesDeviceSharedMemory) {
2215 // Use device shared memory instead, if needed.
2216 Ptr = createOMPAllocShared(Loc: Builder, VarType: V.getType(),
2217 Name: V.getName() + ".reloaded");
2218 for (BasicBlock *DeallocBlock : OuterDeallocBlocks) {
2219 assert(DeallocBlock->getParent() ==
2220 OuterAllocIP.getNodeParent()->getParent() &&
2221 "Dealloc block must be in the allocation's function to reuse "
2222 "its debug location");
2223 createOMPFreeShared(Loc: {DeallocBlock->getFirstInsertionPt(),
2224 Builder.getCurrentDebugLocation()},
2225 Addr: Ptr, VarType: V.getType());
2226 }
2227 } else {
2228 Ptr = Builder.CreateAlloca(Ty: V.getType(), ArraySize: nullptr,
2229 Name: V.getName() + ".reloaded");
2230 }
2231
2232 // Store to stack at end of the block that currently branches to the entry
2233 // block of the to-be-outlined region.
2234 Builder.SetInsertPoint(InsertBB->getTerminator()->getIterator());
2235 Builder.CreateStore(Val: &V, Ptr);
2236
2237 // Load back next to allocations in the to-be-outlined region.
2238 Builder.restoreIP(IP: InnerAllocaIP);
2239 Inner = Builder.CreateLoad(Ty: V.getType(), Ptr);
2240 }
2241
2242 Value *ReplacementValue = nullptr;
2243 CallInst *CI = dyn_cast<CallInst>(Val: &V);
2244 if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
2245 ReplacementValue = PrivTID;
2246 } else {
2247 InsertPointOrErrorTy AfterIP =
2248 PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue);
2249 if (!AfterIP)
2250 return AfterIP.takeError();
2251 Builder.restoreIP(IP: *AfterIP);
2252 InnerAllocaIP =
2253 InnerAllocaIP.getNodeParent()->getTerminator()->getIterator();
2254
2255 assert(ReplacementValue &&
2256 "Expected copy/create callback to set replacement value!");
2257 if (ReplacementValue == &V)
2258 return Error::success();
2259 }
2260
2261 for (Use *UPtr : Uses)
2262 UPtr->set(ReplacementValue);
2263
2264 return Error::success();
2265 };
2266
2267 // Reset the inner alloca insertion as it will be used for loading the values
2268 // wrapped into pointers before passing them into the to-be-outlined region.
2269 // Configure it to insert immediately after the fake use of zero address so
2270 // that they are available in the generated body and so that the
2271 // OpenMP-related values (thread ID and zero address pointers) remain leading
2272 // in the argument list.
2273 InnerAllocaIP = ZeroAddrUse->getNextNode()->getIterator();
2274
2275 // Reset the outer alloca insertion point to the entry of the relevant block
2276 // in case it was invalidated.
2277 OuterAllocIP = OuterAllocaBlock->getFirstInsertionPt();
2278
2279 for (Value *Input : Inputs) {
2280 LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
2281 if (Error Err = PrivHelper(*Input))
2282 return Err;
2283 }
2284 LLVM_DEBUG({
2285 for (Value *Output : Outputs)
2286 LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
2287 });
2288 assert(Outputs.empty() &&
2289 "OpenMP outlining should not produce live-out values!");
2290
2291 LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n");
2292 LLVM_DEBUG({
2293 for (auto *BB : Blocks)
2294 dbgs() << " PBR: " << BB->getName() << "\n";
2295 });
2296
2297 // Adjust the finalization stack, verify the adjustment, and call the
2298 // finalize function a last time to finalize values between the pre-fini
2299 // block and the exit block if we left the parallel "the normal way".
2300 auto FiniInfo = FinalizationStack.pop_back_val();
2301 (void)FiniInfo;
2302 assert(FiniInfo.DK == OMPD_parallel &&
2303 "Unexpected finalization stack state!");
2304
2305 Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
2306
2307 InsertPointTy PreFiniIP(PRegPreFiniTI->getIterator());
2308 Expected<BasicBlock *> FiniBBOrErr = FiniInfo.getFiniBB(Builder);
2309 if (!FiniBBOrErr)
2310 return FiniBBOrErr.takeError();
2311 {
2312 IRBuilderBase::InsertPointGuard Guard(Builder);
2313 Builder.restoreIP(IP: PreFiniIP);
2314 Builder.CreateBr(Dest: *FiniBBOrErr);
2315 // There's currently a branch to omp.par.exit. Delete it. We will get there
2316 // via the fini block
2317 if (Instruction *Term = Builder.GetInsertBlock()->getTerminator())
2318 Term->eraseFromParent();
2319 }
2320
2321 // Register the outlined info.
2322 addOutlineInfo(OI: std::move(OI));
2323
2324 InsertPointTy AfterIP(UI->getParent()->end());
2325 UI->eraseFromParent();
2326
2327 return AfterIP;
2328}
2329
2330void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
2331 // Build call void __kmpc_flush(ident_t *loc)
2332 uint32_t SrcLocStrSize;
2333 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2334 Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
2335
2336 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_flush),
2337 Args);
2338}
2339
2340void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
2341 if (!updateToLocation(Loc))
2342 return;
2343 emitFlush(Loc);
2344}
2345
2346void OpenMPIRBuilder::createError(const LocationDescription &Loc, bool IsFatal,
2347 Value *Message) {
2348 if (!updateToLocation(Loc))
2349 return;
2350
2351 // Build call void __kmpc_error(ident_t *loc, int severity,
2352 // const char *message)
2353 uint32_t SrcLocStrSize;
2354 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2355 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2356 // Severity: 1 = warning, 2 = fatal.
2357 Value *Severity = ConstantInt::get(Ty: Int32, V: IsFatal ? 2 : 1);
2358 Value *MessageArg = Message ? Message : ConstantPointerNull::get(T: Int8Ptr);
2359 Value *Args[] = {Ident, Severity, MessageArg};
2360
2361 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_error),
2362 Args);
2363}
2364
2365void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
2366 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2367 uint32_t SrcLocStrSize;
2368 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2369 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2370 Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
2371 Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
2372
2373 createRuntimeFunctionCall(
2374 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskyield), Args);
2375}
2376
2377void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
2378 if (!updateToLocation(Loc))
2379 return;
2380 emitTaskyieldImpl(Loc);
2381}
2382
2383void OpenMPIRBuilder::emitTaskDependency(IRBuilderBase &Builder, Value *Entry,
2384 const DependData &Dep) {
2385 // Store the pointer to the variable
2386 Value *Addr = Builder.CreateStructGEP(
2387 Ty: DependInfo, Ptr: Entry,
2388 Idx: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
2389 Value *DepValPtr = Builder.CreatePtrToInt(V: Dep.DepVal, DestTy: SizeTy);
2390 Builder.CreateStore(Val: DepValPtr, Ptr: Addr);
2391 // Store the size of the variable
2392 Value *Size = Builder.CreateStructGEP(
2393 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Len));
2394 Builder.CreateStore(
2395 Val: ConstantInt::get(Ty: SizeTy,
2396 V: M.getDataLayout().getTypeStoreSize(Ty: Dep.DepValueType)),
2397 Ptr: Size);
2398 // Store the dependency kind
2399 Value *Flags = Builder.CreateStructGEP(
2400 Ty: DependInfo, Ptr: Entry, Idx: static_cast<unsigned int>(RTLDependInfoFields::Flags));
2401 Builder.CreateStore(Val: ConstantInt::get(Ty: Builder.getInt8Ty(),
2402 V: static_cast<unsigned int>(Dep.DepKind)),
2403 Ptr: Flags);
2404}
2405
2406// Processes the dependencies in Dependencies and does the following
2407// - Allocates space on the stack of an array of DependInfo objects
2408// - Populates each DependInfo object with relevant information of
2409// the corresponding dependence.
2410// - All code is inserted in the entry block of the current function.
2411static Value *emitTaskDependencies(
2412 OpenMPIRBuilder &OMPBuilder,
2413 const SmallVectorImpl<OpenMPIRBuilder::DependData> &Dependencies) {
2414 // Early return if we have no dependencies to process
2415 if (Dependencies.empty())
2416 return nullptr;
2417
2418 // Given a vector of DependData objects, in this function we create an
2419 // array on the stack that holds kmp_depend_info objects corresponding
2420 // to each dependency. This is then passed to the OpenMP runtime.
2421 // For example, if there are 'n' dependencies then the following psedo
2422 // code is generated. Assume the first dependence is on a variable 'a'
2423 //
2424 // \code{c}
2425 // DepArray = alloc(n x sizeof(kmp_depend_info);
2426 // idx = 0;
2427 // DepArray[idx].base_addr = ptrtoint(&a);
2428 // DepArray[idx].len = 8;
2429 // DepArray[idx].flags = Dep.DepKind; /*(See OMPContants.h for DepKind)*/
2430 // ++idx;
2431 // DepArray[idx].base_addr = ...;
2432 // \endcode
2433
2434 IRBuilderBase &Builder = OMPBuilder.Builder;
2435 Type *DependInfo = OMPBuilder.DependInfo;
2436
2437 Value *DepArray = nullptr;
2438 Type *DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.size());
2439 {
2440 // Use a InsertPointGuard to restore the location back along with the
2441 // insertion point.
2442 IRBuilderBase::InsertPointGuard IPGuard(Builder);
2443 Builder.SetInsertPoint(
2444 Builder.GetInsertBlock()->getParent()->getEntryBlock().getTerminator());
2445 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2446 }
2447
2448 for (const auto &[DepIdx, Dep] : enumerate(First: Dependencies)) {
2449 Value *Base =
2450 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2451 OMPBuilder.emitTaskDependency(Builder, Entry: Base, Dep);
2452 }
2453 return DepArray;
2454}
2455
2456void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
2457 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
2458 // global_tid);
2459 uint32_t SrcLocStrSize;
2460 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2461 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2462 Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
2463
2464 // Ignore return result until untied tasks are supported.
2465 createRuntimeFunctionCall(
2466 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskwait), Args);
2467}
2468
2469void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc,
2470 DependenciesInfo Dependencies,
2471 bool IsNowait) {
2472 if (!updateToLocation(Loc))
2473 return;
2474
2475 Value *DepArray = nullptr;
2476 Type *DepArrayTy = nullptr;
2477 Value *NumDeps = nullptr;
2478 if (Dependencies.DepArray) {
2479 DepArray = Dependencies.DepArray;
2480 NumDeps = Dependencies.NumDeps;
2481 } else if (!Dependencies.Deps.empty()) {
2482 DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.Deps.size());
2483 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
2484 {
2485 IRBuilderBase::InsertPointGuard IPGuard(Builder);
2486 BasicBlock &entryBB =
2487 Builder.GetInsertBlock()->getParent()->getEntryBlock();
2488 Builder.SetInsertPoint(entryBB.getFirstInsertionPt());
2489 DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
2490 }
2491
2492 for (const auto &[DepIdx, Dep] : enumerate(First&: Dependencies.Deps)) {
2493 Value *Base =
2494 Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: DepIdx);
2495 this->emitTaskDependency(Builder, Entry: Base, Dep);
2496 }
2497 }
2498
2499 if (DepArray) {
2500 uint32_t SrcLocStrSize;
2501 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2502 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2503 Value *Args[] = {
2504 Ident,
2505 getOrCreateThreadID(Ident),
2506 NumDeps,
2507 DepArray,
2508 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
2509 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext())),
2510 ConstantInt::get(Ty: Builder.getInt32Ty(), V: IsNowait)};
2511 createRuntimeFunctionCall(
2512 Callee: getOrCreateRuntimeFunctionPtr(
2513 FnID: omp::RuntimeFunction::OMPRTL___kmpc_omp_taskwait_deps_51),
2514 Args);
2515 } else {
2516 emitTaskwaitImpl(Loc);
2517 }
2518}
2519
2520/// Create the task duplication function passed to kmpc_taskloop.
2521Expected<Value *> OpenMPIRBuilder::createTaskDuplicationFunction(
2522 Type *PrivatesTy, int32_t PrivatesIndex, TaskDupCallbackTy DupCB) {
2523 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2524 if (!DupCB)
2525 return Constant::getNullValue(
2526 Ty: PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace));
2527
2528 // From OpenMP Runtime p_task_dup_t:
2529 // Routine optionally generated by the compiler for setting the lastprivate
2530 // flag and calling needed constructors for private/firstprivate objects (used
2531 // to form taskloop tasks from pattern task) Parameters: dest task, src task,
2532 // lastprivate flag.
2533 // typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
2534
2535 auto *VoidPtrTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2536
2537 FunctionType *DupFuncTy = FunctionType::get(
2538 Result: Builder.getVoidTy(), Params: {VoidPtrTy, VoidPtrTy, Builder.getInt32Ty()},
2539 /*isVarArg=*/false);
2540
2541 Function *DupFunction = Function::Create(Ty: DupFuncTy, Linkage: Function::InternalLinkage,
2542 N: "omp_taskloop_dup", M);
2543 Value *DestTaskArg = DupFunction->getArg(i: 0);
2544 Value *SrcTaskArg = DupFunction->getArg(i: 1);
2545 Value *LastprivateFlagArg = DupFunction->getArg(i: 2);
2546 DestTaskArg->setName("dest_task");
2547 SrcTaskArg->setName("src_task");
2548 LastprivateFlagArg->setName("lastprivate_flag");
2549
2550 IRBuilderBase::InsertPointGuard Guard(Builder);
2551 Builder.SetInsertPoint(
2552 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: DupFunction));
2553
2554 auto GetTaskContextPtrFromArg = [&](Value *Arg) -> Value * {
2555 Type *TaskWithPrivatesTy =
2556 StructType::get(Context&: Builder.getContext(), Elements: {Task, PrivatesTy});
2557 Value *TaskPrivates = Builder.CreateGEP(
2558 Ty: TaskWithPrivatesTy, Ptr: Arg, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2559 Value *ContextPtr = Builder.CreateGEP(
2560 Ty: PrivatesTy, Ptr: TaskPrivates,
2561 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: PrivatesIndex)});
2562 return ContextPtr;
2563 };
2564
2565 Value *DestTaskContextPtr = GetTaskContextPtrFromArg(DestTaskArg);
2566 Value *SrcTaskContextPtr = GetTaskContextPtrFromArg(SrcTaskArg);
2567
2568 DestTaskContextPtr->setName("destPtr");
2569 SrcTaskContextPtr->setName("srcPtr");
2570
2571 InsertPointTy AllocaIP(DupFunction->getEntryBlock().begin());
2572 InsertPointTy CodeGenIP = Builder.saveIP();
2573 Expected<IRBuilderBase::InsertPoint> AfterIPOrError =
2574 DupCB(AllocaIP, CodeGenIP, DestTaskContextPtr, SrcTaskContextPtr);
2575 if (!AfterIPOrError)
2576 return AfterIPOrError.takeError();
2577 Builder.restoreIP(IP: *AfterIPOrError);
2578
2579 Builder.CreateRetVoid();
2580
2581 return DupFunction;
2582}
2583
2584OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskloop(
2585 const LocationDescription &Loc, InsertPointTy AllocaIP,
2586 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2587 llvm::function_ref<llvm::Expected<llvm::CanonicalLoopInfo *>()> LoopInfo,
2588 Value *LBVal, Value *UBVal, Value *StepVal, bool Untied, Value *IfCond,
2589 Value *GrainSize, bool NoGroup, int Sched, Value *Final, bool Mergeable,
2590 Value *Priority, uint64_t NumOfCollapseLoops, TaskDupCallbackTy DupCB,
2591 Value *TaskContextStructPtrVal, bool FreeAgent) {
2592
2593 if (!updateToLocation(Loc))
2594 return InsertPointTy();
2595
2596 uint32_t SrcLocStrSize;
2597 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2598 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2599
2600 BasicBlock *TaskloopExitBB =
2601 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.exit");
2602 BasicBlock *TaskloopBodyBB =
2603 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.body");
2604 BasicBlock *TaskloopAllocaBB =
2605 splitBB(Builder, /*CreateBranch=*/true, Name: "taskloop.alloca");
2606
2607 InsertPointTy TaskloopAllocaIP = TaskloopAllocaBB->begin();
2608 InsertPointTy TaskloopBodyIP = TaskloopBodyBB->begin();
2609
2610 if (Error Err = BodyGenCB(TaskloopAllocaIP, TaskloopBodyIP, TaskloopExitBB))
2611 return Err;
2612
2613 llvm::Expected<llvm::CanonicalLoopInfo *> result = LoopInfo();
2614 if (!result) {
2615 return result.takeError();
2616 }
2617
2618 llvm::CanonicalLoopInfo *CLI = result.get();
2619 auto OI = std::make_unique<OutlineInfo>();
2620 OI->EntryBB = TaskloopAllocaBB;
2621 OI->OuterAllocBB = AllocaIP.getNodeParent();
2622 OI->ExitBB = TaskloopExitBB;
2623 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2624 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2625
2626 // Add the thread ID argument.
2627 SmallVector<Instruction *> ToBeDeleted;
2628 // dummy instruction to be used as a fake argument
2629 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2630 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskloopAllocaIP, Name: "global.tid", AsPtr: false));
2631 Value *FakeLB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2632 InnerAllocaIP: TaskloopAllocaIP, Name: "lb", AsPtr: false, Is64Bit: true);
2633 Value *FakeUB = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2634 InnerAllocaIP: TaskloopAllocaIP, Name: "ub", AsPtr: false, Is64Bit: true);
2635 Value *FakeStep = createFakeIntVal(Builder, OuterAllocaIP: AllocaIP, ToBeDeleted,
2636 InnerAllocaIP: TaskloopAllocaIP, Name: "step", AsPtr: false, Is64Bit: true);
2637 // For Taskloop, we want to force the bounds being the first 3 inputs in the
2638 // aggregate struct
2639 OI->Inputs.insert(X: FakeLB);
2640 OI->Inputs.insert(X: FakeUB);
2641 OI->Inputs.insert(X: FakeStep);
2642 if (TaskContextStructPtrVal)
2643 OI->Inputs.insert(X: TaskContextStructPtrVal);
2644 assert(((TaskContextStructPtrVal && DupCB) ||
2645 (!TaskContextStructPtrVal && !DupCB)) &&
2646 "Task context struct ptr and duplication callback must be both set "
2647 "or both null");
2648
2649 // It isn't safe to run the duplication bodygen callback inside the post
2650 // outlining callback so this has to be run now before we know the real task
2651 // shareds structure type.
2652 unsigned ProgramAddressSpace = M.getDataLayout().getProgramAddressSpace();
2653 Type *PointerTy = PointerType::get(C&: Builder.getContext(), AddressSpace: ProgramAddressSpace);
2654 Type *FakeSharedsTy = StructType::get(
2655 Context&: Builder.getContext(),
2656 Elements: {FakeLB->getType(), FakeUB->getType(), FakeStep->getType(), PointerTy});
2657 Expected<Value *> TaskDupFnOrErr = createTaskDuplicationFunction(
2658 PrivatesTy: FakeSharedsTy,
2659 /*PrivatesIndex: the pointer after the three indices above*/ PrivatesIndex: 3, DupCB);
2660 if (!TaskDupFnOrErr) {
2661 return TaskDupFnOrErr.takeError();
2662 }
2663 Value *TaskDupFn = *TaskDupFnOrErr;
2664
2665 OI->PostOutlineCB = [this, Ident, LBVal, UBVal, StepVal, Untied,
2666 TaskloopAllocaBB, CLI, TaskDupFn, ToBeDeleted, IfCond,
2667 GrainSize, NoGroup, Sched, FakeLB, FakeUB, FakeStep,
2668 FakeSharedsTy, Final, Mergeable, Priority,
2669 NumOfCollapseLoops,
2670 FreeAgent](Function &OutlinedFn) mutable {
2671 // Replace the Stale CI by appropriate RTL function call.
2672 assert(OutlinedFn.hasOneUse() &&
2673 "there must be a single user for the outlined function");
2674 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2675
2676 /* Create the casting for the Bounds Values that can be used when outlining
2677 * to replace the uses of the fakes with real values */
2678 BasicBlock *CodeReplBB = StaleCI->getParent();
2679 Builder.SetInsertPoint(CodeReplBB->getFirstInsertionPt());
2680 Value *CastedLBVal =
2681 Builder.CreateIntCast(V: LBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "lb64");
2682 Value *CastedUBVal =
2683 Builder.CreateIntCast(V: UBVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "ub64");
2684 Value *CastedStepVal =
2685 Builder.CreateIntCast(V: StepVal, DestTy: Builder.getInt64Ty(), isSigned: true, Name: "step64");
2686
2687 Builder.SetInsertPoint(StaleCI);
2688
2689 // Gather the arguments for emitting the runtime call for
2690 // @__kmpc_omp_task_alloc
2691 Function *TaskAllocFn =
2692 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
2693
2694 Value *ThreadID = getOrCreateThreadID(Ident);
2695
2696 if (!NoGroup) {
2697 // Emit runtime call for @__kmpc_taskgroup
2698 Function *TaskgroupFn =
2699 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
2700 Builder.CreateCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
2701 }
2702
2703 // `flags` Argument Configuration
2704 // Task is tied if (Flags & 1) == 1.
2705 // Task is untied if (Flags & 1) == 0.
2706 // Task is final if (Flags & 2) == 2.
2707 // Task is not final if (Flags & 2) == 0.
2708 // Task is mergeable if (Flags & 4) == 4.
2709 // Task is not mergeable if (Flags & 4) == 0.
2710 // Task is priority if (Flags & 32) == 32.
2711 // Task is not priority if (Flags & 32) == 0.
2712 // Task is free-agent eligible if (Flags & 128) == 128.
2713 // Task is not free-agent eligible if (Flags & 128) == 0.
2714 Value *Flags = Builder.getInt32(C: Untied ? 0 : 1);
2715 if (Final)
2716 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 2), RHS: Flags);
2717 if (Mergeable)
2718 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
2719 if (Priority)
2720 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
2721 if (FreeAgent)
2722 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 128), RHS: Flags);
2723
2724 Value *TaskSize = Builder.getInt64(
2725 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
2726
2727 AllocaInst *ArgStructAlloca =
2728 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
2729 assert(ArgStructAlloca &&
2730 "Unable to find the alloca instruction corresponding to arguments "
2731 "for extracted function");
2732 std::optional<TypeSize> ArgAllocSize =
2733 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
2734 assert(ArgAllocSize &&
2735 "Unable to determine size of arguments for extracted function");
2736 Value *SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
2737
2738 // Emit the @__kmpc_omp_task_alloc runtime call
2739 // The runtime call returns a pointer to an area where the task captured
2740 // variables must be copied before the task is run (TaskData)
2741 CallInst *TaskData = Builder.CreateCall(
2742 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
2743 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
2744 /*task_func=*/&OutlinedFn});
2745
2746 Value *Shareds = StaleCI->getArgOperand(i: 1);
2747 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
2748 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
2749 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
2750 Size: SharedsSize);
2751 // Get the pointer to loop lb, ub, step from task ptr
2752 // and set up the lowerbound,upperbound and step values
2753 llvm::Value *Lb = Builder.CreateGEP(
2754 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
2755
2756 llvm::Value *Ub = Builder.CreateGEP(
2757 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 1)});
2758
2759 llvm::Value *Step = Builder.CreateGEP(
2760 Ty: FakeSharedsTy, Ptr: TaskShareds, IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 2)});
2761 llvm::Value *Loadstep = Builder.CreateLoad(Ty: Builder.getInt64Ty(), Ptr: Step);
2762
2763 // set up the arguments for emitting kmpc_taskloop runtime call
2764 // setting values for ifval, nogroup, sched, grainsize, task_dup
2765 Value *IfCondVal =
2766 IfCond ? Builder.CreateIntCast(V: IfCond, DestTy: Builder.getInt32Ty(), isSigned: true)
2767 : Builder.getInt32(C: 1);
2768 // As __kmpc_taskgroup is called manually in OMPIRBuilder, NoGroupVal should
2769 // always be 1 when calling __kmpc_taskloop to ensure it is not called again
2770 Value *NoGroupVal = Builder.getInt32(C: 1);
2771 Value *SchedVal = Builder.getInt32(C: Sched);
2772 Value *GrainSizeVal =
2773 GrainSize ? Builder.CreateIntCast(V: GrainSize, DestTy: Builder.getInt64Ty(), isSigned: true)
2774 : Builder.getInt64(C: 0);
2775 Value *TaskDup = TaskDupFn;
2776
2777 Value *Args[] = {Ident, ThreadID, TaskData, IfCondVal, Lb, Ub,
2778 Loadstep, NoGroupVal, SchedVal, GrainSizeVal, TaskDup};
2779
2780 // taskloop runtime call
2781 Function *TaskloopFn =
2782 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskloop);
2783 Builder.CreateCall(Callee: TaskloopFn, Args);
2784
2785 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup if
2786 // nogroup is not defined
2787 if (!NoGroup) {
2788 Function *EndTaskgroupFn =
2789 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
2790 Builder.CreateCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
2791 }
2792
2793 StaleCI->eraseFromParent();
2794
2795 Builder.SetInsertPoint(TaskloopAllocaBB->begin());
2796
2797 LoadInst *SharedsOutlined =
2798 Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
2799 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
2800 New: SharedsOutlined,
2801 ShouldReplace: [SharedsOutlined](Use &U) { return U.getUser() != SharedsOutlined; });
2802
2803 Value *IV = CLI->getIndVar();
2804 Type *IVTy = IV->getType();
2805 Constant *One = ConstantInt::get(Ty: Builder.getInt64Ty(), V: 1);
2806
2807 // When outlining, CodeExtractor will create GEP's to the LowerBound and
2808 // UpperBound. These GEP's can be reused for loading the tasks respective
2809 // bounds.
2810 Value *TaskLB = nullptr;
2811 Value *TaskUB = nullptr;
2812 Value *TaskStep = nullptr;
2813 Value *LoadTaskLB = nullptr;
2814 Value *LoadTaskUB = nullptr;
2815 Value *LoadTaskStep = nullptr;
2816 for (Instruction &I : *TaskloopAllocaBB) {
2817 if (I.getOpcode() == Instruction::GetElementPtr) {
2818 GetElementPtrInst &Gep = cast<GetElementPtrInst>(Val&: I);
2819 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Gep.getOperand(i_nocapture: 2))) {
2820 switch (CI->getZExtValue()) {
2821 case 0:
2822 TaskLB = &I;
2823 break;
2824 case 1:
2825 TaskUB = &I;
2826 break;
2827 case 2:
2828 TaskStep = &I;
2829 break;
2830 }
2831 }
2832 } else if (I.getOpcode() == Instruction::Load) {
2833 LoadInst &Load = cast<LoadInst>(Val&: I);
2834 if (Load.getPointerOperand() == TaskLB) {
2835 assert(TaskLB != nullptr && "Expected value for TaskLB");
2836 LoadTaskLB = &I;
2837 } else if (Load.getPointerOperand() == TaskUB) {
2838 assert(TaskUB != nullptr && "Expected value for TaskUB");
2839 LoadTaskUB = &I;
2840 } else if (Load.getPointerOperand() == TaskStep) {
2841 assert(TaskStep != nullptr && "Expected value for TaskStep");
2842 LoadTaskStep = &I;
2843 }
2844 }
2845 }
2846
2847 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2848
2849 assert(LoadTaskLB != nullptr && "Expected value for LoadTaskLB");
2850 assert(LoadTaskUB != nullptr && "Expected value for LoadTaskUB");
2851 assert(LoadTaskStep != nullptr && "Expected value for LoadTaskStep");
2852 Value *TripCountMinusOne = Builder.CreateSDiv(
2853 LHS: Builder.CreateSub(LHS: LoadTaskUB, RHS: LoadTaskLB), RHS: LoadTaskStep);
2854 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One, Name: "trip_cnt");
2855 Value *CastedTripCount = Builder.CreateIntCast(V: TripCount, DestTy: IVTy, isSigned: true);
2856 Value *CastedTaskLB = Builder.CreateIntCast(V: LoadTaskLB, DestTy: IVTy, isSigned: true);
2857 // set the trip count in the CLI
2858 CLI->setTripCount(CastedTripCount);
2859
2860 Builder.SetInsertPoint(CLI->getBody()->getFirstInsertionPt());
2861
2862 if (NumOfCollapseLoops > 1) {
2863 llvm::SmallVector<User *> UsersToReplace;
2864 // When using the collapse clause, the bounds of the loop have to be
2865 // adjusted to properly represent the iterator of the outer loop.
2866 Value *IVPlusTaskLB = Builder.CreateAdd(
2867 LHS: CLI->getIndVar(),
2868 RHS: Builder.CreateSub(LHS: CastedTaskLB, RHS: ConstantInt::get(Ty: IVTy, V: 1)));
2869 // To ensure every Use is correctly captured, we first want to record
2870 // which users to replace the value in, and then replace the value.
2871 for (auto IVUse = CLI->getIndVar()->uses().begin();
2872 IVUse != CLI->getIndVar()->uses().end(); IVUse++) {
2873 User *IVUser = IVUse->getUser();
2874 if (auto *Op = dyn_cast<BinaryOperator>(Val: IVUser)) {
2875 if (Op->getOpcode() == Instruction::URem ||
2876 Op->getOpcode() == Instruction::UDiv) {
2877 UsersToReplace.push_back(Elt: IVUser);
2878 }
2879 }
2880 }
2881 for (User *User : UsersToReplace) {
2882 User->replaceUsesOfWith(From: CLI->getIndVar(), To: IVPlusTaskLB);
2883 }
2884 } else {
2885 // The canonical loop is generated with a fixed lower bound. We need to
2886 // update the index calculation code to use the task's lower bound. The
2887 // generated code looks like this:
2888 // %omp_loop.iv = phi ...
2889 // ...
2890 // %tmp = mul [type] %omp_loop.iv, step
2891 // %user_index = add [type] tmp, lb
2892 // OpenMPIRBuilder constructs canonical loops to have exactly three uses
2893 // of the normalised induction variable:
2894 // 1. This one: converting the normalised IV to the user IV
2895 // 2. The increment (add)
2896 // 3. The comparison against the trip count (icmp)
2897 // (1) is the only use that is a mul followed by an add so this cannot
2898 // match other IR.
2899 assert(CLI->getIndVar()->getNumUses() == 3 &&
2900 "Canonical loop should have exactly three uses of the ind var");
2901 for (User *IVUser : CLI->getIndVar()->users()) {
2902 if (auto *Mul = dyn_cast<BinaryOperator>(Val: IVUser)) {
2903 if (Mul->getOpcode() == Instruction::Mul) {
2904 for (User *MulUser : Mul->users()) {
2905 if (auto *Add = dyn_cast<BinaryOperator>(Val: MulUser)) {
2906 if (Add->getOpcode() == Instruction::Add) {
2907 Add->setOperand(i_nocapture: 1, Val_nocapture: CastedTaskLB);
2908 }
2909 }
2910 }
2911 }
2912 }
2913 }
2914 }
2915
2916 FakeLB->replaceAllUsesWith(V: CastedLBVal);
2917 FakeUB->replaceAllUsesWith(V: CastedUBVal);
2918 FakeStep->replaceAllUsesWith(V: CastedStepVal);
2919 for (Instruction *I : llvm::reverse(C&: ToBeDeleted)) {
2920 I->eraseFromParent();
2921 }
2922 };
2923
2924 addOutlineInfo(OI: std::move(OI));
2925 Builder.SetInsertPoint(TaskloopExitBB->begin());
2926 return Builder.saveIP();
2927}
2928
2929llvm::StructType *OpenMPIRBuilder::getKmpTaskAffinityInfoTy() {
2930 llvm::Type *IntPtrTy = llvm::Type::getIntNTy(
2931 C&: M.getContext(), N: M.getDataLayout().getPointerSizeInBits());
2932 return llvm::StructType::get(elt1: IntPtrTy, elts: IntPtrTy,
2933 elts: llvm::Type::getInt32Ty(C&: M.getContext()));
2934}
2935
2936OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTask(
2937 const LocationDescription &Loc, InsertPointTy AllocaIP,
2938 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB,
2939 bool Tied, Value *Final, Value *IfCondition,
2940 const DependenciesInfo &Dependencies, const AffinityData &Affinities,
2941 bool Mergeable, Value *EventHandle, Value *Priority, bool FreeAgent) {
2942
2943 if (!updateToLocation(Loc))
2944 return InsertPointTy();
2945
2946 uint32_t SrcLocStrSize;
2947 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2948 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2949 // The current basic block is split into four basic blocks. After outlining,
2950 // they will be mapped as follows:
2951 // ```
2952 // def current_fn() {
2953 // current_basic_block:
2954 // br label %task.exit
2955 // task.exit:
2956 // ; instructions after task
2957 // }
2958 // def outlined_fn() {
2959 // task.alloca:
2960 // br label %task.body
2961 // task.body:
2962 // ret void
2963 // }
2964 // ```
2965 BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.exit");
2966 BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.body");
2967 BasicBlock *TaskAllocaBB =
2968 splitBB(Builder, /*CreateBranch=*/true, Name: "task.alloca");
2969
2970 InsertPointTy TaskAllocaIP = TaskAllocaBB->begin();
2971 InsertPointTy TaskBodyIP = TaskBodyBB->begin();
2972 if (Error Err = BodyGenCB(TaskAllocaIP, TaskBodyIP, TaskExitBB))
2973 return Err;
2974
2975 auto OI = std::make_unique<OutlineInfo>();
2976 OI->EntryBB = TaskAllocaBB;
2977 OI->OuterAllocBB = AllocaIP.getNodeParent();
2978 OI->ExitBB = TaskExitBB;
2979 OI->OuterDeallocBBs.reserve(N: DeallocBlocks.size());
2980 copy(Range&: DeallocBlocks, Out: OI->OuterDeallocBBs.end());
2981
2982 // Add the thread ID argument.
2983 SmallVector<Instruction *, 4> ToBeDeleted;
2984 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
2985 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskAllocaIP, Name: "global.tid", AsPtr: false));
2986
2987 OI->PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
2988 Affinities, Mergeable, Priority, EventHandle, FreeAgent,
2989 TaskAllocaBB,
2990 ToBeDeleted](Function &OutlinedFn) mutable {
2991 // Replace the Stale CI by appropriate RTL function call.
2992 assert(OutlinedFn.hasOneUse() &&
2993 "there must be a single user for the outlined function");
2994 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
2995
2996 // HasShareds is true if any variables are captured in the outlined region,
2997 // false otherwise.
2998 bool HasShareds = StaleCI->arg_size() > 1;
2999 Builder.SetInsertPoint(StaleCI);
3000
3001 // Gather the arguments for emitting the runtime call for
3002 // @__kmpc_omp_task_alloc
3003 Function *TaskAllocFn =
3004 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
3005
3006 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
3007 // call.
3008 Value *ThreadID = getOrCreateThreadID(Ident);
3009
3010 // Argument - `flags`
3011 // Task is tied iff (Flags & 1) == 1.
3012 // Task is untied iff (Flags & 1) == 0.
3013 // Task is final iff (Flags & 2) == 2.
3014 // Task is not final iff (Flags & 2) == 0.
3015 // Task is mergeable or merged-if0 iff (Flags & 4) == 4.
3016 // Task is neither mergeable nor merged-if0 iff (Flags & 4) == 0.
3017 // Task is detachable iff (Flags & 64) == 64.
3018 // Task is not detachable iff (Flags & 64) == 0.
3019 // Task is priority iff (Flags & 32) == 32.
3020 // Task is not priority iff (Flags & 32) == 0.
3021 // Task is free-agent eligible iff (Flags & 128) == 128.
3022 // Task is not free-agent eligible iff (Flags & 128) == 0.
3023 // TODO: Handle the other flags.
3024 Value *Flags = Builder.getInt32(C: Tied);
3025 auto *ConstIfCondition = dyn_cast_or_null<ConstantInt>(Val: IfCondition);
3026 bool UseMergedIf0Path = ConstIfCondition && ConstIfCondition->isZero();
3027 if (Final) {
3028 Value *FinalFlag =
3029 Builder.CreateSelect(C: Final, True: Builder.getInt32(C: 2), False: Builder.getInt32(C: 0));
3030 Flags = Builder.CreateOr(LHS: FinalFlag, RHS: Flags);
3031 }
3032
3033 if (Mergeable || UseMergedIf0Path)
3034 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 4), RHS: Flags);
3035 if (EventHandle)
3036 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 64), RHS: Flags);
3037 if (Priority)
3038 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 32), RHS: Flags);
3039 if (FreeAgent)
3040 Flags = Builder.CreateOr(LHS: Builder.getInt32(C: 128), RHS: Flags);
3041
3042 // Argument - `sizeof_kmp_task_t` (TaskSize)
3043 // Tasksize refers to the size in bytes of kmp_task_t data structure
3044 // including private vars accessed in task.
3045 // TODO: add kmp_task_t_with_privates (privates)
3046 Value *TaskSize = Builder.getInt64(
3047 C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
3048
3049 // Argument - `sizeof_shareds` (SharedsSize)
3050 // SharedsSize refers to the shareds array size in the kmp_task_t data
3051 // structure.
3052 Value *SharedsSize = Builder.getInt64(C: 0);
3053 if (HasShareds) {
3054 AllocaInst *ArgStructAlloca =
3055 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
3056 assert(ArgStructAlloca &&
3057 "Unable to find the alloca instruction corresponding to arguments "
3058 "for extracted function");
3059 std::optional<TypeSize> ArgAllocSize =
3060 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
3061 assert(ArgAllocSize &&
3062 "Unable to determine size of arguments for extracted function");
3063 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
3064 }
3065 // Emit the @__kmpc_omp_task_alloc runtime call
3066 // The runtime call returns a pointer to an area where the task captured
3067 // variables must be copied before the task is run (TaskData)
3068 CallInst *TaskData = createRuntimeFunctionCall(
3069 Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
3070 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
3071 /*task_func=*/&OutlinedFn});
3072
3073 if (Affinities.Count && Affinities.Info) {
3074 Function *RegAffFn = getOrCreateRuntimeFunctionPtr(
3075 FnID: OMPRTL___kmpc_omp_reg_task_with_affinity);
3076
3077 createRuntimeFunctionCall(Callee: RegAffFn, Args: {Ident, ThreadID, TaskData,
3078 Affinities.Count, Affinities.Info});
3079 }
3080
3081 // Emit detach clause initialization.
3082 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
3083 // task_descriptor);
3084 if (EventHandle) {
3085 Function *TaskDetachFn = getOrCreateRuntimeFunctionPtr(
3086 FnID: OMPRTL___kmpc_task_allow_completion_event);
3087 llvm::Value *EventVal =
3088 createRuntimeFunctionCall(Callee: TaskDetachFn, Args: {Ident, ThreadID, TaskData});
3089 llvm::Value *EventHandleAddr =
3090 Builder.CreatePointerBitCastOrAddrSpaceCast(V: EventHandle,
3091 DestTy: Builder.getPtrTy(AddrSpace: 0));
3092 EventVal = Builder.CreatePtrToInt(V: EventVal, DestTy: Builder.getInt64Ty());
3093 Builder.CreateStore(Val: EventVal, Ptr: EventHandleAddr);
3094 }
3095 // Copy the arguments for outlined function
3096 if (HasShareds) {
3097 Value *Shareds = StaleCI->getArgOperand(i: 1);
3098 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
3099 Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
3100 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
3101 Size: SharedsSize);
3102 }
3103
3104 if (Priority) {
3105 //
3106 // The return type of "__kmpc_omp_task_alloc" is "kmp_task_t *",
3107 // we populate the priority information into the "kmp_task_t" here
3108 //
3109 // The struct "kmp_task_t" definition is available in kmp.h
3110 // kmp_task_t = { shareds, routine, part_id, data1, data2 }
3111 // data2 is used for priority
3112 //
3113 Type *Int32Ty = Builder.getInt32Ty();
3114 Constant *Zero = ConstantInt::get(Ty: Int32Ty, V: 0);
3115 // kmp_task_t* => { ptr }
3116 Type *TaskPtr = StructType::get(elt1: VoidPtr);
3117 Value *TaskGEP =
3118 Builder.CreateInBoundsGEP(Ty: TaskPtr, Ptr: TaskData, IdxList: {Zero, Zero});
3119 // kmp_task_t => { ptr, ptr, i32, ptr, ptr }
3120 Type *TaskStructType = StructType::get(
3121 elt1: VoidPtr, elts: VoidPtr, elts: Builder.getInt32Ty(), elts: VoidPtr, elts: VoidPtr);
3122 Value *PriorityData = Builder.CreateInBoundsGEP(
3123 Ty: TaskStructType, Ptr: TaskGEP, IdxList: {Zero, ConstantInt::get(Ty: Int32Ty, V: 4)});
3124 // kmp_cmplrdata_t => { ptr, ptr }
3125 Type *CmplrStructType = StructType::get(elt1: VoidPtr, elts: VoidPtr);
3126 Value *CmplrData = Builder.CreateInBoundsGEP(Ty: CmplrStructType,
3127 Ptr: PriorityData, IdxList: {Zero, Zero});
3128 Builder.CreateStore(Val: Priority, Ptr: CmplrData);
3129 }
3130
3131 Value *DepArray = nullptr;
3132 Value *NumDeps = nullptr;
3133 if (Dependencies.DepArray) {
3134 DepArray = Dependencies.DepArray;
3135 NumDeps = Dependencies.NumDeps;
3136 } else if (!Dependencies.Deps.empty()) {
3137 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
3138 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
3139 }
3140
3141 // In the presence of the `if` clause, the following IR is generated:
3142 // ...
3143 // %data = call @__kmpc_omp_task_alloc(...)
3144 // br i1 %if_condition, label %then, label %else
3145 // then:
3146 // call @__kmpc_omp_task(...)
3147 // br label %exit
3148 // else:
3149 // ;; Wait for resolution of dependencies, if any, before
3150 // ;; beginning the task
3151 // call @__kmpc_omp_wait_deps(...)
3152 // call @__kmpc_omp_task_begin_if0(...)
3153 // call @outlined_fn(...)
3154 // call @__kmpc_omp_task_complete_if0(...)
3155 // br label %exit
3156 // exit:
3157 // ...
3158 if (IfCondition && !UseMergedIf0Path) {
3159 // `SplitBlockAndInsertIfThenElse` requires the block to have a
3160 // terminator.
3161 splitBB(Builder, /*CreateBranch=*/true, Name: "if.end");
3162 Instruction *IfTerminator =
3163 Builder.GetInsertPoint()->getParent()->getTerminator();
3164 Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
3165 Builder.SetInsertPoint(IfTerminator);
3166 SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: IfTerminator, ThenTerm: &ThenTI,
3167 ElseTerm: &ElseTI);
3168 Builder.SetInsertPoint(ElseTI);
3169
3170 if (DepArray) {
3171 Function *TaskWaitFn =
3172 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
3173 createRuntimeFunctionCall(
3174 Callee: TaskWaitFn,
3175 Args: {Ident, ThreadID, NumDeps, DepArray,
3176 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3177 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3178 }
3179 Function *TaskBeginFn =
3180 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
3181 Function *TaskCompleteFn =
3182 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
3183 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
3184 CallInst *CI = nullptr;
3185 if (HasShareds)
3186 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID, TaskData});
3187 else
3188 CI = createRuntimeFunctionCall(Callee: &OutlinedFn, Args: {ThreadID});
3189 CI->setDebugLoc(StaleCI->getDebugLoc());
3190 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
3191 Builder.SetInsertPoint(ThenTI);
3192 }
3193
3194 if (DepArray) {
3195 Function *TaskFn =
3196 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
3197 createRuntimeFunctionCall(
3198 Callee: TaskFn,
3199 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
3200 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
3201 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
3202
3203 } else {
3204 // Emit the @__kmpc_omp_task runtime call to spawn the task
3205 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
3206 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
3207 }
3208
3209 StaleCI->eraseFromParent();
3210
3211 Builder.SetInsertPoint(TaskAllocaBB->begin());
3212 if (HasShareds) {
3213 LoadInst *Shareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
3214 OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
3215 New: Shareds, ShouldReplace: [Shareds](Use &U) { return U.getUser() != Shareds; });
3216 }
3217
3218 // The insert point may refer to one of the instructions about to be
3219 // deleted. It is not needed anymore so clear it instead of leaving it
3220 // dangling.
3221 Builder.ClearInsertionPoint();
3222 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
3223 I->eraseFromParent();
3224 };
3225
3226 addOutlineInfo(OI: std::move(OI));
3227 Builder.SetInsertPoint(TaskExitBB->begin());
3228
3229 return Builder.saveIP();
3230}
3231
3232OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTaskgroup(
3233 const LocationDescription &Loc, InsertPointTy AllocaIP,
3234 ArrayRef<BasicBlock *> DeallocBlocks, BodyGenCallbackTy BodyGenCB) {
3235 if (!updateToLocation(Loc))
3236 return InsertPointTy();
3237
3238 uint32_t SrcLocStrSize;
3239 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3240 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3241 Value *ThreadID = getOrCreateThreadID(Ident);
3242
3243 // Emit the @__kmpc_taskgroup runtime call to start the taskgroup
3244 Function *TaskgroupFn =
3245 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
3246 createRuntimeFunctionCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
3247
3248 BasicBlock *TaskgroupExitBB = splitBB(Builder, CreateBranch: true, Name: "taskgroup.exit");
3249 if (Error Err = BodyGenCB(AllocaIP, Builder.saveIP(), DeallocBlocks))
3250 return Err;
3251
3252 Builder.SetInsertPoint(TaskgroupExitBB);
3253 // Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
3254 Function *EndTaskgroupFn =
3255 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
3256 createRuntimeFunctionCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
3257
3258 return Builder.saveIP();
3259}
3260
3261OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSections(
3262 const LocationDescription &Loc, InsertPointTy AllocaIP,
3263 ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
3264 FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
3265 assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");
3266
3267 if (!updateToLocation(Loc))
3268 return Loc.IP;
3269
3270 FinalizationStack.push_back(Elt: {FiniCB, OMPD_sections, IsCancellable});
3271
3272 // Each section is emitted as a switch case
3273 // Each finalization callback is handled from clang.EmitOMPSectionDirective()
3274 // -> OMP.createSection() which generates the IR for each section
3275 // Iterate through all sections and emit a switch construct:
3276 // switch (IV) {
3277 // case 0:
3278 // <SectionStmt[0]>;
3279 // break;
3280 // ...
3281 // case <NumSection> - 1:
3282 // <SectionStmt[<NumSection> - 1]>;
3283 // break;
3284 // }
3285 // ...
3286 // section_loop.after:
3287 // <FiniCB>;
3288 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) -> Error {
3289 Builder.restoreIP(IP: CodeGenIP);
3290 BasicBlock *Continue =
3291 splitBBWithSuffix(Builder, /*CreateBranch=*/false, Suffix: ".sections.after");
3292 Function *CurFn = Continue->getParent();
3293 SwitchInst *SwitchStmt = Builder.CreateSwitch(V: IndVar, Dest: Continue);
3294
3295 unsigned CaseNumber = 0;
3296 for (auto SectionCB : SectionCBs) {
3297 BasicBlock *CaseBB = BasicBlock::Create(
3298 Context&: M.getContext(), Name: "omp_section_loop.body.case", Parent: CurFn, InsertBefore: Continue);
3299 SwitchStmt->addCase(OnVal: Builder.getInt32(C: CaseNumber), Dest: CaseBB);
3300 Builder.SetInsertPoint(CaseBB);
3301 UncondBrInst *CaseEndBr = Builder.CreateBr(Dest: Continue);
3302 if (Error Err = SectionCB(InsertPointTy(), CaseEndBr->getIterator(), {}))
3303 return Err;
3304 CaseNumber++;
3305 }
3306 // remove the existing terminator from body BB since there can be no
3307 // terminators after switch/case
3308 return Error::success();
3309 };
3310 // Loop body ends here
3311 // LowerBound, UpperBound, and STride for createCanonicalLoop
3312 Type *I32Ty = Type::getInt32Ty(C&: M.getContext());
3313 Value *LB = ConstantInt::get(Ty: I32Ty, V: 0);
3314 Value *UB = ConstantInt::get(Ty: I32Ty, V: SectionCBs.size());
3315 Value *ST = ConstantInt::get(Ty: I32Ty, V: 1);
3316 Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
3317 Loc, BodyGenCB: LoopBodyGenCB, Start: LB, Stop: UB, Step: ST, IsSigned: true, InclusiveStop: false, ComputeIP: AllocaIP, Name: "section_loop");
3318 if (!LoopInfo)
3319 return LoopInfo.takeError();
3320
3321 InsertPointOrErrorTy WsloopIP =
3322 applyStaticWorkshareLoop(DL: Loc.DL, CLI: *LoopInfo, AllocaIP,
3323 LoopType: WorksharingLoopType::ForStaticLoop, NeedsBarrier: !IsNowait);
3324 if (!WsloopIP)
3325 return WsloopIP.takeError();
3326 InsertPointTy AfterIP = *WsloopIP;
3327
3328 BasicBlock *LoopFini = AfterIP.getNodeParent()->getSinglePredecessor();
3329 assert(LoopFini && "Bad structure of static workshare loop finalization");
3330
3331 // Apply the finalization callback in LoopAfterBB
3332 auto FiniInfo = FinalizationStack.pop_back_val();
3333 assert(FiniInfo.DK == OMPD_sections &&
3334 "Unexpected finalization stack state!");
3335 if (Error Err = FiniInfo.mergeFiniBB(Builder, OtherFiniBB: LoopFini))
3336 return Err;
3337
3338 return AfterIP;
3339}
3340
3341OpenMPIRBuilder::InsertPointOrErrorTy
3342OpenMPIRBuilder::createSection(const LocationDescription &Loc,
3343 BodyGenCallbackTy BodyGenCB,
3344 FinalizeCallbackTy FiniCB) {
3345 if (!updateToLocation(Loc))
3346 return Loc.IP;
3347
3348 auto FiniCBWrapper = [&](InsertPointTy IP) {
3349 if (IP != IP.getNodeParent()->end())
3350 return FiniCB(IP);
3351 // This must be done otherwise any nested constructs using FinalizeOMPRegion
3352 // will fail because that function requires the Finalization Basic Block to
3353 // have a terminator, which is already removed by EmitOMPRegionBody.
3354 // IP is currently at cancelation block.
3355 // We need to backtrack to the condition block to fetch
3356 // the exit block and create a branch from cancelation
3357 // to exit block.
3358 IRBuilder<>::InsertPointGuard IPG(Builder);
3359 Builder.restoreIP(IP);
3360 auto *CaseBB = Loc.IP.getNodeParent();
3361 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
3362 auto *ExitBB = CondBB->getTerminator()->getSuccessor(Idx: 1);
3363 Instruction *I = Builder.CreateBr(Dest: ExitBB);
3364 IP = I->getIterator();
3365 return FiniCB(IP);
3366 };
3367
3368 Directive OMPD = Directive::OMPD_sections;
3369 // Since we are using Finalization Callback here, HasFinalize
3370 // and IsCancellable have to be true
3371 return EmitOMPInlinedRegion(OMPD, EntryCall: nullptr, ExitCall: nullptr, BodyGenCB, FiniCB: FiniCBWrapper,
3372 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true,
3373 /*IsCancellable*/ true);
3374}
3375
3376static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I) {
3377 BasicBlock::iterator IT(I);
3378 IT++;
3379 return IT;
3380}
3381
3382Value *OpenMPIRBuilder::getGPUThreadID() {
3383 return createRuntimeFunctionCall(
3384 Callee: getOrCreateRuntimeFunction(M,
3385 FnID: OMPRTL___kmpc_get_hardware_thread_id_in_block),
3386 Args: {});
3387}
3388
3389Value *OpenMPIRBuilder::getGPUWarpSize() {
3390 return createRuntimeFunctionCall(
3391 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_get_warp_size), Args: {});
3392}
3393
3394Value *OpenMPIRBuilder::getNVPTXWarpID() {
3395 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3396 return Builder.CreateAShr(LHS: getGPUThreadID(), RHS: LaneIDBits, Name: "nvptx_warp_id");
3397}
3398
3399Value *OpenMPIRBuilder::getNVPTXLaneID() {
3400 unsigned LaneIDBits = Log2_32(Value: Config.getGridValue().GV_Warp_Size);
3401 assert(LaneIDBits < 32 && "Invalid LaneIDBits size in NVPTX device.");
3402 unsigned LaneIDMask = ~0u >> (32u - LaneIDBits);
3403 return Builder.CreateAnd(LHS: getGPUThreadID(), RHS: Builder.getInt32(C: LaneIDMask),
3404 Name: "nvptx_lane_id");
3405}
3406
3407Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP, Value *From,
3408 Type *ToType) {
3409 Type *FromType = From->getType();
3410 uint64_t FromSize = M.getDataLayout().getTypeStoreSize(Ty: FromType);
3411 uint64_t ToSize = M.getDataLayout().getTypeStoreSize(Ty: ToType);
3412 assert(FromSize > 0 && "From size must be greater than zero");
3413 assert(ToSize > 0 && "To size must be greater than zero");
3414 if (FromType == ToType)
3415 return From;
3416 if (FromSize == ToSize)
3417 return Builder.CreateBitCast(V: From, DestTy: ToType);
3418 if (ToType->isIntegerTy() && FromType->isIntegerTy())
3419 return Builder.CreateIntCast(V: From, DestTy: ToType, /*isSigned*/ true);
3420 InsertPointTy SaveIP = Builder.saveIP();
3421 Builder.restoreIP(IP: AllocaIP);
3422 Value *CastItem = Builder.CreateAlloca(Ty: ToType);
3423 Builder.restoreIP(IP: SaveIP);
3424
3425 Value *ValCastItem = Builder.CreatePointerBitCastOrAddrSpaceCast(
3426 V: CastItem, DestTy: Builder.getPtrTy(AddrSpace: 0));
3427 Builder.CreateStore(Val: From, Ptr: ValCastItem);
3428 return Builder.CreateLoad(Ty: ToType, Ptr: CastItem);
3429}
3430
3431Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
3432 Value *Element,
3433 Type *ElementType,
3434 Value *Offset) {
3435 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElementType);
3436 assert(Size <= 8 && "Unsupported bitwidth in shuffle instruction");
3437
3438 // Cast all types to 32- or 64-bit values before calling shuffle routines.
3439 Type *CastTy = Builder.getIntNTy(N: Size <= 4 ? 32 : 64);
3440 Value *ElemCast = castValueToType(AllocaIP, From: Element, ToType: CastTy);
3441 Value *WarpSize =
3442 Builder.CreateIntCast(V: getGPUWarpSize(), DestTy: Builder.getInt16Ty(), isSigned: true);
3443 Function *ShuffleFunc = getOrCreateRuntimeFunctionPtr(
3444 FnID: Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
3445 : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
3446 Value *WarpSizeCast =
3447 Builder.CreateIntCast(V: WarpSize, DestTy: Builder.getInt16Ty(), /*isSigned=*/true);
3448 Value *ShuffleCall =
3449 createRuntimeFunctionCall(Callee: ShuffleFunc, Args: {ElemCast, Offset, WarpSizeCast});
3450 // The shuffle runtime functions return a 32- or 64-bit value. Cast it back
3451 // down to the requested element type, otherwise storing the result would
3452 // write past the end of an element narrower than the shuffle width.
3453 return castValueToType(AllocaIP, From: ShuffleCall, ToType: ElementType);
3454}
3455
3456void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP, Value *SrcAddr,
3457 Value *DstAddr, Type *ElemType,
3458 Value *Offset, Type *ReductionArrayTy,
3459 bool IsByRefElem) {
3460 uint64_t Size = M.getDataLayout().getTypeStoreSize(Ty: ElemType);
3461 // Create the loop over the big sized data.
3462 // ptr = (void*)Elem;
3463 // ptrEnd = (void*) Elem + 1;
3464 // Step = 8;
3465 // while (ptr + Step < ptrEnd)
3466 // shuffle((int64_t)*ptr);
3467 // Step = 4;
3468 // while (ptr + Step < ptrEnd)
3469 // shuffle((int32_t)*ptr);
3470 // ...
3471 Type *IndexTy = Builder.getIndexTy(
3472 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3473 Value *ElemPtr = DstAddr;
3474 Value *Ptr = SrcAddr;
3475 for (unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
3476 if (Size < IntSize)
3477 continue;
3478 Type *IntType = Builder.getIntNTy(N: IntSize * 8);
3479 Ptr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3480 V: Ptr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: Ptr->getName() + ".ascast");
3481 Value *SrcAddrGEP =
3482 Builder.CreateGEP(Ty: ElemType, Ptr: SrcAddr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3483 ElemPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3484 V: ElemPtr, DestTy: Builder.getPtrTy(AddrSpace: 0), Name: ElemPtr->getName() + ".ascast");
3485
3486 Function *CurFunc = Builder.GetInsertBlock()->getParent();
3487 if ((Size / IntSize) > 1) {
3488 Value *PtrEnd = Builder.CreatePointerBitCastOrAddrSpaceCast(
3489 V: SrcAddrGEP, DestTy: Builder.getPtrTy());
3490 BasicBlock *PreCondBB =
3491 BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.pre_cond");
3492 BasicBlock *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.then");
3493 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: ".shuffle.exit");
3494 BasicBlock *CurrentBB = Builder.GetInsertBlock();
3495 emitBlock(BB: PreCondBB, CurFn: CurFunc);
3496 PHINode *PhiSrc =
3497 Builder.CreatePHI(Ty: Ptr->getType(), /*NumReservedValues=*/2);
3498 PhiSrc->addIncoming(V: Ptr, BB: CurrentBB);
3499 PHINode *PhiDest =
3500 Builder.CreatePHI(Ty: ElemPtr->getType(), /*NumReservedValues=*/2);
3501 PhiDest->addIncoming(V: ElemPtr, BB: CurrentBB);
3502 Ptr = PhiSrc;
3503 ElemPtr = PhiDest;
3504 Value *PtrDiff = Builder.CreatePtrDiff(
3505 ElemTy: Builder.getInt8Ty(), LHS: PtrEnd,
3506 RHS: Builder.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: Builder.getPtrTy()));
3507 Builder.CreateCondBr(
3508 Cond: Builder.CreateICmpSGT(LHS: PtrDiff, RHS: Builder.getInt64(C: IntSize - 1)), True: ThenBB,
3509 False: ExitBB);
3510 emitBlock(BB: ThenBB, CurFn: CurFunc);
3511 Value *Res = createRuntimeShuffleFunction(
3512 AllocaIP,
3513 Element: Builder.CreateAlignedLoad(
3514 Ty: IntType, Ptr, Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType)),
3515 ElementType: IntType, Offset);
3516 Builder.CreateAlignedStore(Val: Res, Ptr: ElemPtr,
3517 Align: M.getDataLayout().getPrefTypeAlign(Ty: ElemType));
3518 Value *LocalPtr =
3519 Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3520 Value *LocalElemPtr =
3521 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3522 PhiSrc->addIncoming(V: LocalPtr, BB: ThenBB);
3523 PhiDest->addIncoming(V: LocalElemPtr, BB: ThenBB);
3524 emitBranch(Target: PreCondBB);
3525 emitBlock(BB: ExitBB, CurFn: CurFunc);
3526 } else {
3527 // The shuffled value comes back as the chunk's integer type, so the
3528 // store covers exactly this chunk regardless of what ElemType is.
3529 Value *Res = createRuntimeShuffleFunction(
3530 AllocaIP, Element: Builder.CreateLoad(Ty: IntType, Ptr), ElementType: IntType, Offset);
3531 Builder.CreateStore(Val: Res, Ptr: ElemPtr);
3532 Ptr = Builder.CreateGEP(Ty: IntType, Ptr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3533 ElemPtr =
3534 Builder.CreateGEP(Ty: IntType, Ptr: ElemPtr, IdxList: {ConstantInt::get(Ty: IndexTy, V: 1)});
3535 }
3536 Size = Size % IntSize;
3537 }
3538}
3539
3540Error OpenMPIRBuilder::emitReductionListCopy(
3541 InsertPointTy AllocaIP, CopyAction Action, Type *ReductionArrayTy,
3542 ArrayRef<ReductionInfo> ReductionInfos, Value *SrcBase, Value *DestBase,
3543 ArrayRef<bool> IsByRef, CopyOptionsTy CopyOptions) {
3544 Type *IndexTy = Builder.getIndexTy(
3545 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3546 Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
3547
3548 // Iterates, element-by-element, through the source Reduce list and
3549 // make a copy.
3550 for (auto En : enumerate(First&: ReductionInfos)) {
3551 const ReductionInfo &RI = En.value();
3552 Value *SrcElementAddr = nullptr;
3553 AllocaInst *DestAlloca = nullptr;
3554 Value *DestElementAddr = nullptr;
3555 Value *DestElementPtrAddr = nullptr;
3556 // Should we shuffle in an element from a remote lane?
3557 bool ShuffleInElement = false;
3558 // Set to true to update the pointer in the dest Reduce list to a
3559 // newly created element.
3560 bool UpdateDestListPtr = false;
3561
3562 // Step 1.1: Get the address for the src element in the Reduce list.
3563 Value *SrcElementPtrAddr = Builder.CreateInBoundsGEP(
3564 Ty: ReductionArrayTy, Ptr: SrcBase,
3565 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3566 SrcElementAddr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrAddr);
3567
3568 // Step 1.2: Create a temporary to store the element in the destination
3569 // Reduce list.
3570 DestElementPtrAddr = Builder.CreateInBoundsGEP(
3571 Ty: ReductionArrayTy, Ptr: DestBase,
3572 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
3573 bool IsByRefElem = (!IsByRef.empty() && IsByRef[En.index()]);
3574 switch (Action) {
3575 case CopyAction::RemoteLaneToThread: {
3576 InsertPointTy CurIP = Builder.saveIP();
3577 Builder.restoreIP(IP: AllocaIP);
3578
3579 Type *DestAllocaType =
3580 IsByRefElem ? RI.ByRefAllocatedType : RI.ElementType;
3581 DestAlloca = Builder.CreateAlloca(Ty: DestAllocaType, ArraySize: nullptr,
3582 Name: ".omp.reduction.element");
3583 DestAlloca->setAlignment(
3584 M.getDataLayout().getPrefTypeAlign(Ty: DestAllocaType));
3585 DestElementAddr = DestAlloca;
3586 DestElementAddr =
3587 Builder.CreateAddrSpaceCast(V: DestElementAddr, DestTy: Builder.getPtrTy(),
3588 Name: DestElementAddr->getName() + ".ascast");
3589 Builder.restoreIP(IP: CurIP);
3590 ShuffleInElement = true;
3591 UpdateDestListPtr = true;
3592 break;
3593 }
3594 case CopyAction::ThreadCopy: {
3595 DestElementAddr =
3596 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DestElementPtrAddr);
3597 break;
3598 }
3599 }
3600
3601 // Now that all active lanes have read the element in the
3602 // Reduce list, shuffle over the value from the remote lane.
3603 if (ShuffleInElement) {
3604 Type *ShuffleType = RI.ElementType;
3605 Value *ShuffleSrcAddr = SrcElementAddr;
3606 Value *ShuffleDestAddr = DestElementAddr;
3607 AllocaInst *LocalStorage = nullptr;
3608
3609 if (IsByRefElem) {
3610 assert(RI.ByRefElementType && "Expected by-ref element type to be set");
3611 assert(RI.ByRefAllocatedType &&
3612 "Expected by-ref allocated type to be set");
3613 // For by-ref reductions, we need to copy from the remote lane the
3614 // actual value of the partial reduction computed by that remote lane;
3615 // rather than, for example, a pointer to that data or, even worse, a
3616 // pointer to the descriptor of the by-ref reduction element.
3617 ShuffleType = RI.ByRefElementType;
3618
3619 if (RI.DataPtrPtrGen) {
3620 // Descriptor-based by-ref: extract data pointer from descriptor.
3621 InsertPointOrErrorTy GenResult = RI.DataPtrPtrGen(
3622 Builder.saveIP(), ShuffleSrcAddr, ShuffleSrcAddr);
3623
3624 if (!GenResult)
3625 return GenResult.takeError();
3626
3627 ShuffleSrcAddr =
3628 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ShuffleSrcAddr);
3629
3630 {
3631 InsertPointTy OldIP = Builder.saveIP();
3632 Builder.restoreIP(IP: AllocaIP);
3633
3634 LocalStorage = Builder.CreateAlloca(Ty: ShuffleType);
3635 Builder.restoreIP(IP: OldIP);
3636 ShuffleDestAddr = LocalStorage;
3637 }
3638 } else {
3639 // Non-descriptor by-ref: the pointer already references data
3640 // directly. Shuffle into the destination alloca.
3641 ShuffleDestAddr = DestElementAddr;
3642 }
3643 }
3644
3645 shuffleAndStore(AllocaIP, SrcAddr: ShuffleSrcAddr, DstAddr: ShuffleDestAddr, ElemType: ShuffleType,
3646 Offset: RemoteLaneOffset, ReductionArrayTy, IsByRefElem);
3647
3648 if (IsByRefElem && RI.DataPtrPtrGen) {
3649 // Copy descriptor from source and update base_ptr to shuffled data
3650 Value *DestDescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3651 V: DestAlloca, DestTy: Builder.getPtrTy(), Name: ".ascast");
3652
3653 InsertPointOrErrorTy GenResult = generateReductionDescriptor(
3654 DescriptorAddr: DestDescriptorAddr, DataPtr: LocalStorage, SrcDescriptorAddr: SrcElementAddr,
3655 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
3656
3657 if (!GenResult)
3658 return GenResult.takeError();
3659 }
3660 } else {
3661 switch (RI.EvaluationKind) {
3662 case EvalKind::Scalar: {
3663 Value *Elem = Builder.CreateLoad(Ty: RI.ElementType, Ptr: SrcElementAddr);
3664 // Store the source element value to the dest element address.
3665 Builder.CreateStore(Val: Elem, Ptr: DestElementAddr);
3666 break;
3667 }
3668 case EvalKind::Complex: {
3669 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
3670 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3671 Value *SrcReal = Builder.CreateLoad(
3672 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
3673 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
3674 Ty: RI.ElementType, Ptr: SrcElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3675 Value *SrcImg = Builder.CreateLoad(
3676 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
3677
3678 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
3679 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 0, Name: ".realp");
3680 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
3681 Ty: RI.ElementType, Ptr: DestElementAddr, Idx0: 0, Idx1: 1, Name: ".imagp");
3682 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
3683 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
3684 break;
3685 }
3686 case EvalKind::Aggregate: {
3687 Value *SizeVal = Builder.getInt64(
3688 C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
3689 Builder.CreateMemCpy(
3690 Dst: DestElementAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3691 Src: SrcElementAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
3692 Size: SizeVal, isVolatile: false);
3693 break;
3694 }
3695 };
3696 }
3697
3698 // Step 3.1: Modify reference in dest Reduce list as needed.
3699 // Modifying the reference in Reduce list to point to the newly
3700 // created element. The element is live in the current function
3701 // scope and that of functions it invokes (i.e., reduce_function).
3702 // RemoteReduceData[i] = (void*)&RemoteElem
3703 if (UpdateDestListPtr) {
3704 Value *CastDestAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
3705 V: DestElementAddr, DestTy: Builder.getPtrTy(),
3706 Name: DestElementAddr->getName() + ".ascast");
3707 Builder.CreateStore(Val: CastDestAddr, Ptr: DestElementPtrAddr);
3708 }
3709 }
3710
3711 return Error::success();
3712}
3713
3714Expected<Function *> OpenMPIRBuilder::emitInterWarpCopyFunction(
3715 const LocationDescription &Loc, ArrayRef<ReductionInfo> ReductionInfos,
3716 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3717 IRBuilder<>::InsertPointGuard IPG(Builder);
3718 LLVMContext &Ctx = M.getContext();
3719 FunctionType *FuncTy = FunctionType::get(
3720 Result: Builder.getVoidTy(), Params: {Builder.getPtrTy(), Builder.getInt32Ty()},
3721 /* IsVarArg */ isVarArg: false);
3722 Function *WcFunc =
3723 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3724 N: "_omp_reduction_inter_warp_copy_func", M: &M);
3725 WcFunc->setCallingConv(Config.getRuntimeCC());
3726 WcFunc->setAttributes(FuncAttrs);
3727 WcFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3728 WcFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3729 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: WcFunc);
3730 Builder.SetInsertPoint(EntryBB);
3731 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3732
3733 // ReduceList: thread local Reduce list.
3734 // At the stage of the computation when this function is called, partially
3735 // aggregated values reside in the first lane of every active warp.
3736 Argument *ReduceListArg = WcFunc->getArg(i: 0);
3737 // NumWarps: number of warps active in the parallel region. This could
3738 // be smaller than 32 (max warps in a CTA) for partial block reduction.
3739 Argument *NumWarpsArg = WcFunc->getArg(i: 1);
3740
3741 // This array is used as a medium to transfer, one reduce element at a time,
3742 // the data from the first lane of every warp to lanes in the first warp
3743 // in order to perform the final step of a reduction in a parallel region
3744 // (reduction across warps). The array is placed in NVPTX __shared__ memory
3745 // for reduced latency, as well as to have a distinct copy for concurrently
3746 // executing target regions. The array is declared with common linkage so
3747 // as to be shared across compilation units.
3748 StringRef TransferMediumName =
3749 "__openmp_nvptx_data_transfer_temporary_storage";
3750 GlobalVariable *TransferMedium = M.getGlobalVariable(Name: TransferMediumName);
3751 unsigned WarpSize = Config.getGridValue().GV_Warp_Size;
3752 ArrayType *ArrayTy = ArrayType::get(ElementType: Builder.getInt32Ty(), NumElements: WarpSize);
3753 if (!TransferMedium) {
3754 TransferMedium = new GlobalVariable(
3755 M, ArrayTy, /*isConstant=*/false, GlobalVariable::WeakAnyLinkage,
3756 UndefValue::get(T: ArrayTy), TransferMediumName,
3757 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
3758 /*AddressSpace=*/3);
3759 }
3760
3761 // Get the CUDA thread id of the current OpenMP thread on the GPU.
3762 Value *GPUThreadID = getGPUThreadID();
3763 // nvptx_lane_id = nvptx_id % warpsize
3764 Value *LaneID = getNVPTXLaneID();
3765 // nvptx_warp_id = nvptx_id / warpsize
3766 Value *WarpID = getNVPTXWarpID();
3767
3768 InsertPointTy AllocaIP = Builder.GetInsertBlock()->getFirstInsertionPt();
3769 Type *Arg0Type = ReduceListArg->getType();
3770 Type *Arg1Type = NumWarpsArg->getType();
3771 Builder.restoreIP(IP: AllocaIP);
3772 AllocaInst *ReduceListAlloca = Builder.CreateAlloca(
3773 Ty: Arg0Type, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
3774 AllocaInst *NumWarpsAlloca =
3775 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: NumWarpsArg->getName() + ".addr");
3776 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3777 V: ReduceListAlloca, DestTy: Arg0Type, Name: ReduceListAlloca->getName() + ".ascast");
3778 Value *NumWarpsAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
3779 V: NumWarpsAlloca, DestTy: Builder.getPtrTy(AddrSpace: 0),
3780 Name: NumWarpsAlloca->getName() + ".ascast");
3781 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
3782 Builder.CreateStore(Val: NumWarpsArg, Ptr: NumWarpsAddrCast);
3783 AllocaIP = getInsertPointAfterInstr(I: NumWarpsAlloca);
3784 InsertPointTy CodeGenIP =
3785 getInsertPointAfterInstr(I: &Builder.GetInsertBlock()->back());
3786 Builder.restoreIP(IP: CodeGenIP);
3787
3788 Value *ReduceList =
3789 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListAddrCast);
3790
3791 for (auto En : enumerate(First&: ReductionInfos)) {
3792 //
3793 // Warp master copies reduce element to transfer medium in __shared__
3794 // memory.
3795 //
3796 const ReductionInfo &RI = En.value();
3797 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
3798 unsigned RealTySize = M.getDataLayout().getTypeAllocSize(
3799 Ty: IsByRefElem ? RI.ByRefElementType : RI.ElementType);
3800 for (unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
3801 Type *CType = Builder.getIntNTy(N: TySize * 8);
3802
3803 unsigned NumIters = RealTySize / TySize;
3804 if (NumIters == 0)
3805 continue;
3806 Value *Cnt = nullptr;
3807 Value *CntAddr = nullptr;
3808 BasicBlock *PrecondBB = nullptr;
3809 BasicBlock *ExitBB = nullptr;
3810 if (NumIters > 1) {
3811 CodeGenIP = Builder.saveIP();
3812 Builder.restoreIP(IP: AllocaIP);
3813 CntAddr =
3814 Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr, Name: ".cnt.addr");
3815
3816 CntAddr = Builder.CreateAddrSpaceCast(V: CntAddr, DestTy: Builder.getPtrTy(),
3817 Name: CntAddr->getName() + ".ascast");
3818 Builder.restoreIP(IP: CodeGenIP);
3819 Builder.CreateStore(Val: Constant::getNullValue(Ty: Builder.getInt32Ty()),
3820 Ptr: CntAddr,
3821 /*Volatile=*/isVolatile: false);
3822 PrecondBB = BasicBlock::Create(Context&: Ctx, Name: "precond");
3823 ExitBB = BasicBlock::Create(Context&: Ctx, Name: "exit");
3824 BasicBlock *BodyBB = BasicBlock::Create(Context&: Ctx, Name: "body");
3825 emitBlock(BB: PrecondBB, CurFn: Builder.GetInsertBlock()->getParent());
3826 Cnt = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: CntAddr,
3827 /*Volatile=*/isVolatile: false);
3828 Value *Cmp = Builder.CreateICmpULT(
3829 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), V: NumIters));
3830 Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitBB);
3831 emitBlock(BB: BodyBB, CurFn: Builder.GetInsertBlock()->getParent());
3832 }
3833
3834 // kmpc_barrier.
3835 InsertPointOrErrorTy BarrierIP1 =
3836 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3837 Kind: omp::Directive::OMPD_unknown,
3838 /* ForceSimpleCall */ false,
3839 /* CheckCancelFlag */ true);
3840 if (!BarrierIP1)
3841 return BarrierIP1.takeError();
3842 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3843 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3844 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3845
3846 // if (lane_id == 0)
3847 Value *IsWarpMaster = Builder.CreateIsNull(Arg: LaneID, Name: "warp_master");
3848 Builder.CreateCondBr(Cond: IsWarpMaster, True: ThenBB, False: ElseBB);
3849 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3850
3851 // Reduce element = LocalReduceList[i]
3852 auto *RedListArrayTy =
3853 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
3854 Type *IndexTy = Builder.getIndexTy(
3855 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
3856 Value *ElemPtrPtr =
3857 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3858 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3859 ConstantInt::get(Ty: IndexTy, V: En.index())});
3860 // elemptr = ((CopyType*)(elemptrptr)) + I
3861 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
3862
3863 if (IsByRefElem && RI.DataPtrPtrGen) {
3864 InsertPointOrErrorTy GenRes =
3865 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
3866
3867 if (!GenRes)
3868 return GenRes.takeError();
3869
3870 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
3871 }
3872
3873 if (NumIters > 1)
3874 ElemPtr = Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: ElemPtr, IdxList: Cnt);
3875
3876 // Get pointer to location in transfer medium.
3877 // MediumPtr = &medium[warp_id]
3878 Value *MediumPtr = Builder.CreateInBoundsGEP(
3879 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), WarpID});
3880 // elem = *elemptr
3881 //*MediumPtr = elem
3882 Value *Elem = Builder.CreateLoad(Ty: CType, Ptr: ElemPtr);
3883 // Store the source element value to the dest element address.
3884 Builder.CreateStore(Val: Elem, Ptr: MediumPtr,
3885 /*IsVolatile*/ isVolatile: true);
3886 Builder.CreateBr(Dest: MergeBB);
3887
3888 // else
3889 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3890 Builder.CreateBr(Dest: MergeBB);
3891
3892 // endif
3893 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3894 InsertPointOrErrorTy BarrierIP2 =
3895 createBarrier(Loc: LocationDescription(Builder.saveIP(), DebugLoc()),
3896 Kind: omp::Directive::OMPD_unknown,
3897 /* ForceSimpleCall */ false,
3898 /* CheckCancelFlag */ true);
3899 if (!BarrierIP2)
3900 return BarrierIP2.takeError();
3901
3902 // Warp 0 copies reduce element from transfer medium
3903 BasicBlock *W0ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
3904 BasicBlock *W0ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
3905 BasicBlock *W0MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
3906
3907 Value *NumWarpsVal =
3908 Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: NumWarpsAddrCast);
3909 // Up to 32 threads in warp 0 are active.
3910 Value *IsActiveThread =
3911 Builder.CreateICmpULT(LHS: GPUThreadID, RHS: NumWarpsVal, Name: "is_active_thread");
3912 Builder.CreateCondBr(Cond: IsActiveThread, True: W0ThenBB, False: W0ElseBB);
3913
3914 emitBlock(BB: W0ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
3915
3916 // SecMediumPtr = &medium[tid]
3917 // SrcMediumVal = *SrcMediumPtr
3918 Value *SrcMediumPtrVal = Builder.CreateInBoundsGEP(
3919 Ty: ArrayTy, Ptr: TransferMedium, IdxList: {Builder.getInt64(C: 0), GPUThreadID});
3920 // TargetElemPtr = (CopyType*)(SrcDataAddr[i]) + I
3921 Value *TargetElemPtrPtr =
3922 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
3923 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
3924 ConstantInt::get(Ty: IndexTy, V: En.index())});
3925 Value *TargetElemPtrVal =
3926 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtrPtr);
3927 Value *TargetElemPtr = TargetElemPtrVal;
3928
3929 if (IsByRefElem && RI.DataPtrPtrGen) {
3930 InsertPointOrErrorTy GenRes =
3931 RI.DataPtrPtrGen(Builder.saveIP(), TargetElemPtr, TargetElemPtr);
3932
3933 if (!GenRes)
3934 return GenRes.takeError();
3935
3936 TargetElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: TargetElemPtr);
3937 }
3938
3939 if (NumIters > 1)
3940 TargetElemPtr =
3941 Builder.CreateGEP(Ty: Builder.getInt32Ty(), Ptr: TargetElemPtr, IdxList: Cnt);
3942
3943 // *TargetElemPtr = SrcMediumVal;
3944 Value *SrcMediumValue =
3945 Builder.CreateLoad(Ty: CType, Ptr: SrcMediumPtrVal, /*IsVolatile*/ isVolatile: true);
3946 Builder.CreateStore(Val: SrcMediumValue, Ptr: TargetElemPtr);
3947 Builder.CreateBr(Dest: W0MergeBB);
3948
3949 emitBlock(BB: W0ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
3950 Builder.CreateBr(Dest: W0MergeBB);
3951
3952 emitBlock(BB: W0MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
3953
3954 if (NumIters > 1) {
3955 Cnt = Builder.CreateNSWAdd(
3956 LHS: Cnt, RHS: ConstantInt::get(Ty: Builder.getInt32Ty(), /*V=*/1));
3957 Builder.CreateStore(Val: Cnt, Ptr: CntAddr, /*Volatile=*/isVolatile: false);
3958
3959 auto *CurFn = Builder.GetInsertBlock()->getParent();
3960 emitBranch(Target: PrecondBB);
3961 emitBlock(BB: ExitBB, CurFn);
3962 }
3963 RealTySize %= TySize;
3964 }
3965 }
3966
3967 Builder.CreateRetVoid();
3968
3969 return WcFunc;
3970}
3971
3972Expected<Function *> OpenMPIRBuilder::emitShuffleAndReduceFunction(
3973 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
3974 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
3975 LLVMContext &Ctx = M.getContext();
3976 IRBuilder<>::InsertPointGuard IPG(Builder);
3977 FunctionType *FuncTy =
3978 FunctionType::get(Result: Builder.getVoidTy(),
3979 Params: {Builder.getPtrTy(), Builder.getInt16Ty(),
3980 Builder.getInt16Ty(), Builder.getInt16Ty()},
3981 /* IsVarArg */ isVarArg: false);
3982 Function *SarFunc =
3983 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
3984 N: "_omp_reduction_shuffle_and_reduce_func", M: &M);
3985 SarFunc->setCallingConv(Config.getRuntimeCC());
3986 SarFunc->setAttributes(FuncAttrs);
3987 SarFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
3988 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
3989 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
3990 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
3991 SarFunc->addParamAttr(ArgNo: 1, Kind: Attribute::SExt);
3992 SarFunc->addParamAttr(ArgNo: 2, Kind: Attribute::SExt);
3993 SarFunc->addParamAttr(ArgNo: 3, Kind: Attribute::SExt);
3994 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SarFunc);
3995 Builder.SetInsertPoint(EntryBB);
3996 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
3997
3998 // Thread local Reduce list used to host the values of data to be reduced.
3999 Argument *ReduceListArg = SarFunc->getArg(i: 0);
4000 // Current lane id; could be logical.
4001 Argument *LaneIDArg = SarFunc->getArg(i: 1);
4002 // Offset of the remote source lane relative to the current lane.
4003 Argument *RemoteLaneOffsetArg = SarFunc->getArg(i: 2);
4004 // Algorithm version. This is expected to be known at compile time.
4005 Argument *AlgoVerArg = SarFunc->getArg(i: 3);
4006
4007 Type *ReduceListArgType = ReduceListArg->getType();
4008 Type *LaneIDArgType = LaneIDArg->getType();
4009 Type *LaneIDArgPtrType = Builder.getPtrTy(AddrSpace: 0);
4010 Value *ReduceListAlloca = Builder.CreateAlloca(
4011 Ty: ReduceListArgType, ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4012 Value *LaneIdAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
4013 Name: LaneIDArg->getName() + ".addr");
4014 Value *RemoteLaneOffsetAlloca = Builder.CreateAlloca(
4015 Ty: LaneIDArgType, ArraySize: nullptr, Name: RemoteLaneOffsetArg->getName() + ".addr");
4016 Value *AlgoVerAlloca = Builder.CreateAlloca(Ty: LaneIDArgType, ArraySize: nullptr,
4017 Name: AlgoVerArg->getName() + ".addr");
4018 ArrayType *RedListArrayTy =
4019 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4020
4021 // Create a local thread-private variable to host the Reduce list
4022 // from a remote lane.
4023 Instruction *RemoteReductionListAlloca = Builder.CreateAlloca(
4024 Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.remote_reduce_list");
4025
4026 Value *ReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4027 V: ReduceListAlloca, DestTy: ReduceListArgType,
4028 Name: ReduceListAlloca->getName() + ".ascast");
4029 Value *LaneIdAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4030 V: LaneIdAlloca, DestTy: LaneIDArgPtrType, Name: LaneIdAlloca->getName() + ".ascast");
4031 Value *RemoteLaneOffsetAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4032 V: RemoteLaneOffsetAlloca, DestTy: LaneIDArgPtrType,
4033 Name: RemoteLaneOffsetAlloca->getName() + ".ascast");
4034 Value *AlgoVerAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4035 V: AlgoVerAlloca, DestTy: LaneIDArgPtrType, Name: AlgoVerAlloca->getName() + ".ascast");
4036 Value *RemoteListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4037 V: RemoteReductionListAlloca, DestTy: Builder.getPtrTy(),
4038 Name: RemoteReductionListAlloca->getName() + ".ascast");
4039
4040 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListAddrCast);
4041 Builder.CreateStore(Val: LaneIDArg, Ptr: LaneIdAddrCast);
4042 Builder.CreateStore(Val: RemoteLaneOffsetArg, Ptr: RemoteLaneOffsetAddrCast);
4043 Builder.CreateStore(Val: AlgoVerArg, Ptr: AlgoVerAddrCast);
4044
4045 Value *ReduceList = Builder.CreateLoad(Ty: ReduceListArgType, Ptr: ReduceListAddrCast);
4046 Value *LaneId = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: LaneIdAddrCast);
4047 Value *RemoteLaneOffset =
4048 Builder.CreateLoad(Ty: LaneIDArgType, Ptr: RemoteLaneOffsetAddrCast);
4049 Value *AlgoVer = Builder.CreateLoad(Ty: LaneIDArgType, Ptr: AlgoVerAddrCast);
4050
4051 InsertPointTy AllocaIP = getInsertPointAfterInstr(I: RemoteReductionListAlloca);
4052
4053 // This loop iterates through the list of reduce elements and copies,
4054 // element by element, from a remote lane in the warp to RemoteReduceList,
4055 // hosted on the thread's stack.
4056 Error EmitRedLsCpRes = emitReductionListCopy(
4057 AllocaIP, Action: CopyAction::RemoteLaneToThread, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4058 SrcBase: ReduceList, DestBase: RemoteListAddrCast, IsByRef,
4059 CopyOptions: {.RemoteLaneOffset: RemoteLaneOffset, .ScratchpadIndex: nullptr, .ScratchpadWidth: nullptr});
4060
4061 if (EmitRedLsCpRes)
4062 return EmitRedLsCpRes;
4063
4064 // The actions to be performed on the Remote Reduce list is dependent
4065 // on the algorithm version.
4066 //
4067 // if (AlgoVer==0) || (AlgoVer==1 && (LaneId < Offset)) || (AlgoVer==2 &&
4068 // LaneId % 2 == 0 && Offset > 0):
4069 // do the reduction value aggregation
4070 //
4071 // The thread local variable Reduce list is mutated in place to host the
4072 // reduced data, which is the aggregated value produced from local and
4073 // remote lanes.
4074 //
4075 // Note that AlgoVer is expected to be a constant integer known at compile
4076 // time.
4077 // When AlgoVer==0, the first conjunction evaluates to true, making
4078 // the entire predicate true during compile time.
4079 // When AlgoVer==1, the second conjunction has only the second part to be
4080 // evaluated during runtime. Other conjunctions evaluates to false
4081 // during compile time.
4082 // When AlgoVer==2, the third conjunction has only the second part to be
4083 // evaluated during runtime. Other conjunctions evaluates to false
4084 // during compile time.
4085 Value *CondAlgo0 = Builder.CreateIsNull(Arg: AlgoVer);
4086 Value *Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4087 Value *LaneComp = Builder.CreateICmpULT(LHS: LaneId, RHS: RemoteLaneOffset);
4088 Value *CondAlgo1 = Builder.CreateAnd(LHS: Algo1, RHS: LaneComp);
4089 Value *Algo2 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 2));
4090 Value *LaneIdAnd1 = Builder.CreateAnd(LHS: LaneId, RHS: Builder.getInt16(C: 1));
4091 Value *LaneIdComp = Builder.CreateIsNull(Arg: LaneIdAnd1);
4092 Value *Algo2AndLaneIdComp = Builder.CreateAnd(LHS: Algo2, RHS: LaneIdComp);
4093 Value *RemoteOffsetComp =
4094 Builder.CreateICmpSGT(LHS: RemoteLaneOffset, RHS: Builder.getInt16(C: 0));
4095 Value *CondAlgo2 = Builder.CreateAnd(LHS: Algo2AndLaneIdComp, RHS: RemoteOffsetComp);
4096 Value *CA0OrCA1 = Builder.CreateOr(LHS: CondAlgo0, RHS: CondAlgo1);
4097 Value *CondReduce = Builder.CreateOr(LHS: CA0OrCA1, RHS: CondAlgo2);
4098
4099 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4100 BasicBlock *ElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4101 BasicBlock *MergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4102
4103 Builder.CreateCondBr(Cond: CondReduce, True: ThenBB, False: ElseBB);
4104 emitBlock(BB: ThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4105 Value *LocalReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4106 V: ReduceList, DestTy: Builder.getPtrTy());
4107 Value *RemoteReduceListPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4108 V: RemoteListAddrCast, DestTy: Builder.getPtrTy());
4109 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListPtr, RemoteReduceListPtr})
4110 ->addFnAttr(Kind: Attribute::NoUnwind);
4111 Builder.CreateBr(Dest: MergeBB);
4112
4113 emitBlock(BB: ElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4114 Builder.CreateBr(Dest: MergeBB);
4115
4116 emitBlock(BB: MergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4117
4118 // if (AlgoVer==1 && (LaneId >= Offset)) copy Remote Reduce list to local
4119 // Reduce list.
4120 Algo1 = Builder.CreateICmpEQ(LHS: AlgoVer, RHS: Builder.getInt16(C: 1));
4121 Value *LaneIdGtOffset = Builder.CreateICmpUGE(LHS: LaneId, RHS: RemoteLaneOffset);
4122 Value *CondCopy = Builder.CreateAnd(LHS: Algo1, RHS: LaneIdGtOffset);
4123
4124 BasicBlock *CpyThenBB = BasicBlock::Create(Context&: Ctx, Name: "then");
4125 BasicBlock *CpyElseBB = BasicBlock::Create(Context&: Ctx, Name: "else");
4126 BasicBlock *CpyMergeBB = BasicBlock::Create(Context&: Ctx, Name: "ifcont");
4127 Builder.CreateCondBr(Cond: CondCopy, True: CpyThenBB, False: CpyElseBB);
4128
4129 emitBlock(BB: CpyThenBB, CurFn: Builder.GetInsertBlock()->getParent());
4130
4131 EmitRedLsCpRes = emitReductionListCopy(
4132 AllocaIP, Action: CopyAction::ThreadCopy, ReductionArrayTy: RedListArrayTy, ReductionInfos,
4133 SrcBase: RemoteListAddrCast, DestBase: ReduceList, IsByRef);
4134
4135 if (EmitRedLsCpRes)
4136 return EmitRedLsCpRes;
4137
4138 Builder.CreateBr(Dest: CpyMergeBB);
4139
4140 emitBlock(BB: CpyElseBB, CurFn: Builder.GetInsertBlock()->getParent());
4141 Builder.CreateBr(Dest: CpyMergeBB);
4142
4143 emitBlock(BB: CpyMergeBB, CurFn: Builder.GetInsertBlock()->getParent());
4144
4145 Builder.CreateRetVoid();
4146
4147 return SarFunc;
4148}
4149
4150OpenMPIRBuilder::InsertPointOrErrorTy
4151OpenMPIRBuilder::generateReductionDescriptor(
4152 Value *DescriptorAddr, Value *DataPtr, Value *SrcDescriptorAddr,
4153 Type *DescriptorType,
4154 function_ref<InsertPointOrErrorTy(InsertPointTy, Value *, Value *&)>
4155 DataPtrPtrGen) {
4156
4157 // Copy the source descriptor to preserve all metadata (rank, extents,
4158 // strides, etc.)
4159 Value *DescriptorSize =
4160 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: DescriptorType));
4161 Builder.CreateMemCpy(
4162 Dst: DescriptorAddr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4163 Src: SrcDescriptorAddr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: DescriptorType),
4164 Size: DescriptorSize);
4165
4166 // Update the base pointer field to point to the local shuffled data
4167 Value *DataPtrField;
4168 InsertPointOrErrorTy GenResult =
4169 DataPtrPtrGen(Builder.saveIP(), DescriptorAddr, DataPtrField);
4170
4171 if (!GenResult)
4172 return GenResult.takeError();
4173
4174 Builder.CreateStore(Val: Builder.CreatePointerBitCastOrAddrSpaceCast(
4175 V: DataPtr, DestTy: Builder.getPtrTy(), Name: ".ascast"),
4176 Ptr: DataPtrField);
4177
4178 return Builder.saveIP();
4179}
4180
4181Expected<Value *> OpenMPIRBuilder::createReductionDescriptorCopy(
4182 InsertPointTy AllocaIP, const ReductionInfo &RI, Value *DataPtr,
4183 Value *SrcDescriptorAddr, Type *DescriptorPtrTy, const Twine &Name) {
4184 InsertPointTy OldIP = Builder.saveIP();
4185 Builder.restoreIP(IP: AllocaIP);
4186
4187 AllocaInst *DescriptorAlloca =
4188 Builder.CreateAlloca(Ty: RI.ByRefAllocatedType, ArraySize: nullptr, Name);
4189 DescriptorAlloca->setAlignment(
4190 M.getDataLayout().getPrefTypeAlign(Ty: RI.ByRefAllocatedType));
4191 Value *DescriptorAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4192 V: DescriptorAlloca, DestTy: DescriptorPtrTy,
4193 Name: DescriptorAlloca->getName() + ".ascast");
4194
4195 Builder.restoreIP(IP: OldIP);
4196
4197 InsertPointOrErrorTy GenResult =
4198 generateReductionDescriptor(DescriptorAddr, DataPtr, SrcDescriptorAddr,
4199 DescriptorType: RI.ByRefAllocatedType, DataPtrPtrGen: RI.DataPtrPtrGen);
4200 if (!GenResult)
4201 return GenResult.takeError();
4202
4203 return DescriptorAddr;
4204}
4205
4206Expected<Function *> OpenMPIRBuilder::emitListToGlobalCopyFunction(
4207 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4208 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4209 IRBuilder<>::InsertPointGuard IPG(Builder);
4210 LLVMContext &Ctx = M.getContext();
4211 FunctionType *FuncTy = FunctionType::get(
4212 Result: Builder.getVoidTy(),
4213 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4214 /* IsVarArg */ isVarArg: false);
4215 Function *LtGCFunc =
4216 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4217 N: "_omp_reduction_list_to_global_copy_func", M: &M);
4218 LtGCFunc->setAttributes(FuncAttrs);
4219 LtGCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4220 LtGCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4221 LtGCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4222
4223 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGCFunc);
4224 Builder.SetInsertPoint(EntryBlock);
4225 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4226
4227 // Buffer: global reduction buffer.
4228 Argument *BufferArg = LtGCFunc->getArg(i: 0);
4229 // Idx: index of the buffer.
4230 Argument *IdxArg = LtGCFunc->getArg(i: 1);
4231 // ReduceList: thread local Reduce list.
4232 Argument *ReduceListArg = LtGCFunc->getArg(i: 2);
4233
4234 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4235 Name: BufferArg->getName() + ".addr");
4236 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4237 Name: IdxArg->getName() + ".addr");
4238 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4239 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4240 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4241 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4242 Name: BufferArgAlloca->getName() + ".ascast");
4243 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4244 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4245 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4246 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4247 Name: ReduceListArgAlloca->getName() + ".ascast");
4248
4249 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4250 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4251 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4252
4253 Value *LocalReduceList =
4254 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4255 Value *BufferArgVal =
4256 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4257 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4258 Type *IndexTy = Builder.getIndexTy(
4259 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4260 for (auto En : enumerate(First&: ReductionInfos)) {
4261 const ReductionInfo &RI = En.value();
4262 auto *RedListArrayTy =
4263 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4264 // Reduce element = LocalReduceList[i]
4265 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4266 Ty: RedListArrayTy, Ptr: LocalReduceList,
4267 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4268 // elemptr = ((CopyType*)(elemptrptr)) + I
4269 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4270
4271 // Global = Buffer.VD[Idx];
4272 Value *BufferVD =
4273 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferArgVal, IdxList: Idxs);
4274 Value *GlobVal = Builder.CreateConstInBoundsGEP2_32(
4275 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4276
4277 switch (RI.EvaluationKind) {
4278 case EvalKind::Scalar: {
4279 Value *TargetElement;
4280
4281 if (IsByRef.empty() || !IsByRef[En.index()]) {
4282 TargetElement = Builder.CreateLoad(Ty: RI.ElementType, Ptr: ElemPtr);
4283 } else {
4284 if (RI.DataPtrPtrGen) {
4285 InsertPointOrErrorTy GenResult =
4286 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4287
4288 if (!GenResult)
4289 return GenResult.takeError();
4290
4291 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4292 }
4293 TargetElement = Builder.CreateLoad(Ty: RI.ByRefElementType, Ptr: ElemPtr);
4294 }
4295
4296 Builder.CreateStore(Val: TargetElement, Ptr: GlobVal);
4297 break;
4298 }
4299 case EvalKind::Complex: {
4300 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4301 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4302 Value *SrcReal = Builder.CreateLoad(
4303 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4304 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4305 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4306 Value *SrcImg = Builder.CreateLoad(
4307 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4308
4309 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4310 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 0, Name: ".realp");
4311 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4312 Ty: RI.ElementType, Ptr: GlobVal, Idx0: 0, Idx1: 1, Name: ".imagp");
4313 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4314 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4315 break;
4316 }
4317 case EvalKind::Aggregate: {
4318 Value *SizeVal =
4319 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4320 Builder.CreateMemCpy(
4321 Dst: GlobVal, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Src: ElemPtr,
4322 SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType), Size: SizeVal, isVolatile: false);
4323 break;
4324 }
4325 }
4326 }
4327
4328 Builder.CreateRetVoid();
4329 return LtGCFunc;
4330}
4331
4332Expected<Function *> OpenMPIRBuilder::emitListToGlobalReduceFunction(
4333 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4334 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4335 IRBuilder<>::InsertPointGuard IPG(Builder);
4336 LLVMContext &Ctx = M.getContext();
4337 FunctionType *FuncTy = FunctionType::get(
4338 Result: Builder.getVoidTy(),
4339 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4340 /* IsVarArg */ isVarArg: false);
4341 Function *LtGRFunc =
4342 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4343 N: "_omp_reduction_list_to_global_reduce_func", M: &M);
4344 LtGRFunc->setAttributes(FuncAttrs);
4345 LtGRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4346 LtGRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4347 LtGRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4348
4349 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: LtGRFunc);
4350 Builder.SetInsertPoint(EntryBlock);
4351 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4352
4353 // Buffer: global reduction buffer.
4354 Argument *BufferArg = LtGRFunc->getArg(i: 0);
4355 // Idx: index of the buffer.
4356 Argument *IdxArg = LtGRFunc->getArg(i: 1);
4357 // ReduceList: thread local Reduce list.
4358 Argument *ReduceListArg = LtGRFunc->getArg(i: 2);
4359
4360 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4361 Name: BufferArg->getName() + ".addr");
4362 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4363 Name: IdxArg->getName() + ".addr");
4364 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4365 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4366 auto *RedListArrayTy =
4367 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4368
4369 // 1. Build a list of reduction variables.
4370 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4371 Value *LocalReduceList =
4372 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4373
4374 InsertPointTy AllocaIP(EntryBlock->begin());
4375
4376 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4377 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4378 Name: BufferArgAlloca->getName() + ".ascast");
4379 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4380 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4381 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4382 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4383 Name: ReduceListArgAlloca->getName() + ".ascast");
4384 Value *LocalReduceListAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4385 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4386 Name: LocalReduceList->getName() + ".ascast");
4387
4388 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4389 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4390 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4391
4392 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4393 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4394 Type *IndexTy = Builder.getIndexTy(
4395 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4396 for (auto En : enumerate(First&: ReductionInfos)) {
4397 const ReductionInfo &RI = En.value();
4398
4399 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4400 Ty: RedListArrayTy, Ptr: LocalReduceListAddrCast,
4401 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4402 Value *BufferVD =
4403 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4404 // Global = Buffer.VD[Idx];
4405 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4406 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4407
4408 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4409 // Get source descriptor from the reduce list argument
4410 Value *ReduceList =
4411 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4412 Value *SrcElementPtrPtr =
4413 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceList,
4414 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4415 ConstantInt::get(Ty: IndexTy, V: En.index())});
4416 Value *SrcDescriptorAddr =
4417 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4418
4419 // Copy descriptor from source and update base_ptr to global buffer data
4420 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4421 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4422 if (!ByRefAlloc)
4423 return ByRefAlloc.takeError();
4424
4425 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4426 } else {
4427 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4428 }
4429 }
4430
4431 // Call reduce_function(GlobalReduceList, ReduceList)
4432 Value *ReduceList =
4433 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4434 createRuntimeFunctionCall(Callee: ReduceFn, Args: {LocalReduceListAddrCast, ReduceList})
4435 ->addFnAttr(Kind: Attribute::NoUnwind);
4436 Builder.CreateRetVoid();
4437 return LtGRFunc;
4438}
4439
4440Expected<Function *> OpenMPIRBuilder::emitGlobalToListCopyFunction(
4441 ArrayRef<ReductionInfo> ReductionInfos, Type *ReductionsBufferTy,
4442 AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4443 IRBuilder<>::InsertPointGuard IPG(Builder);
4444 LLVMContext &Ctx = M.getContext();
4445 FunctionType *FuncTy = FunctionType::get(
4446 Result: Builder.getVoidTy(),
4447 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4448 /* IsVarArg */ isVarArg: false);
4449 Function *GtLCFunc =
4450 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4451 N: "_omp_reduction_global_to_list_copy_func", M: &M);
4452 GtLCFunc->setAttributes(FuncAttrs);
4453 GtLCFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4454 GtLCFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4455 GtLCFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4456
4457 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLCFunc);
4458 Builder.SetInsertPoint(EntryBlock);
4459 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4460
4461 // Buffer: global reduction buffer.
4462 Argument *BufferArg = GtLCFunc->getArg(i: 0);
4463 // Idx: index of the buffer.
4464 Argument *IdxArg = GtLCFunc->getArg(i: 1);
4465 // ReduceList: thread local Reduce list.
4466 Argument *ReduceListArg = GtLCFunc->getArg(i: 2);
4467
4468 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4469 Name: BufferArg->getName() + ".addr");
4470 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4471 Name: IdxArg->getName() + ".addr");
4472 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4473 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4474 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4475 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4476 Name: BufferArgAlloca->getName() + ".ascast");
4477 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4478 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4479 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4480 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4481 Name: ReduceListArgAlloca->getName() + ".ascast");
4482 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4483 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4484 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4485
4486 Value *LocalReduceList =
4487 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4488 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4489 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4490 Type *IndexTy = Builder.getIndexTy(
4491 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4492 for (auto En : enumerate(First&: ReductionInfos)) {
4493 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
4494 auto *RedListArrayTy =
4495 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4496 // Reduce element = LocalReduceList[i]
4497 Value *ElemPtrPtr = Builder.CreateInBoundsGEP(
4498 Ty: RedListArrayTy, Ptr: LocalReduceList,
4499 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4500 // elemptr = ((CopyType*)(elemptrptr)) + I
4501 Value *ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtrPtr);
4502 // Global = Buffer.VD[Idx];
4503 Value *BufferVD =
4504 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4505 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4506 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4507
4508 switch (RI.EvaluationKind) {
4509 case EvalKind::Scalar: {
4510 Type *ElemType = RI.ElementType;
4511
4512 if (!IsByRef.empty() && IsByRef[En.index()]) {
4513 ElemType = RI.ByRefElementType;
4514 if (RI.DataPtrPtrGen) {
4515 InsertPointOrErrorTy GenResult =
4516 RI.DataPtrPtrGen(Builder.saveIP(), ElemPtr, ElemPtr);
4517
4518 if (!GenResult)
4519 return GenResult.takeError();
4520
4521 ElemPtr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ElemPtr);
4522 }
4523 }
4524
4525 Value *TargetElement = Builder.CreateLoad(Ty: ElemType, Ptr: GlobValPtr);
4526 Builder.CreateStore(Val: TargetElement, Ptr: ElemPtr);
4527 break;
4528 }
4529 case EvalKind::Complex: {
4530 Value *SrcRealPtr = Builder.CreateConstInBoundsGEP2_32(
4531 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4532 Value *SrcReal = Builder.CreateLoad(
4533 Ty: RI.ElementType->getStructElementType(N: 0), Ptr: SrcRealPtr, Name: ".real");
4534 Value *SrcImgPtr = Builder.CreateConstInBoundsGEP2_32(
4535 Ty: RI.ElementType, Ptr: GlobValPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4536 Value *SrcImg = Builder.CreateLoad(
4537 Ty: RI.ElementType->getStructElementType(N: 1), Ptr: SrcImgPtr, Name: ".imag");
4538
4539 Value *DestRealPtr = Builder.CreateConstInBoundsGEP2_32(
4540 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 0, Name: ".realp");
4541 Value *DestImgPtr = Builder.CreateConstInBoundsGEP2_32(
4542 Ty: RI.ElementType, Ptr: ElemPtr, Idx0: 0, Idx1: 1, Name: ".imagp");
4543 Builder.CreateStore(Val: SrcReal, Ptr: DestRealPtr);
4544 Builder.CreateStore(Val: SrcImg, Ptr: DestImgPtr);
4545 break;
4546 }
4547 case EvalKind::Aggregate: {
4548 Value *SizeVal =
4549 Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: RI.ElementType));
4550 Builder.CreateMemCpy(
4551 Dst: ElemPtr, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4552 Src: GlobValPtr, SrcAlign: M.getDataLayout().getPrefTypeAlign(Ty: RI.ElementType),
4553 Size: SizeVal, isVolatile: false);
4554 break;
4555 }
4556 }
4557 }
4558
4559 Builder.CreateRetVoid();
4560 return GtLCFunc;
4561}
4562
4563Expected<Function *> OpenMPIRBuilder::emitGlobalToListReduceFunction(
4564 ArrayRef<ReductionInfo> ReductionInfos, Function *ReduceFn,
4565 Type *ReductionsBufferTy, AttributeList FuncAttrs, ArrayRef<bool> IsByRef) {
4566 IRBuilder<>::InsertPointGuard IPG(Builder);
4567 LLVMContext &Ctx = M.getContext();
4568 auto *FuncTy = FunctionType::get(
4569 Result: Builder.getVoidTy(),
4570 Params: {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
4571 /* IsVarArg */ isVarArg: false);
4572 Function *GtLRFunc =
4573 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
4574 N: "_omp_reduction_global_to_list_reduce_func", M: &M);
4575 GtLRFunc->setAttributes(FuncAttrs);
4576 GtLRFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4577 GtLRFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4578 GtLRFunc->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
4579
4580 BasicBlock *EntryBlock = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: GtLRFunc);
4581 Builder.SetInsertPoint(EntryBlock);
4582 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4583
4584 // Buffer: global reduction buffer.
4585 Argument *BufferArg = GtLRFunc->getArg(i: 0);
4586 // Idx: index of the buffer.
4587 Argument *IdxArg = GtLRFunc->getArg(i: 1);
4588 // ReduceList: thread local Reduce list.
4589 Argument *ReduceListArg = GtLRFunc->getArg(i: 2);
4590
4591 Value *BufferArgAlloca = Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr,
4592 Name: BufferArg->getName() + ".addr");
4593 Value *IdxArgAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr,
4594 Name: IdxArg->getName() + ".addr");
4595 Value *ReduceListArgAlloca = Builder.CreateAlloca(
4596 Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: ReduceListArg->getName() + ".addr");
4597 ArrayType *RedListArrayTy =
4598 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4599
4600 // 1. Build a list of reduction variables.
4601 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4602 Value *LocalReduceList =
4603 Builder.CreateAlloca(Ty: RedListArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4604
4605 InsertPointTy AllocaIP(EntryBlock->begin());
4606
4607 Value *BufferArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4608 V: BufferArgAlloca, DestTy: Builder.getPtrTy(),
4609 Name: BufferArgAlloca->getName() + ".ascast");
4610 Value *IdxArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4611 V: IdxArgAlloca, DestTy: Builder.getPtrTy(), Name: IdxArgAlloca->getName() + ".ascast");
4612 Value *ReduceListArgAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4613 V: ReduceListArgAlloca, DestTy: Builder.getPtrTy(),
4614 Name: ReduceListArgAlloca->getName() + ".ascast");
4615 Value *ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4616 V: LocalReduceList, DestTy: Builder.getPtrTy(),
4617 Name: LocalReduceList->getName() + ".ascast");
4618
4619 Builder.CreateStore(Val: BufferArg, Ptr: BufferArgAddrCast);
4620 Builder.CreateStore(Val: IdxArg, Ptr: IdxArgAddrCast);
4621 Builder.CreateStore(Val: ReduceListArg, Ptr: ReduceListArgAddrCast);
4622
4623 Value *BufferVal = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BufferArgAddrCast);
4624 Value *Idxs[] = {Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: IdxArgAddrCast)};
4625 Type *IndexTy = Builder.getIndexTy(
4626 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4627 for (auto En : enumerate(First&: ReductionInfos)) {
4628 const ReductionInfo &RI = En.value();
4629
4630 Value *TargetElementPtrPtr = Builder.CreateInBoundsGEP(
4631 Ty: RedListArrayTy, Ptr: ReductionList,
4632 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4633 // Global = Buffer.VD[Idx];
4634 Value *BufferVD =
4635 Builder.CreateInBoundsGEP(Ty: ReductionsBufferTy, Ptr: BufferVal, IdxList: Idxs);
4636 Value *GlobValPtr = Builder.CreateConstInBoundsGEP2_32(
4637 Ty: ReductionsBufferTy, Ptr: BufferVD, Idx0: 0, Idx1: En.index());
4638
4639 if (!IsByRef.empty() && IsByRef[En.index()] && RI.DataPtrPtrGen) {
4640 // Get source descriptor from the reduce list
4641 Value *ReduceListVal =
4642 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4643 Value *SrcElementPtrPtr =
4644 Builder.CreateInBoundsGEP(Ty: RedListArrayTy, Ptr: ReduceListVal,
4645 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0),
4646 ConstantInt::get(Ty: IndexTy, V: En.index())});
4647 Value *SrcDescriptorAddr =
4648 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: SrcElementPtrPtr);
4649
4650 // Copy descriptor from source and update base_ptr to global buffer data
4651 Expected<Value *> ByRefAlloc = createReductionDescriptorCopy(
4652 AllocaIP, RI, DataPtr: GlobValPtr, SrcDescriptorAddr, DescriptorPtrTy: Builder.getPtrTy());
4653 if (!ByRefAlloc)
4654 return ByRefAlloc.takeError();
4655
4656 Builder.CreateStore(Val: *ByRefAlloc, Ptr: TargetElementPtrPtr);
4657 } else {
4658 Builder.CreateStore(Val: GlobValPtr, Ptr: TargetElementPtrPtr);
4659 }
4660 }
4661
4662 // Call reduce_function(ReduceList, GlobalReduceList)
4663 Value *ReduceList =
4664 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: ReduceListArgAddrCast);
4665 createRuntimeFunctionCall(Callee: ReduceFn, Args: {ReduceList, ReductionList})
4666 ->addFnAttr(Kind: Attribute::NoUnwind);
4667 Builder.CreateRetVoid();
4668 return GtLRFunc;
4669}
4670
4671std::string OpenMPIRBuilder::getReductionFuncName(StringRef Name) const {
4672 std::string Suffix =
4673 createPlatformSpecificName(Parts: {"omp", "reduction", "reduction_func"});
4674 return (Name + Suffix).str();
4675}
4676
4677Expected<Function *> OpenMPIRBuilder::createReductionFunction(
4678 StringRef ReducerName, ArrayRef<ReductionInfo> ReductionInfos,
4679 ArrayRef<bool> IsByRef, ReductionGenCBKind ReductionGenCBKind,
4680 AttributeList FuncAttrs) {
4681 IRBuilder<>::InsertPointGuard IPG(Builder);
4682 auto *FuncTy = FunctionType::get(Result: Builder.getVoidTy(),
4683 Params: {Builder.getPtrTy(), Builder.getPtrTy()},
4684 /* IsVarArg */ isVarArg: false);
4685 std::string Name = getReductionFuncName(Name: ReducerName);
4686 Function *ReductionFunc =
4687 Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage, N: Name, M: &M);
4688 ReductionFunc->setCallingConv(Config.getRuntimeCC());
4689 ReductionFunc->setAttributes(FuncAttrs);
4690 ReductionFunc->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
4691 ReductionFunc->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
4692 BasicBlock *EntryBB =
4693 BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: ReductionFunc);
4694 Builder.SetInsertPoint(EntryBB);
4695 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
4696
4697 // Need to alloca memory here and deal with the pointers before getting
4698 // LHS/RHS pointers out
4699 Value *LHSArrayPtr = nullptr;
4700 Value *RHSArrayPtr = nullptr;
4701 Argument *Arg0 = ReductionFunc->getArg(i: 0);
4702 Argument *Arg1 = ReductionFunc->getArg(i: 1);
4703 Type *Arg0Type = Arg0->getType();
4704 Type *Arg1Type = Arg1->getType();
4705
4706 Value *LHSAlloca =
4707 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
4708 Value *RHSAlloca =
4709 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
4710 Value *LHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4711 V: LHSAlloca, DestTy: Arg0Type, Name: LHSAlloca->getName() + ".ascast");
4712 Value *RHSAddrCast = Builder.CreatePointerBitCastOrAddrSpaceCast(
4713 V: RHSAlloca, DestTy: Arg1Type, Name: RHSAlloca->getName() + ".ascast");
4714 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
4715 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
4716 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
4717 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
4718
4719 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: ReductionInfos.size());
4720 Type *IndexTy = Builder.getIndexTy(
4721 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4722 SmallVector<Value *> LHSPtrs, RHSPtrs;
4723 for (auto En : enumerate(First&: ReductionInfos)) {
4724 const ReductionInfo &RI = En.value();
4725 Value *RHSI8PtrPtr = Builder.CreateInBoundsGEP(
4726 Ty: RedArrayTy, Ptr: RHSArrayPtr,
4727 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4728 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
4729 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4730 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType(),
4731 Name: RHSI8Ptr->getName() + ".ascast");
4732
4733 Value *LHSI8PtrPtr = Builder.CreateInBoundsGEP(
4734 Ty: RedArrayTy, Ptr: LHSArrayPtr,
4735 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4736 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
4737 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4738 V: LHSI8Ptr, DestTy: RI.Variable->getType(), Name: LHSI8Ptr->getName() + ".ascast");
4739
4740 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
4741 LHSPtrs.emplace_back(Args&: LHSPtr);
4742 RHSPtrs.emplace_back(Args&: RHSPtr);
4743 } else {
4744 Value *LHS = LHSPtr;
4745 Value *RHS = RHSPtr;
4746
4747 if (!IsByRef.empty() && !IsByRef[En.index()]) {
4748 LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
4749 RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
4750 }
4751
4752 Value *Reduced;
4753 InsertPointOrErrorTy AfterIP =
4754 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
4755 if (!AfterIP)
4756 return AfterIP.takeError();
4757 if (!Builder.GetInsertBlock())
4758 return ReductionFunc;
4759
4760 Builder.restoreIP(IP: *AfterIP);
4761
4762 if (!IsByRef.empty() && !IsByRef[En.index()])
4763 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
4764 }
4765 }
4766
4767 if (ReductionGenCBKind == ReductionGenCBKind::Clang)
4768 for (auto En : enumerate(First&: ReductionInfos)) {
4769 unsigned Index = En.index();
4770 const ReductionInfo &RI = En.value();
4771 Value *LHSFixupPtr, *RHSFixupPtr;
4772 Builder.restoreIP(IP: RI.ReductionGenClang(
4773 Builder.saveIP(), Index, &LHSFixupPtr, &RHSFixupPtr, ReductionFunc));
4774
4775 // Fix the CallBack code genereated to use the correct Values for the LHS
4776 // and RHS
4777 LHSFixupPtr->replaceUsesWithIf(
4778 New: LHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4779 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4780 ReductionFunc;
4781 });
4782 RHSFixupPtr->replaceUsesWithIf(
4783 New: RHSPtrs[Index], ShouldReplace: [ReductionFunc](const Use &U) {
4784 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
4785 ReductionFunc;
4786 });
4787 }
4788
4789 Builder.CreateRetVoid();
4790 // Compiling with `-O0`, `alloca`s emitted in non-entry blocks are not hoisted
4791 // to the entry block (this is dones for higher opt levels by later passes in
4792 // the pipeline). This has caused issues because non-entry `alloca`s force the
4793 // function to use dynamic stack allocations and we might run out of scratch
4794 // memory.
4795 hoistNonEntryAllocasToEntryBlock(Func: ReductionFunc);
4796
4797 return ReductionFunc;
4798}
4799
4800static void
4801checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
4802 bool IsGPU) {
4803 for (const OpenMPIRBuilder::ReductionInfo &RI : ReductionInfos) {
4804 (void)RI;
4805 assert(RI.Variable && "expected non-null variable");
4806 assert(RI.PrivateVariable && "expected non-null private variable");
4807 assert((RI.ReductionGen || RI.ReductionGenClang) &&
4808 "expected non-null reduction generator callback");
4809 if (!IsGPU) {
4810 assert(
4811 RI.Variable->getType() == RI.PrivateVariable->getType() &&
4812 "expected variables and their private equivalents to have the same "
4813 "type");
4814 }
4815 assert(RI.Variable->getType()->isPointerTy() &&
4816 "expected variables to be pointers");
4817 }
4818}
4819
4820// The atomic cross-team reduction fast path applies when every reduction in the
4821// set can be represented by an atomicrmw. Clang only populates it for scalar
4822// reductions with a supported atomic operator.
4823static bool isAtomicableReductionSet(
4824 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos) {
4825 return all_of(Range&: ReductionInfos, P: [](const OpenMPIRBuilder::ReductionInfo &RI) {
4826 return static_cast<bool>(RI.AtomicReductionGen);
4827 });
4828}
4829
4830OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
4831 const LocationDescription &Loc, InsertPointTy AllocaIP,
4832 InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
4833 ArrayRef<bool> IsByRef, bool IsNoWait, bool IsTeamsReduction, bool IsSPMD,
4834 ReductionGenCBKind ReductionGenCBKind, std::optional<omp::GV> GridValue,
4835 Value *SrcLocInfo) {
4836 if (!updateToLocation(Loc))
4837 return InsertPointTy();
4838 Builder.restoreIP(IP: CodeGenIP);
4839 Builder.SetCurrentDebugLocation(Loc.DL);
4840 checkReductionInfos(ReductionInfos, /*IsGPU*/ true);
4841 LLVMContext &Ctx = M.getContext();
4842
4843 // Source location for the ident struct
4844 if (!SrcLocInfo) {
4845 uint32_t SrcLocStrSize;
4846 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4847 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4848 }
4849
4850 if (ReductionInfos.size() == 0)
4851 return Builder.saveIP();
4852
4853 BasicBlock *ContinuationBlock = nullptr;
4854 if (ReductionGenCBKind != ReductionGenCBKind::Clang) {
4855 // Copied code from createReductions
4856 BasicBlock *InsertBlock = Loc.IP.getNodeParent();
4857 ContinuationBlock = InsertBlock->splitBasicBlock(I: Loc.IP, BBName: "reduce.finalize");
4858 InsertBlock->getTerminator()->eraseFromParent();
4859 Builder.SetInsertPoint(InsertBlock->end());
4860 }
4861
4862 Function *CurFunc = Builder.GetInsertBlock()->getParent();
4863 AttributeList FuncAttrs;
4864 AttrBuilder AttrBldr(Ctx);
4865 for (auto Attr : CurFunc->getAttributes().getFnAttrs())
4866 AttrBldr.addAttribute(A: Attr);
4867 AttrBldr.removeAttribute(Val: Attribute::OptimizeNone);
4868 FuncAttrs = FuncAttrs.addFnAttributes(C&: Ctx, B: AttrBldr);
4869
4870 Expected<Function *> ReductionResult = createReductionFunction(
4871 ReducerName: Builder.GetInsertBlock()->getParent()->getName(), ReductionInfos, IsByRef,
4872 ReductionGenCBKind, FuncAttrs);
4873 if (!ReductionResult)
4874 return ReductionResult.takeError();
4875 Function *ReductionFunc = *ReductionResult;
4876
4877 // Set the grid value in the config needed for lowering later on
4878 if (GridValue.has_value())
4879 Config.setGridValue(GridValue.value());
4880 else
4881 Config.setGridValue(getGridValue(T, Kernel: ReductionFunc));
4882
4883 // Build res = __kmpc_reduce{_nowait}(<gtid>, <n>, sizeof(RedList),
4884 // RedList, shuffle_reduce_func, interwarp_copy_func);
4885 // or
4886 // Build res = __kmpc_reduce_teams_nowait_simple(<loc>, <gtid>, <lck>);
4887 Value *Res;
4888
4889 // 1. Build a list of reduction variables.
4890 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
4891 auto Size = ReductionInfos.size();
4892 Type *PtrTy = PointerType::get(C&: Ctx, AddressSpace: Config.getDefaultTargetAS());
4893 Type *FuncPtrTy =
4894 Builder.getPtrTy(AddrSpace: M.getDataLayout().getProgramAddressSpace());
4895 Type *RedArrayTy = ArrayType::get(ElementType: PtrTy, NumElements: Size);
4896 Value *ReductionList;
4897 {
4898 IRBuilder<>::InsertPointGuard IPG(Builder);
4899 Builder.restoreIP(IP: AllocaIP);
4900 Value *ReductionListAlloca =
4901 Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: ".omp.reduction.red_list");
4902 ReductionList = Builder.CreatePointerBitCastOrAddrSpaceCast(
4903 V: ReductionListAlloca, DestTy: PtrTy, Name: ReductionListAlloca->getName() + ".ascast");
4904 }
4905 Type *IndexTy = Builder.getIndexTy(
4906 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
4907 for (auto En : enumerate(First&: ReductionInfos)) {
4908 const ReductionInfo &RI = En.value();
4909 Value *ElemPtr = Builder.CreateInBoundsGEP(
4910 Ty: RedArrayTy, Ptr: ReductionList,
4911 IdxList: {ConstantInt::get(Ty: IndexTy, V: 0), ConstantInt::get(Ty: IndexTy, V: En.index())});
4912
4913 Value *PrivateVar = RI.PrivateVariable;
4914 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
4915 if (IsByRefElem)
4916 PrivateVar = Builder.CreateLoad(Ty: RI.ElementType, Ptr: PrivateVar);
4917
4918 Value *CastElem =
4919 Builder.CreatePointerBitCastOrAddrSpaceCast(V: PrivateVar, DestTy: PtrTy);
4920 Builder.CreateStore(Val: CastElem, Ptr: ElemPtr);
4921 }
4922 Expected<Function *> SarFunc = emitShuffleAndReduceFunction(
4923 ReductionInfos, ReduceFn: ReductionFunc, FuncAttrs, IsByRef);
4924
4925 if (!SarFunc)
4926 return SarFunc.takeError();
4927
4928 Expected<Function *> CopyResult =
4929 emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs, IsByRef);
4930 if (!CopyResult)
4931 return CopyResult.takeError();
4932 Function *WcFunc = *CopyResult;
4933
4934 Value *RL = Builder.CreatePointerBitCastOrAddrSpaceCast(V: ReductionList, DestTy: PtrTy);
4935
4936 // NOTE: ReductionDataSize is passed as the reduce_data_size argument to
4937 // __kmpc_nvptx_parallel_reduce_nowait_v2, but the runtime implementations do
4938 // not currently use it. It is computed here conservatively as max(element
4939 // sizes) * N rather than the exact sum, which over-calculates the size for
4940 // mixed reduction types but is harmless given the argument is unused.
4941 // TODO: Consider dropping this computation if the runtime API is ever revised
4942 // to remove the unused parameter.
4943 unsigned MaxDataSize = 0;
4944 SmallVector<Type *> ReductionTypeArgs;
4945 for (auto En : enumerate(First&: ReductionInfos)) {
4946 // Use ByRefElementType for by-ref reductions so that MaxDataSize matches
4947 // the actual data size stored in the global reduction buffer, consistent
4948 // with the ReductionsBufferTy struct used for GEP offsets below.
4949 Type *RedTypeArg = (!IsByRef.empty() && IsByRef[En.index()])
4950 ? En.value().ByRefElementType
4951 : En.value().ElementType;
4952 auto Size = M.getDataLayout().getTypeStoreSize(Ty: RedTypeArg);
4953 if (Size > MaxDataSize)
4954 MaxDataSize = Size;
4955 ReductionTypeArgs.emplace_back(Args&: RedTypeArg);
4956 }
4957 Value *ReductionDataSize =
4958 Builder.getInt64(C: MaxDataSize * ReductionInfos.size());
4959
4960 // Helper function to copy thread-local data back to the original reduction
4961 // list.
4962 Function *CopyScratchToListFunc = nullptr;
4963 // Thread-local storage for the reduction variables.
4964 Value *ScratchForCopyBack = nullptr;
4965 // RL pointer to which the final value from the per-thread scratch should be
4966 // copied back. (Basically RL, appropriately casted if necessary.)
4967 Value *RLForCopyBack = RL;
4968
4969 bool IsAtomicReduction =
4970 IsTeamsReduction && isAtomicableReductionSet(ReductionInfos);
4971
4972 if (!IsTeamsReduction) {
4973 Value *SarFuncCast =
4974 Builder.CreatePointerBitCastOrAddrSpaceCast(V: *SarFunc, DestTy: FuncPtrTy);
4975 Value *WcFuncCast =
4976 Builder.CreatePointerBitCastOrAddrSpaceCast(V: WcFunc, DestTy: FuncPtrTy);
4977 Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
4978 WcFuncCast};
4979 Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
4980 FnID: RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
4981 Res = createRuntimeFunctionCall(Callee: Pv2Ptr, Args);
4982 } else if (IsAtomicReduction) {
4983 // Atomic cross-team reduction fast path: determine the team's main thread
4984 // that is later to fold its value atomically into the mapped variable.
4985 Function *IsMainThreadFn = getOrCreateRuntimeFunctionPtr(
4986 FnID: RuntimeFunction::OMPRTL___kmpc_is_team_main_thread);
4987 Res = createRuntimeFunctionCall(Callee: IsMainThreadFn, Args: {});
4988 } else {
4989 StructType *ReductionsBufferTy = StructType::create(
4990 Context&: Ctx, Elements: ReductionTypeArgs, Name: "struct._globalized_locals_ty");
4991
4992 Expected<Function *> LtGCFunc = emitListToGlobalCopyFunction(
4993 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4994 if (!LtGCFunc)
4995 return LtGCFunc.takeError();
4996
4997 Expected<Function *> GtLCFunc = emitGlobalToListCopyFunction(
4998 ReductionInfos, ReductionsBufferTy, FuncAttrs, IsByRef);
4999 if (!GtLCFunc)
5000 return GtLCFunc.takeError();
5001
5002 Expected<Function *> GtLRFunc = emitGlobalToListReduceFunction(
5003 ReductionInfos, ReduceFn: ReductionFunc, ReductionsBufferTy, FuncAttrs, IsByRef);
5004 if (!GtLRFunc)
5005 return GtLRFunc.takeError();
5006
5007 // The runtime's cross-team final aggregate uses the storage pointed at by
5008 // its reduce-list argument as per-thread scratch. When the surrounding
5009 // kernel is already in SPMD execution mode, clang emitted each reduction
5010 // private as a per-thread `alloca addrspace(5)`, so the original red_list
5011 // (RL) is already per-thread and nothing else is needed.
5012 //
5013 // When the kernel is in Non-SPMD execution mode at codegen time, clang's
5014 // Generic-mode globalization put the reduction private into team-shared
5015 // LDS. OpenMPOpt may later upgrade the kernel to Generic-SPMD, at which
5016 // point all threads of the last team would race on the shared LDS slot.
5017 // Emit a per-thread scratch buffer and a per-thread RL, copy the team-local
5018 // value in, and hand the per-thread RL to the runtime instead. The writer
5019 // thread copies the final value from that per-thread scratch back to RL
5020 // before running the existing combine path below.
5021
5022 // Thread-local RL (might need localization below before being passed to the
5023 // runtime).
5024 Value *RuntimeRL = RL;
5025
5026 if (!IsSPMD) {
5027 Value *PerThreadScratch;
5028
5029 {
5030 IRBuilder<>::InsertPointGuard IPG(Builder);
5031 Builder.restoreIP(IP: AllocaIP);
5032 // Allocate thread-local buffer for the reduction variables.
5033 Value *PerThreadScratchAlloca =
5034 Builder.CreateAlloca(Ty: ReductionsBufferTy, /*ArraySize=*/nullptr,
5035 Name: ".omp.reduction.scratch");
5036 PerThreadScratch = Builder.CreatePointerBitCastOrAddrSpaceCast(
5037 V: PerThreadScratchAlloca, DestTy: PtrTy,
5038 Name: PerThreadScratchAlloca->getName() + ".ascast");
5039 // Allocate thread-local buffer for the pointers to the reduction
5040 // variables.
5041 Value *PerThreadRedListAlloca =
5042 Builder.CreateAlloca(Ty: RedArrayTy, /*ArraySize=*/nullptr,
5043 Name: ".omp.reduction.per_thread_red_list");
5044 RuntimeRL = Builder.CreatePointerBitCastOrAddrSpaceCast(
5045 V: PerThreadRedListAlloca, DestTy: PtrTy,
5046 Name: PerThreadRedListAlloca->getName() + ".ascast");
5047 }
5048
5049 // Iterate over the reduction variables and copy the team-local value to
5050 // the thread-local buffer.
5051 for (auto En : enumerate(First&: ReductionInfos)) {
5052 const ReductionInfo &RI = En.value();
5053 bool IsByRefElem = !IsByRef.empty() && IsByRef[En.index()];
5054
5055 Value *FieldPtr = Builder.CreateConstInBoundsGEP2_32(
5056 Ty: ReductionsBufferTy, Ptr: PerThreadScratch, Idx0: 0, Idx1: En.index());
5057 Value *Slot = Builder.CreateConstInBoundsGEP2_32(Ty: RedArrayTy, Ptr: RuntimeRL,
5058 Idx0: 0, Idx1: En.index());
5059
5060 Value *RuntimeListEntry = FieldPtr;
5061 if (IsByRefElem && RI.DataPtrPtrGen) {
5062 Value *SrcDescriptor =
5063 Builder.CreateLoad(Ty: RI.ElementType, Ptr: RI.PrivateVariable);
5064 Expected<Value *> Descriptor = createReductionDescriptorCopy(
5065 AllocaIP, RI, DataPtr: FieldPtr, SrcDescriptorAddr: SrcDescriptor, DescriptorPtrTy: PtrTy);
5066 if (!Descriptor)
5067 return Descriptor.takeError();
5068 RuntimeListEntry = *Descriptor;
5069 }
5070 Builder.CreateStore(Val: RuntimeListEntry, Ptr: Slot);
5071 }
5072 // The copy helpers were emitted with default-AS (AS 0) pointer params
5073 // (see emitListToGlobalCopyFunction / emitGlobalToListCopyFunction),
5074 // but PerThreadScratch and RL live in the target's default AS, which
5075 // is non-zero on e.g. SPIRV. (See Config.getDefaultTargetAS().)
5076 Type *CopyArg0Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 0);
5077 Type *CopyArg2Ty = (*LtGCFunc)->getFunctionType()->getParamType(i: 2);
5078 ScratchForCopyBack = Builder.CreatePointerBitCastOrAddrSpaceCast(
5079 V: PerThreadScratch, DestTy: CopyArg0Ty);
5080 RLForCopyBack =
5081 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RL, DestTy: CopyArg2Ty);
5082 // Use index 0 because there is no array of target values to index into,
5083 // there is only one thread-local memory slot.
5084 // restoreIP above left a stale/empty debug location; this inlinable call
5085 // to a debug-info-bearing helper needs one or the verifier rejects the
5086 // module ("!dbg attachment points at wrong subprogram") after inlining.
5087 Builder.SetCurrentDebugLocation(Loc.DL);
5088 Builder.CreateCall(
5089 Callee: *LtGCFunc, Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5090 CopyScratchToListFunc = *GtLCFunc;
5091 }
5092
5093 Value *Args3[] = {SrcLocInfo, RuntimeRL, *SarFunc, WcFunc,
5094 *LtGCFunc, *GtLCFunc, *GtLRFunc};
5095
5096 Function *TeamsReduceFn = getOrCreateRuntimeFunctionPtr(
5097 FnID: RuntimeFunction::OMPRTL___kmpc_gpu_xteam_reduce_nowait);
5098 Res = createRuntimeFunctionCall(Callee: TeamsReduceFn, Args: Args3);
5099 }
5100
5101 // 5. Build if (res == 1)
5102 BasicBlock *ExitBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.done");
5103 BasicBlock *ThenBB = BasicBlock::Create(Context&: Ctx, Name: ".omp.reduction.then");
5104 Value *Cond = Builder.CreateICmpEQ(LHS: Res, RHS: Builder.getInt32(C: 1));
5105 Builder.CreateCondBr(Cond, True: ThenBB, False: ExitBB);
5106
5107 // 6. Build then branch: where we have reduced values in the master
5108 // thread in each team.
5109 // __kmpc_end_reduce{_nowait}(<gtid>);
5110 // break;
5111 emitBlock(BB: ThenBB, CurFn: CurFunc);
5112
5113 // Copy the writer thread's per-thread scratch result back into the original
5114 // red-list storage before the existing combine path reads RI.PrivateVariable.
5115 // Set a debug location: this inlinable call to a debug-info-bearing helper
5116 // needs one or the verifier rejects the module after inlining.
5117 if (ScratchForCopyBack) {
5118 Builder.SetCurrentDebugLocation(Loc.DL);
5119 Builder.CreateCall(
5120 Callee: CopyScratchToListFunc,
5121 Args: {ScratchForCopyBack, Builder.getInt32(C: 0), RLForCopyBack});
5122 }
5123
5124 // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
5125 for (auto En : enumerate(First&: ReductionInfos)) {
5126 const ReductionInfo &RI = En.value();
5127
5128 // Atomic cross-team fast path: each team's main thread folds its
5129 // team-reduced value directly into the mapped reduction variable with a
5130 // single atomicrmw.
5131 if (IsAtomicReduction) {
5132 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5133 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5134 if (!AfterIP)
5135 return AfterIP.takeError();
5136 Builder.restoreIP(IP: *AfterIP);
5137 continue;
5138 }
5139
5140 Type *ValueType = RI.ElementType;
5141 Value *RedValue = RI.Variable;
5142
5143 Value *RHS =
5144 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RI.PrivateVariable, DestTy: PtrTy);
5145
5146 if (ReductionGenCBKind == ReductionGenCBKind::Clang) {
5147 Value *LHSPtr, *RHSPtr;
5148 Builder.restoreIP(IP: RI.ReductionGenClang(Builder.saveIP(), En.index(),
5149 &LHSPtr, &RHSPtr, CurFunc));
5150
5151 // Fix the CallBack code genereated to use the correct Values for the LHS
5152 // and RHS. Cast to match types before replacing (necessary to handle
5153 // different address spaces).
5154 if (LHSPtr->getType() != RedValue->getType())
5155 RedValue = Builder.CreatePointerBitCastOrAddrSpaceCast(
5156 V: RedValue, DestTy: LHSPtr->getType());
5157 if (RHSPtr->getType() != RHS->getType())
5158 RHS =
5159 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHS, DestTy: RHSPtr->getType());
5160
5161 LHSPtr->replaceUsesWithIf(New: RedValue, ShouldReplace: [ReductionFunc](const Use &U) {
5162 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5163 ReductionFunc;
5164 });
5165 RHSPtr->replaceUsesWithIf(New: RHS, ShouldReplace: [ReductionFunc](const Use &U) {
5166 return cast<Instruction>(Val: U.getUser())->getParent()->getParent() ==
5167 ReductionFunc;
5168 });
5169 } else {
5170 if (IsByRef.empty() || !IsByRef[En.index()]) {
5171 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5172 Name: "red.value." + Twine(En.index()));
5173 }
5174 Value *PrivateRedValue = Builder.CreateLoad(
5175 Ty: ValueType, Ptr: RHS, Name: "red.private.value" + Twine(En.index()));
5176 Value *Reduced;
5177 InsertPointOrErrorTy AfterIP =
5178 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5179 if (!AfterIP)
5180 return AfterIP.takeError();
5181 Builder.restoreIP(IP: *AfterIP);
5182
5183 if (!IsByRef.empty() && !IsByRef[En.index()])
5184 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5185 }
5186 }
5187 emitBlock(BB: ExitBB, CurFn: CurFunc);
5188 if (ContinuationBlock) {
5189 Builder.CreateBr(Dest: ContinuationBlock);
5190 Builder.SetInsertPoint(ContinuationBlock);
5191 }
5192 Config.setEmitLLVMUsed();
5193
5194 return Builder.saveIP();
5195}
5196
5197static Function *getFreshReductionFunc(Module &M) {
5198 Type *VoidTy = Type::getVoidTy(C&: M.getContext());
5199 Type *Int8PtrTy = PointerType::getUnqual(C&: M.getContext());
5200 auto *FuncTy =
5201 FunctionType::get(Result: VoidTy, Params: {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ isVarArg: false);
5202 return Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
5203 N: ".omp.reduction.func", M: &M);
5204}
5205
5206static Error populateReductionFunction(
5207 Function *ReductionFunc,
5208 ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5209 IRBuilder<> &Builder, ArrayRef<bool> IsByRef, bool IsGPU) {
5210 IRBuilder<>::InsertPointGuard IPG(Builder);
5211 Module *Module = ReductionFunc->getParent();
5212 BasicBlock *ReductionFuncBlock =
5213 BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: ReductionFunc);
5214 Builder.SetInsertPoint(ReductionFuncBlock);
5215 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
5216 Value *LHSArrayPtr = nullptr;
5217 Value *RHSArrayPtr = nullptr;
5218 if (IsGPU) {
5219 // Need to alloca memory here and deal with the pointers before getting
5220 // LHS/RHS pointers out
5221 //
5222 Argument *Arg0 = ReductionFunc->getArg(i: 0);
5223 Argument *Arg1 = ReductionFunc->getArg(i: 1);
5224 Type *Arg0Type = Arg0->getType();
5225 Type *Arg1Type = Arg1->getType();
5226
5227 Value *LHSAlloca =
5228 Builder.CreateAlloca(Ty: Arg0Type, ArraySize: nullptr, Name: Arg0->getName() + ".addr");
5229 Value *RHSAlloca =
5230 Builder.CreateAlloca(Ty: Arg1Type, ArraySize: nullptr, Name: Arg1->getName() + ".addr");
5231 Value *LHSAddrCast =
5232 Builder.CreatePointerBitCastOrAddrSpaceCast(V: LHSAlloca, DestTy: Arg0Type);
5233 Value *RHSAddrCast =
5234 Builder.CreatePointerBitCastOrAddrSpaceCast(V: RHSAlloca, DestTy: Arg1Type);
5235 Builder.CreateStore(Val: Arg0, Ptr: LHSAddrCast);
5236 Builder.CreateStore(Val: Arg1, Ptr: RHSAddrCast);
5237 LHSArrayPtr = Builder.CreateLoad(Ty: Arg0Type, Ptr: LHSAddrCast);
5238 RHSArrayPtr = Builder.CreateLoad(Ty: Arg1Type, Ptr: RHSAddrCast);
5239 } else {
5240 LHSArrayPtr = ReductionFunc->getArg(i: 0);
5241 RHSArrayPtr = ReductionFunc->getArg(i: 1);
5242 }
5243
5244 unsigned NumReductions = ReductionInfos.size();
5245 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5246
5247 for (auto En : enumerate(First&: ReductionInfos)) {
5248 const OpenMPIRBuilder::ReductionInfo &RI = En.value();
5249 Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5250 Ty: RedArrayTy, Ptr: LHSArrayPtr, Idx0: 0, Idx1: En.index());
5251 Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
5252 Value *LHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5253 V: LHSI8Ptr, DestTy: RI.Variable->getType());
5254 Value *LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
5255 Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
5256 Ty: RedArrayTy, Ptr: RHSArrayPtr, Idx0: 0, Idx1: En.index());
5257 Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
5258 Value *RHSPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5259 V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType());
5260 Value *RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
5261 Value *Reduced;
5262 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5263 RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced);
5264 if (!AfterIP)
5265 return AfterIP.takeError();
5266
5267 Builder.restoreIP(IP: *AfterIP);
5268 // TODO: Consider flagging an error.
5269 if (!Builder.GetInsertBlock())
5270 return Error::success();
5271
5272 // store is inside of the reduction region when using by-ref
5273 if (!IsByRef[En.index()])
5274 Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
5275 }
5276 Builder.CreateRetVoid();
5277 return Error::success();
5278}
5279
5280OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductions(
5281 const LocationDescription &Loc, InsertPointTy AllocaIP,
5282 ArrayRef<ReductionInfo> ReductionInfos, ArrayRef<bool> IsByRef,
5283 bool IsNoWait, bool IsTeamsReduction) {
5284 assert(ReductionInfos.size() == IsByRef.size());
5285 if (Config.isGPU())
5286 return createReductionsGPU(Loc, AllocaIP, CodeGenIP: Builder.saveIP(), ReductionInfos,
5287 IsByRef, IsNoWait, IsTeamsReduction);
5288
5289 checkReductionInfos(ReductionInfos, /*IsGPU*/ false);
5290
5291 if (!updateToLocation(Loc))
5292 return InsertPointTy();
5293
5294 if (ReductionInfos.size() == 0)
5295 return Builder.saveIP();
5296
5297 BasicBlock *InsertBlock = Loc.IP.getNodeParent();
5298 BasicBlock *ContinuationBlock =
5299 InsertBlock->splitBasicBlock(I: Loc.IP, BBName: "reduce.finalize");
5300 InsertBlock->getTerminator()->eraseFromParent();
5301
5302 // Create and populate array of type-erased pointers to private reduction
5303 // values.
5304 unsigned NumReductions = ReductionInfos.size();
5305 Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
5306 Builder.SetInsertPoint(AllocaIP.getNodeParent()->getTerminator());
5307 Value *RedArray = Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: "red.array");
5308
5309 Builder.SetInsertPoint(InsertBlock->end());
5310 // Emitting the alloca moved the insertion point into the alloca block and
5311 // can clear the debug loc. Restore back to Loc.DL.
5312 Builder.SetCurrentDebugLocation(Loc.DL);
5313
5314 for (auto En : enumerate(First&: ReductionInfos)) {
5315 unsigned Index = En.index();
5316 const ReductionInfo &RI = En.value();
5317 Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
5318 Ty: RedArrayTy, Ptr: RedArray, Idx0: 0, Idx1: Index, Name: "red.array.elem." + Twine(Index));
5319 Builder.CreateStore(Val: RI.PrivateVariable, Ptr: RedArrayElemPtr);
5320 }
5321
5322 // Emit a call to the runtime function that orchestrates the reduction.
5323 // Declare the reduction function in the process.
5324 Type *IndexTy = Builder.getIndexTy(
5325 DL: M.getDataLayout(), AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace());
5326 Function *Func = Builder.GetInsertBlock()->getParent();
5327 Module *Module = Func->getParent();
5328 uint32_t SrcLocStrSize;
5329 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5330 bool CanGenerateAtomic = all_of(Range&: ReductionInfos, P: [](const ReductionInfo &RI) {
5331 return RI.AtomicReductionGen;
5332 });
5333 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
5334 LocFlags: CanGenerateAtomic
5335 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
5336 : IdentFlag(0));
5337 Value *ThreadId = getOrCreateThreadID(Ident);
5338 Constant *NumVariables = Builder.getInt32(C: NumReductions);
5339 const DataLayout &DL = Module->getDataLayout();
5340 unsigned RedArrayByteSize = DL.getTypeStoreSize(Ty: RedArrayTy);
5341 Constant *RedArraySize = ConstantInt::get(Ty: IndexTy, V: RedArrayByteSize);
5342 Function *ReductionFunc = getFreshReductionFunc(M&: *Module);
5343 Value *Lock = getOMPCriticalRegionLock(CriticalName: ".reduction");
5344 Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
5345 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
5346 : RuntimeFunction::OMPRTL___kmpc_reduce);
5347 CallInst *ReduceCall =
5348 createRuntimeFunctionCall(Callee: ReduceFunc,
5349 Args: {Ident, ThreadId, NumVariables, RedArraySize,
5350 RedArray, ReductionFunc, Lock},
5351 Name: "reduce");
5352
5353 // Create final reduction entry blocks for the atomic and non-atomic case.
5354 // Emit IR that dispatches control flow to one of the blocks based on the
5355 // reduction supporting the atomic mode.
5356 BasicBlock *NonAtomicRedBlock =
5357 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.nonatomic", Parent: Func);
5358 BasicBlock *AtomicRedBlock =
5359 BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.atomic", Parent: Func);
5360 SwitchInst *Switch =
5361 Builder.CreateSwitch(V: ReduceCall, Dest: ContinuationBlock, /* NumCases */ 2);
5362 Switch->addCase(OnVal: Builder.getInt32(C: 1), Dest: NonAtomicRedBlock);
5363 Switch->addCase(OnVal: Builder.getInt32(C: 2), Dest: AtomicRedBlock);
5364
5365 // Populate the non-atomic reduction using the elementwise reduction function.
5366 // This loads the elements from the global and private variables and reduces
5367 // them before storing back the result to the global variable.
5368 Builder.SetInsertPoint(NonAtomicRedBlock);
5369 for (auto En : enumerate(First&: ReductionInfos)) {
5370 const ReductionInfo &RI = En.value();
5371 Type *ValueType = RI.ElementType;
5372 // We have one less load for by-ref case because that load is now inside of
5373 // the reduction region
5374 Value *RedValue = RI.Variable;
5375 if (!IsByRef[En.index()]) {
5376 RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
5377 Name: "red.value." + Twine(En.index()));
5378 }
5379 Value *PrivateRedValue =
5380 Builder.CreateLoad(Ty: ValueType, Ptr: RI.PrivateVariable,
5381 Name: "red.private.value." + Twine(En.index()));
5382 Value *Reduced;
5383 InsertPointOrErrorTy AfterIP =
5384 RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced);
5385 if (!AfterIP)
5386 return AfterIP.takeError();
5387 Builder.restoreIP(IP: *AfterIP);
5388
5389 if (!Builder.GetInsertBlock())
5390 return InsertPointTy();
5391 // for by-ref case, the load is inside of the reduction region
5392 if (!IsByRef[En.index()])
5393 Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
5394 }
5395 Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
5396 FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
5397 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
5398 createRuntimeFunctionCall(Callee: EndReduceFunc, Args: {Ident, ThreadId, Lock});
5399 Builder.CreateBr(Dest: ContinuationBlock);
5400
5401 // Populate the atomic reduction using the atomic elementwise reduction
5402 // function. There are no loads/stores here because they will be happening
5403 // inside the atomic elementwise reduction.
5404 Builder.SetInsertPoint(AtomicRedBlock);
5405 if (CanGenerateAtomic && llvm::none_of(Range&: IsByRef, P: [](bool P) { return P; })) {
5406 for (const ReductionInfo &RI : ReductionInfos) {
5407 InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
5408 Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
5409 if (!AfterIP)
5410 return AfterIP.takeError();
5411 Builder.restoreIP(IP: *AfterIP);
5412 if (!Builder.GetInsertBlock())
5413 return InsertPointTy();
5414 }
5415 Builder.CreateBr(Dest: ContinuationBlock);
5416 } else {
5417 Builder.CreateUnreachable();
5418 }
5419
5420 // Populate the outlined reduction function using the elementwise reduction
5421 // function. Partial values are extracted from the type-erased array of
5422 // pointers to private variables.
5423 Error Err = populateReductionFunction(ReductionFunc, ReductionInfos, Builder,
5424 IsByRef, /*isGPU=*/IsGPU: false);
5425 if (Err)
5426 return Err;
5427
5428 if (!Builder.GetInsertBlock())
5429 return InsertPointTy();
5430
5431 Builder.SetInsertPoint(ContinuationBlock);
5432 return Builder.saveIP();
5433}
5434
5435OpenMPIRBuilder::InsertPointOrErrorTy
5436OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
5437 BodyGenCallbackTy BodyGenCB,
5438 FinalizeCallbackTy FiniCB) {
5439 if (!updateToLocation(Loc))
5440 return Loc.IP;
5441
5442 Directive OMPD = Directive::OMPD_master;
5443 uint32_t SrcLocStrSize;
5444 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5445 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5446 Value *ThreadId = getOrCreateThreadID(Ident);
5447 Value *Args[] = {Ident, ThreadId};
5448
5449 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_master);
5450 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5451
5452 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_master);
5453 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
5454
5455 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5456 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5457}
5458
5459OpenMPIRBuilder::InsertPointOrErrorTy
5460OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
5461 BodyGenCallbackTy BodyGenCB,
5462 FinalizeCallbackTy FiniCB, Value *Filter) {
5463 IRBuilder<>::InsertPointGuard IPG(Builder);
5464 if (!updateToLocation(Loc))
5465 return Loc.IP;
5466
5467 Directive OMPD = Directive::OMPD_masked;
5468 uint32_t SrcLocStrSize;
5469 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
5470 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
5471 Value *ThreadId = getOrCreateThreadID(Ident);
5472 Value *Args[] = {Ident, ThreadId, Filter};
5473 Value *ArgsEnd[] = {Ident, ThreadId};
5474
5475 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_masked);
5476 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
5477
5478 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_masked);
5479 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args: ArgsEnd);
5480
5481 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5482 /*Conditional*/ true, /*hasFinalize*/ HasFinalize: true);
5483}
5484
5485static llvm::CallInst *emitNoUnwindRuntimeCall(IRBuilder<> &Builder,
5486 llvm::FunctionCallee Callee,
5487 ArrayRef<llvm::Value *> Args,
5488 const llvm::Twine &Name) {
5489 llvm::CallInst *Call = Builder.CreateCall(
5490 Callee, Args, OpBundles: SmallVector<llvm::OperandBundleDef, 1>(), Name);
5491 Call->setDoesNotThrow();
5492 return Call;
5493}
5494
5495// Expects input basic block is dominated by BeforeScanBB.
5496// Once Scan directive is encountered, the code after scan directive should be
5497// dominated by AfterScanBB. Scan directive splits the code sequence to
5498// scan and input phase. Based on whether inclusive or exclusive
5499// clause is used in the scan directive and whether input loop or scan loop
5500// is lowered, it adds jumps to input and scan phase. First Scan loop is the
5501// input loop and second is the scan loop. The code generated handles only
5502// inclusive scans now.
5503OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createScan(
5504 const LocationDescription &Loc, InsertPointTy AllocaIP,
5505 ArrayRef<llvm::Value *> ScanVars, ArrayRef<llvm::Type *> ScanVarsType,
5506 bool IsInclusive, ScanInfo *ScanRedInfo) {
5507 if (ScanRedInfo->OMPFirstScanLoop) {
5508 llvm::Error Err = emitScanBasedDirectiveDeclsIR(AllocaIP, ScanVars,
5509 ScanVarsType, ScanRedInfo);
5510 if (Err)
5511 return Err;
5512 }
5513 if (!updateToLocation(Loc))
5514 return Loc.IP;
5515
5516 llvm::Value *IV = ScanRedInfo->IV;
5517
5518 if (ScanRedInfo->OMPFirstScanLoop) {
5519 // Emit buffer[i] = red; at the end of the input phase.
5520 for (size_t i = 0; i < ScanVars.size(); i++) {
5521 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5522 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5523 Type *DestTy = ScanVarsType[i];
5524 Value *Val = Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5525 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: ScanVars[i]);
5526
5527 Builder.CreateStore(Val: Src, Ptr: Val);
5528 }
5529 }
5530 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5531 emitBlock(BB: ScanRedInfo->OMPScanDispatch,
5532 CurFn: Builder.GetInsertBlock()->getParent());
5533
5534 if (!ScanRedInfo->OMPFirstScanLoop) {
5535 IV = ScanRedInfo->IV;
5536 // Emit red = buffer[i]; at the entrance to the scan phase.
5537 // TODO: if exclusive scan, the red = buffer[i-1] needs to be updated.
5538 for (size_t i = 0; i < ScanVars.size(); i++) {
5539 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
5540 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5541 Type *DestTy = ScanVarsType[i];
5542 Value *SrcPtr =
5543 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5544 Value *Src = Builder.CreateLoad(Ty: DestTy, Ptr: SrcPtr);
5545 Builder.CreateStore(Val: Src, Ptr: ScanVars[i]);
5546 }
5547 }
5548
5549 // TODO: Update it to CreateBr and remove dead blocks
5550 llvm::Value *CmpI = Builder.getInt1(V: true);
5551 if (ScanRedInfo->OMPFirstScanLoop == IsInclusive) {
5552 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPBeforeScanBlock,
5553 False: ScanRedInfo->OMPAfterScanBlock);
5554 } else {
5555 Builder.CreateCondBr(Cond: CmpI, True: ScanRedInfo->OMPAfterScanBlock,
5556 False: ScanRedInfo->OMPBeforeScanBlock);
5557 }
5558 emitBlock(BB: ScanRedInfo->OMPAfterScanBlock,
5559 CurFn: Builder.GetInsertBlock()->getParent());
5560 Builder.SetInsertPoint(ScanRedInfo->OMPAfterScanBlock);
5561 return Builder.saveIP();
5562}
5563
5564Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
5565 InsertPointTy AllocaIP, ArrayRef<Value *> ScanVars,
5566 ArrayRef<Type *> ScanVarsType, ScanInfo *ScanRedInfo) {
5567
5568 Builder.restoreIP(IP: AllocaIP);
5569 // Create the shared pointer at alloca IP.
5570 for (size_t i = 0; i < ScanVars.size(); i++) {
5571 llvm::Value *BuffPtr =
5572 Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: nullptr, Name: "vla");
5573 (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]] = BuffPtr;
5574 }
5575
5576 // Allocate temporary buffer by master thread
5577 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5578 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5579 Builder.restoreIP(IP: CodeGenIP);
5580 Value *AllocSpan =
5581 Builder.CreateAdd(LHS: ScanRedInfo->Span, RHS: Builder.getInt32(C: 1));
5582 for (size_t i = 0; i < ScanVars.size(); i++) {
5583 Type *IntPtrTy = Builder.getInt32Ty();
5584 Value *Allocsize = Builder.CreateTypeSize(
5585 Ty: IntPtrTy, Size: M.getDataLayout().getTypeAllocSize(Ty: ScanVarsType[i]));
5586 Value *Buff =
5587 Builder.CreateMalloc(IntPtrTy, AllocSize: Allocsize, ArraySize: AllocSpan, MallocF: nullptr, Name: "arr");
5588 Builder.CreateStore(Val: Buff, Ptr: (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]]);
5589 }
5590 return Error::success();
5591 };
5592 // TODO: Perform finalization actions for variables. This has to be
5593 // called for variables which have destructors/finalizers.
5594 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5595
5596 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit->getTerminator());
5597 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5598 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5599 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5600
5601 if (!AfterIP)
5602 return AfterIP.takeError();
5603 Builder.restoreIP(IP: *AfterIP);
5604 BasicBlock *InputBB = Builder.GetInsertBlock();
5605 if (InputBB->hasTerminator())
5606 Builder.SetInsertPoint(InputBB->getTerminator());
5607 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5608 if (!AfterIP)
5609 return AfterIP.takeError();
5610 Builder.restoreIP(IP: *AfterIP);
5611
5612 return Error::success();
5613}
5614
5615Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
5616 ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
5617 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5618 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5619 Builder.restoreIP(IP: CodeGenIP);
5620 for (ReductionInfo RedInfo : ReductionInfos) {
5621 Value *PrivateVar = RedInfo.PrivateVariable;
5622 Value *OrigVar = RedInfo.Variable;
5623 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[PrivateVar];
5624 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5625
5626 Type *SrcTy = RedInfo.ElementType;
5627 Value *Val = Builder.CreateInBoundsGEP(Ty: SrcTy, Ptr: Buff, IdxList: ScanRedInfo->Span,
5628 Name: "arrayOffset");
5629 Value *Src = Builder.CreateLoad(Ty: SrcTy, Ptr: Val);
5630
5631 Builder.CreateStore(Val: Src, Ptr: OrigVar);
5632 Builder.CreateFree(Source: Buff);
5633 }
5634 return Error::success();
5635 };
5636 // TODO: Perform finalization actions for variables. This has to be
5637 // called for variables which have destructors/finalizers.
5638 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5639
5640 if (Instruction *TI = ScanRedInfo->OMPScanFinish->getTerminatorOrNull())
5641 Builder.SetInsertPoint(TI);
5642 else
5643 Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
5644
5645 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5646 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5647 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5648
5649 if (!AfterIP)
5650 return AfterIP.takeError();
5651 Builder.restoreIP(IP: *AfterIP);
5652 BasicBlock *InputBB = Builder.GetInsertBlock();
5653 if (InputBB->hasTerminator())
5654 Builder.SetInsertPoint(InputBB->getTerminator());
5655 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5656 if (!AfterIP)
5657 return AfterIP.takeError();
5658 Builder.restoreIP(IP: *AfterIP);
5659 return Error::success();
5660}
5661
5662OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
5663 const LocationDescription &Loc,
5664 ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
5665 ScanInfo *ScanRedInfo) {
5666
5667 if (!updateToLocation(Loc))
5668 return Loc.IP;
5669 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5670 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
5671 Builder.restoreIP(IP: CodeGenIP);
5672 Function *CurFn = Builder.GetInsertBlock()->getParent();
5673 // for (int k = 0; k <= ceil(log2(n)); ++k)
5674 llvm::BasicBlock *LoopBB =
5675 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.outer.log.scan.body");
5676 llvm::BasicBlock *ExitBB =
5677 splitBB(Builder, CreateBranch: false, Name: "omp.outer.log.scan.exit");
5678 llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
5679 M: Builder.getModule(), id: (llvm::Intrinsic::ID)llvm::Intrinsic::log2,
5680 OverloadTys: Builder.getDoubleTy());
5681 llvm::BasicBlock *InputBB = Builder.GetInsertBlock();
5682 llvm::Value *Arg =
5683 Builder.CreateUIToFP(V: ScanRedInfo->Span, DestTy: Builder.getDoubleTy());
5684 llvm::Value *LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: Arg, Name: "");
5685 F = llvm::Intrinsic::getOrInsertDeclaration(
5686 M: Builder.getModule(), id: (llvm::Intrinsic::ID)llvm::Intrinsic::ceil,
5687 OverloadTys: Builder.getDoubleTy());
5688 LogVal = emitNoUnwindRuntimeCall(Builder, Callee: F, Args: LogVal, Name: "");
5689 LogVal = Builder.CreateFPToUI(V: LogVal, DestTy: Builder.getInt32Ty());
5690 llvm::Value *NMin1 = Builder.CreateNUWSub(
5691 LHS: ScanRedInfo->Span,
5692 RHS: llvm::ConstantInt::get(Ty: ScanRedInfo->Span->getType(), V: 1));
5693 Builder.SetInsertPoint(InputBB);
5694 Builder.CreateBr(Dest: LoopBB);
5695 emitBlock(BB: LoopBB, CurFn);
5696 Builder.SetInsertPoint(LoopBB);
5697
5698 PHINode *Counter = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5699 // size pow2k = 1;
5700 PHINode *Pow2K = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5701 Counter->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
5702 BB: InputBB);
5703 Pow2K->addIncoming(V: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1),
5704 BB: InputBB);
5705 // for (size i = n - 1; i >= 2 ^ k; --i)
5706 // tmp[i] op= tmp[i-pow2k];
5707 llvm::BasicBlock *InnerLoopBB =
5708 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.body");
5709 llvm::BasicBlock *InnerExitBB =
5710 BasicBlock::Create(Context&: CurFn->getContext(), Name: "omp.inner.log.scan.exit");
5711 llvm::Value *CmpI = Builder.CreateICmpUGE(LHS: NMin1, RHS: Pow2K);
5712 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5713 emitBlock(BB: InnerLoopBB, CurFn);
5714 Builder.SetInsertPoint(InnerLoopBB);
5715 PHINode *IVal = Builder.CreatePHI(Ty: Builder.getInt32Ty(), NumReservedValues: 2);
5716 IVal->addIncoming(V: NMin1, BB: LoopBB);
5717 for (ReductionInfo RedInfo : ReductionInfos) {
5718 Value *ReductionVal = RedInfo.PrivateVariable;
5719 Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
5720 Value *Buff = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: BuffPtr);
5721 Type *DestTy = RedInfo.ElementType;
5722 Value *IV = Builder.CreateAdd(LHS: IVal, RHS: Builder.getInt32(C: 1));
5723 Value *LHSPtr =
5724 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: IV, Name: "arrayOffset");
5725 Value *OffsetIval = Builder.CreateNUWSub(LHS: IV, RHS: Pow2K);
5726 Value *RHSPtr =
5727 Builder.CreateInBoundsGEP(Ty: DestTy, Ptr: Buff, IdxList: OffsetIval, Name: "arrayOffset");
5728 Value *LHS = Builder.CreateLoad(Ty: DestTy, Ptr: LHSPtr);
5729 Value *RHS = Builder.CreateLoad(Ty: DestTy, Ptr: RHSPtr);
5730 llvm::Value *Result;
5731 InsertPointOrErrorTy AfterIP =
5732 RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
5733 if (!AfterIP)
5734 return AfterIP.takeError();
5735 Builder.CreateStore(Val: Result, Ptr: LHSPtr);
5736 }
5737 llvm::Value *NextIVal = Builder.CreateNUWSub(
5738 LHS: IVal, RHS: llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 1));
5739 IVal->addIncoming(V: NextIVal, BB: Builder.GetInsertBlock());
5740 CmpI = Builder.CreateICmpUGE(LHS: NextIVal, RHS: Pow2K);
5741 Builder.CreateCondBr(Cond: CmpI, True: InnerLoopBB, False: InnerExitBB);
5742 emitBlock(BB: InnerExitBB, CurFn);
5743 llvm::Value *Next = Builder.CreateNUWAdd(
5744 LHS: Counter, RHS: llvm::ConstantInt::get(Ty: Counter->getType(), V: 1));
5745 Counter->addIncoming(V: Next, BB: Builder.GetInsertBlock());
5746 // pow2k <<= 1;
5747 llvm::Value *NextPow2K = Builder.CreateShl(LHS: Pow2K, RHS: 1, Name: "", /*HasNUW=*/true);
5748 Pow2K->addIncoming(V: NextPow2K, BB: Builder.GetInsertBlock());
5749 llvm::Value *Cmp = Builder.CreateICmpNE(LHS: Next, RHS: LogVal);
5750 Builder.CreateCondBr(Cond: Cmp, True: LoopBB, False: ExitBB);
5751 Builder.SetInsertPoint(ExitBB->getFirstInsertionPt());
5752 return Error::success();
5753 };
5754
5755 // TODO: Perform finalization actions for variables. This has to be
5756 // called for variables which have destructors/finalizers.
5757 auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
5758
5759 llvm::Value *FilterVal = Builder.getInt32(C: 0);
5760 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
5761 createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter: FilterVal);
5762
5763 if (!AfterIP)
5764 return AfterIP.takeError();
5765 Builder.restoreIP(IP: *AfterIP);
5766 AfterIP = createBarrier(Loc: Builder, Kind: llvm::omp::OMPD_barrier);
5767
5768 if (!AfterIP)
5769 return AfterIP.takeError();
5770 Builder.restoreIP(IP: *AfterIP);
5771 Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
5772 if (Err)
5773 return Err;
5774
5775 return AfterIP;
5776}
5777
5778Error OpenMPIRBuilder::emitScanBasedDirectiveIR(
5779 llvm::function_ref<Error()> InputLoopGen,
5780 llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,
5781 ScanInfo *ScanRedInfo) {
5782
5783 {
5784 // Emit loop with input phase:
5785 // for (i: 0..<num_iters>) {
5786 // <input phase>;
5787 // buffer[i] = red;
5788 // }
5789 ScanRedInfo->OMPFirstScanLoop = true;
5790 Error Err = InputLoopGen();
5791 if (Err)
5792 return Err;
5793 }
5794 {
5795 // Emit loop with scan phase:
5796 // for (i: 0..<num_iters>) {
5797 // red = buffer[i];
5798 // <scan phase>;
5799 // }
5800 ScanRedInfo->OMPFirstScanLoop = false;
5801 Error Err = ScanLoopGen(Builder);
5802 if (Err)
5803 return Err;
5804 }
5805 return Error::success();
5806}
5807
5808void OpenMPIRBuilder::createScanBBs(ScanInfo *ScanRedInfo) {
5809 Function *Fun = Builder.GetInsertBlock()->getParent();
5810 ScanRedInfo->OMPScanDispatch =
5811 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.inscan.dispatch");
5812 ScanRedInfo->OMPAfterScanBlock =
5813 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.after.scan.bb");
5814 ScanRedInfo->OMPBeforeScanBlock =
5815 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.before.scan.bb");
5816 ScanRedInfo->OMPScanLoopExit =
5817 BasicBlock::Create(Context&: Fun->getContext(), Name: "omp.scan.loop.exit");
5818}
5819CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
5820 DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
5821 BasicBlock *PostInsertBefore, const Twine &Name, bool IsCollapsed) {
5822 Module *M = F->getParent();
5823 LLVMContext &Ctx = M->getContext();
5824 Type *IndVarTy = TripCount->getType();
5825
5826 // Create the basic block structure.
5827 BasicBlock *Preheader =
5828 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".preheader", Parent: F, InsertBefore: PreInsertBefore);
5829 BasicBlock *Header =
5830 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".header", Parent: F, InsertBefore: PreInsertBefore);
5831 BasicBlock *Cond =
5832 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".cond", Parent: F, InsertBefore: PreInsertBefore);
5833 BasicBlock *Body =
5834 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".body", Parent: F, InsertBefore: PreInsertBefore);
5835 BasicBlock *Latch =
5836 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".inc", Parent: F, InsertBefore: PostInsertBefore);
5837 BasicBlock *Exit =
5838 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".exit", Parent: F, InsertBefore: PostInsertBefore);
5839 BasicBlock *After =
5840 BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".after", Parent: F, InsertBefore: PostInsertBefore);
5841
5842 // Use specified DebugLoc for new instructions.
5843 Builder.SetCurrentDebugLocation(DL);
5844
5845 Builder.SetInsertPoint(Preheader);
5846 Builder.CreateBr(Dest: Header);
5847
5848 Builder.SetInsertPoint(Header);
5849 PHINode *IndVarPHI = Builder.CreatePHI(Ty: IndVarTy, NumReservedValues: 2, Name: "omp_" + Name + ".iv");
5850 IndVarPHI->addIncoming(V: ConstantInt::get(Ty: IndVarTy, V: 0), BB: Preheader);
5851 Builder.CreateBr(Dest: Cond);
5852
5853 Builder.SetInsertPoint(Cond);
5854 Value *Cmp =
5855 Builder.CreateICmpULT(LHS: IndVarPHI, RHS: TripCount, Name: "omp_" + Name + ".cmp");
5856 Builder.CreateCondBr(Cond: Cmp, True: Body, False: Exit);
5857
5858 Builder.SetInsertPoint(Body);
5859 Builder.CreateBr(Dest: Latch);
5860
5861 Builder.SetInsertPoint(Latch);
5862 // Decide whether the induction variable increment can carry nsw.
5863 //
5864 // Single loops: nsw is always kept (matching Clang). Any Fortran program
5865 // whose trip count overflows i32 is non-conforming per F2018 11.1.7.4.1, so
5866 // for valid programs 0 <= count <= INT_MAX always holds.
5867 //
5868 // Collapsed loops: the trip count is a product that can overflow i32 even for
5869 // a conforming program, so nsw is kept only when the product is a constant
5870 // that provably fits, dropped otherwise.
5871 bool HasNSW = Config.hasNoSignedWrap();
5872 if (HasNSW) {
5873 if (auto *CI = dyn_cast<ConstantInt>(Val: TripCount)) {
5874 unsigned BitWidth = CI->getType()->getIntegerBitWidth();
5875 APInt SignedMax = APInt::getSignedMaxValue(numBits: BitWidth);
5876 if (CI->getValue().ugt(RHS: SignedMax))
5877 HasNSW = false;
5878 } else if (IsCollapsed) {
5879 HasNSW = false;
5880 }
5881 }
5882 Value *Next =
5883 Builder.CreateAdd(LHS: IndVarPHI, RHS: ConstantInt::get(Ty: IndVarTy, V: 1),
5884 Name: "omp_" + Name + ".next", /*HasNUW=*/true, HasNSW);
5885 Builder.CreateBr(Dest: Header);
5886 IndVarPHI->addIncoming(V: Next, BB: Latch);
5887
5888 Builder.SetInsertPoint(Exit);
5889 Builder.CreateBr(Dest: After);
5890
5891 // Remember and return the canonical control flow.
5892 LoopInfos.emplace_front();
5893 CanonicalLoopInfo *CL = &LoopInfos.front();
5894
5895 CL->Header = Header;
5896 CL->Cond = Cond;
5897 CL->Latch = Latch;
5898 CL->Exit = Exit;
5899
5900#ifndef NDEBUG
5901 CL->assertOK();
5902#endif
5903 return CL;
5904}
5905
5906Expected<CanonicalLoopInfo *>
5907OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
5908 LoopBodyGenCallbackTy BodyGenCB,
5909 Value *TripCount, const Twine &Name) {
5910 BasicBlock *BB = Loc.IP.getNodeParent();
5911 BasicBlock *NextBB = BB->getNextNode();
5912
5913 CanonicalLoopInfo *CL = createLoopSkeleton(DL: Loc.DL, TripCount, F: BB->getParent(),
5914 PreInsertBefore: NextBB, PostInsertBefore: NextBB, Name);
5915 BasicBlock *After = CL->getAfter();
5916
5917 // If location is not set, don't connect the loop.
5918 if (updateToLocation(Loc)) {
5919 // Split the loop at the insertion point: Branch to the preheader and move
5920 // every following instruction to after the loop (the After BB). Also, the
5921 // new successor is the loop's after block.
5922 spliceBB(Builder, New: After, /*CreateBranch=*/false);
5923 Builder.CreateBr(Dest: CL->getPreheader());
5924 }
5925
5926 // Emit the body content. We do it after connecting the loop to the CFG to
5927 // avoid that the callback encounters degenerate BBs.
5928 if (Error Err = BodyGenCB(CL->getBodyIP(), CL->getIndVar()))
5929 return Err;
5930
5931#ifndef NDEBUG
5932 CL->assertOK();
5933#endif
5934 return CL;
5935}
5936
5937Expected<ScanInfo *> OpenMPIRBuilder::scanInfoInitialize() {
5938 ScanInfos.emplace_front();
5939 ScanInfo *Result = &ScanInfos.front();
5940 return Result;
5941}
5942
5943Expected<SmallVector<llvm::CanonicalLoopInfo *>>
5944OpenMPIRBuilder::createCanonicalScanLoops(
5945 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
5946 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
5947 InsertPointTy ComputeIP, const Twine &Name, ScanInfo *ScanRedInfo) {
5948 LocationDescription ComputeLoc =
5949 ComputeIP.isValid() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
5950 updateToLocation(Loc: ComputeLoc);
5951
5952 SmallVector<CanonicalLoopInfo *> Result;
5953
5954 Value *TripCount = calculateCanonicalLoopTripCount(
5955 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
5956 ScanRedInfo->Span = TripCount;
5957 ScanRedInfo->OMPScanInit = splitBB(Builder, CreateBranch: true, Name: "scan.init");
5958 Builder.SetInsertPoint(ScanRedInfo->OMPScanInit);
5959
5960 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
5961 Builder.restoreIP(IP: CodeGenIP);
5962 ScanRedInfo->IV = IV;
5963 createScanBBs(ScanRedInfo);
5964 BasicBlock *InputBlock = Builder.GetInsertBlock();
5965 Instruction *Terminator = InputBlock->getTerminator();
5966 assert(Terminator->getNumSuccessors() == 1);
5967 BasicBlock *ContinueBlock = Terminator->getSuccessor(Idx: 0);
5968 Terminator->setSuccessor(Idx: 0, BB: ScanRedInfo->OMPScanDispatch);
5969 emitBlock(BB: ScanRedInfo->OMPBeforeScanBlock,
5970 CurFn: Builder.GetInsertBlock()->getParent());
5971 Builder.CreateBr(Dest: ScanRedInfo->OMPScanLoopExit);
5972 emitBlock(BB: ScanRedInfo->OMPScanLoopExit,
5973 CurFn: Builder.GetInsertBlock()->getParent());
5974 Builder.CreateBr(Dest: ContinueBlock);
5975 Builder.SetInsertPoint(
5976 ScanRedInfo->OMPBeforeScanBlock->getFirstInsertionPt());
5977 return BodyGenCB(Builder.saveIP(), IV);
5978 };
5979
5980 const auto &&InputLoopGen = [&]() -> Error {
5981 Expected<CanonicalLoopInfo *> LoopInfo =
5982 createCanonicalLoop(Loc: Builder, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5983 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5984 if (!LoopInfo)
5985 return LoopInfo.takeError();
5986 Result.push_back(Elt: *LoopInfo);
5987 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5988 return Error::success();
5989 };
5990 const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
5991 Expected<CanonicalLoopInfo *> LoopInfo =
5992 createCanonicalLoop(Loc, BodyGenCB: BodyGen, Start, Stop, Step, IsSigned,
5993 InclusiveStop, ComputeIP, Name, InScan: true, ScanRedInfo);
5994 if (!LoopInfo)
5995 return LoopInfo.takeError();
5996 Result.push_back(Elt: *LoopInfo);
5997 Builder.restoreIP(IP: (*LoopInfo)->getAfterIP());
5998 ScanRedInfo->OMPScanFinish = Builder.GetInsertBlock();
5999 return Error::success();
6000 };
6001 Error Err = emitScanBasedDirectiveIR(InputLoopGen, ScanLoopGen, ScanRedInfo);
6002 if (Err)
6003 return Err;
6004 return Result;
6005}
6006
6007Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(
6008 const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,
6009 bool IsSigned, bool InclusiveStop, const Twine &Name) {
6010
6011 // Consider the following difficulties (assuming 8-bit signed integers):
6012 // * Adding \p Step to the loop counter which passes \p Stop may overflow:
6013 // DO I = 1, 100, 50
6014 /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
6015 // DO I = 100, 0, -128
6016
6017 // Start, Stop and Step must be of the same integer type.
6018 auto *IndVarTy = cast<IntegerType>(Val: Start->getType());
6019 assert(IndVarTy == Stop->getType() && "Stop type mismatch");
6020 assert(IndVarTy == Step->getType() && "Step type mismatch");
6021
6022 updateToLocation(Loc);
6023
6024 ConstantInt *Zero = ConstantInt::get(Ty: IndVarTy, V: 0);
6025 ConstantInt *One = ConstantInt::get(Ty: IndVarTy, V: 1);
6026
6027 // Like Step, but always positive.
6028 Value *Incr = Step;
6029
6030 // Distance between Start and Stop; always positive.
6031 Value *Span;
6032
6033 // Condition whether there are no iterations are executed at all, e.g. because
6034 // UB < LB.
6035 Value *ZeroCmp;
6036
6037 if (IsSigned) {
6038 // Ensure that increment is positive. If not, negate and invert LB and UB.
6039 Value *IsNeg = Builder.CreateICmpSLT(LHS: Step, RHS: Zero);
6040 Incr = Builder.CreateSelect(C: IsNeg, True: Builder.CreateNeg(V: Step), False: Step);
6041 Value *LB = Builder.CreateSelect(C: IsNeg, True: Stop, False: Start);
6042 Value *UB = Builder.CreateSelect(C: IsNeg, True: Start, False: Stop);
6043 Span = Builder.CreateSub(LHS: UB, RHS: LB, Name: "", HasNUW: false, HasNSW: true);
6044 ZeroCmp = Builder.CreateICmp(
6045 P: InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, LHS: UB, RHS: LB);
6046 } else {
6047 Span = Builder.CreateSub(LHS: Stop, RHS: Start, Name: "", HasNUW: true);
6048 ZeroCmp = Builder.CreateICmp(
6049 P: InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, LHS: Stop, RHS: Start);
6050 }
6051
6052 Value *CountIfLooping;
6053 if (InclusiveStop) {
6054 CountIfLooping = Builder.CreateAdd(LHS: Builder.CreateUDiv(LHS: Span, RHS: Incr), RHS: One);
6055 } else {
6056 // Avoid incrementing past stop since it could overflow.
6057 Value *CountIfTwo = Builder.CreateAdd(
6058 LHS: Builder.CreateUDiv(LHS: Builder.CreateSub(LHS: Span, RHS: One), RHS: Incr), RHS: One);
6059 Value *OneCmp = Builder.CreateICmp(P: CmpInst::ICMP_ULE, LHS: Span, RHS: Incr);
6060 CountIfLooping = Builder.CreateSelect(C: OneCmp, True: One, False: CountIfTwo);
6061 }
6062
6063 return Builder.CreateSelect(C: ZeroCmp, True: Zero, False: CountIfLooping,
6064 Name: "omp_" + Name + ".tripcount");
6065}
6066
6067Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
6068 const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
6069 Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
6070 InsertPointTy ComputeIP, const Twine &Name, bool InScan,
6071 ScanInfo *ScanRedInfo) {
6072 LocationDescription ComputeLoc =
6073 ComputeIP.isValid() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
6074
6075 Value *TripCount = calculateCanonicalLoopTripCount(
6076 Loc: ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
6077
6078 auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
6079 Builder.restoreIP(IP: CodeGenIP);
6080 Value *Span = Builder.CreateMul(LHS: IV, RHS: Step, Name: "", /*HasNUW=*/false,
6081 /*HasNSW=*/Config.hasNoSignedWrap());
6082 Value *IndVar = Builder.CreateAdd(LHS: Span, RHS: Start, Name: "", /*HasNUW=*/false,
6083 /*HasNSW=*/Config.hasNoSignedWrap());
6084 if (InScan)
6085 ScanRedInfo->IV = IndVar;
6086 return BodyGenCB(Builder.saveIP(), IndVar);
6087 };
6088 LocationDescription LoopLoc =
6089 ComputeIP.isValid()
6090 ? Loc
6091 : LocationDescription(Builder.saveIP(),
6092 Builder.getCurrentDebugLocation());
6093 return createCanonicalLoop(Loc: LoopLoc, BodyGenCB: BodyGen, TripCount, Name);
6094}
6095
6096// Returns an LLVM function to call for initializing loop bounds using OpenMP
6097// static scheduling for composite `distribute parallel for` depending on
6098// `type`. Only i32 and i64 are supported by the runtime. Always interpret
6099// integers as unsigned similarly to CanonicalLoopInfo.
6100static FunctionCallee
6101getKmpcDistForStaticInitForType(Type *Ty, Module &M,
6102 OpenMPIRBuilder &OMPBuilder) {
6103 unsigned Bitwidth = Ty->getIntegerBitWidth();
6104 if (Bitwidth == 32)
6105 return OMPBuilder.getOrCreateRuntimeFunction(
6106 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_4u);
6107 if (Bitwidth == 64)
6108 return OMPBuilder.getOrCreateRuntimeFunction(
6109 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dist_for_static_init_8u);
6110 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6111}
6112
6113// Returns an LLVM function to call for initializing loop bounds using OpenMP
6114// static scheduling depending on `type`. Only i32 and i64 are supported by the
6115// runtime. Always interpret integers as unsigned similarly to
6116// CanonicalLoopInfo.
6117static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
6118 OpenMPIRBuilder &OMPBuilder) {
6119 unsigned Bitwidth = Ty->getIntegerBitWidth();
6120 if (Bitwidth == 32)
6121 return OMPBuilder.getOrCreateRuntimeFunction(
6122 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
6123 if (Bitwidth == 64)
6124 return OMPBuilder.getOrCreateRuntimeFunction(
6125 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
6126 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6127}
6128
6129OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyStaticWorkshareLoop(
6130 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6131 WorksharingLoopType LoopType, bool NeedsBarrier, bool HasDistSchedule,
6132 OMPScheduleType DistScheduleSchedType) {
6133 assert(CLI->isValid() && "Requires a valid canonical loop");
6134 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6135 "Require dedicated allocate IP");
6136
6137 // Set up the source location value for OpenMP runtime.
6138 Builder.restoreIP(IP: CLI->getPreheaderIP());
6139 Builder.SetCurrentDebugLocation(DL);
6140
6141 uint32_t SrcLocStrSize;
6142 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6143 IdentFlag Flag = IdentFlag(0);
6144 switch (LoopType) {
6145 case WorksharingLoopType::ForStaticLoop:
6146 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6147 break;
6148 case WorksharingLoopType::DistributeStaticLoop:
6149 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6150 break;
6151 case WorksharingLoopType::DistributeForStaticLoop:
6152 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6153 break;
6154 }
6155 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6156
6157 // Declare useful OpenMP runtime functions.
6158 Value *IV = CLI->getIndVar();
6159 Type *IVTy = IV->getType();
6160 FunctionCallee StaticInit =
6161 LoopType == WorksharingLoopType::DistributeForStaticLoop
6162 ? getKmpcDistForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this)
6163 : getKmpcForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6164 FunctionCallee StaticFini =
6165 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6166
6167 // Allocate space for computed loop bounds as expected by the "init" function.
6168 Builder.SetInsertPoint(AllocaIP.getNodeParent()->getFirstNonPHIOrAlloca());
6169
6170 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6171 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6172 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6173 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6174 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6175 CLI->setLastIter(PLastIter);
6176
6177 // At the end of the preheader, prepare for calling the "init" function by
6178 // storing the current loop bounds into the allocated space. A canonical loop
6179 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6180 // and produces an inclusive upper bound.
6181 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6182 Constant *Zero = ConstantInt::get(Ty: IVTy, V: 0);
6183 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6184 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6185 Value *UpperBound = Builder.CreateSub(LHS: CLI->getTripCount(), RHS: One);
6186 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6187 Builder.CreateStore(Val: One, Ptr: PStride);
6188
6189 Value *ThreadNum =
6190 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6191
6192 OMPScheduleType SchedType =
6193 (LoopType == WorksharingLoopType::DistributeStaticLoop)
6194 ? OMPScheduleType::OrderedDistribute
6195 : OMPScheduleType::UnorderedStatic;
6196 Constant *SchedulingType =
6197 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6198
6199 // Call the "init" function and update the trip count of the loop with the
6200 // value it produced.
6201 auto BuildInitCall = [LoopType, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6202 PUpperBound, IVTy, PStride, One, Zero, StaticInit,
6203 this](Value *SchedulingType, auto &Builder) {
6204 SmallVector<Value *, 10> Args({SrcLoc, ThreadNum, SchedulingType, PLastIter,
6205 PLowerBound, PUpperBound});
6206 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6207 Value *PDistUpperBound =
6208 Builder.CreateAlloca(IVTy, nullptr, "p.distupperbound");
6209 Args.push_back(Elt: PDistUpperBound);
6210 }
6211 Args.append(IL: {PStride, One, Zero});
6212 createRuntimeFunctionCall(Callee: StaticInit, Args);
6213 };
6214 BuildInitCall(SchedulingType, Builder);
6215 if (HasDistSchedule &&
6216 LoopType != WorksharingLoopType::DistributeStaticLoop) {
6217 Constant *DistScheduleSchedType = ConstantInt::get(
6218 Ty: I32Type, V: static_cast<int>(omp::OMPScheduleType::OrderedDistribute));
6219 // We want to emit a second init function call for the dist_schedule clause
6220 // to the Distribute construct. This should only be done however if a
6221 // Workshare Loop is nested within a Distribute Construct
6222 BuildInitCall(DistScheduleSchedType, Builder);
6223 }
6224 Value *LowerBound = Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound);
6225 Value *InclusiveUpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound);
6226 Value *TripCountMinusOne = Builder.CreateSub(LHS: InclusiveUpperBound, RHS: LowerBound);
6227 Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One);
6228 CLI->setTripCount(TripCount);
6229
6230 // Update all uses of the induction variable except the one in the condition
6231 // block that compares it with the actual upper bound, and the increment in
6232 // the latch block.
6233
6234 CLI->mapIndVar(Updater: [&](Instruction *OldIV) -> Value * {
6235 Builder.SetInsertPoint(CLI->getBody()->getFirstInsertionPt());
6236 Builder.SetCurrentDebugLocation(DL);
6237 return Builder.CreateAdd(LHS: OldIV, RHS: LowerBound, Name: "", /*HasNUW=*/false,
6238 /*HasNSW=*/Config.hasNoSignedWrap());
6239 });
6240
6241 // In the "exit" block, call the "fini" function.
6242 Builder.SetInsertPoint(CLI->getExit()->getTerminator()->getIterator());
6243 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6244
6245 // Add the barrier if requested.
6246 if (NeedsBarrier) {
6247 InsertPointOrErrorTy BarrierIP =
6248 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
6249 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
6250 /* CheckCancelFlag */ false);
6251 if (!BarrierIP)
6252 return BarrierIP.takeError();
6253 }
6254
6255 InsertPointTy AfterIP = CLI->getAfterIP();
6256 CLI->invalidate();
6257
6258 return AfterIP;
6259}
6260
6261static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
6262 LoopInfo &LI);
6263static void addLoopMetadata(CanonicalLoopInfo *Loop,
6264 ArrayRef<Metadata *> Properties);
6265
6266static void applyParallelAccessesMetadata(CanonicalLoopInfo *CLI,
6267 LLVMContext &Ctx, Loop *Loop,
6268 LoopInfo &LoopInfo,
6269 SmallVector<Metadata *> &LoopMDList) {
6270 SmallSet<BasicBlock *, 8> Reachable;
6271
6272 // Get the basic blocks from the loop in which memref instructions
6273 // can be found.
6274 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
6275 // preferably without running any passes.
6276 for (BasicBlock *Block : Loop->getBlocks()) {
6277 if (Block == CLI->getCond() || Block == CLI->getHeader())
6278 continue;
6279 Reachable.insert(Ptr: Block);
6280 }
6281
6282 // Add access group metadata to memory-access instructions.
6283 MDNode *AccessGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
6284 for (BasicBlock *BB : Reachable)
6285 addAccessGroupMetadata(Block: BB, AccessGroup, LI&: LoopInfo);
6286 // TODO: If the loop has existing parallel access metadata, have
6287 // to combine two lists.
6288 LoopMDList.push_back(Elt: MDNode::get(
6289 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccessGroup}));
6290}
6291
6292OpenMPIRBuilder::InsertPointOrErrorTy
6293OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
6294 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6295 bool NeedsBarrier, Value *ChunkSize, OMPScheduleType SchedType,
6296 Value *DistScheduleChunkSize, OMPScheduleType DistScheduleSchedType) {
6297 assert(CLI->isValid() && "Requires a valid canonical loop");
6298 assert((ChunkSize || DistScheduleChunkSize) && "Chunk size is required");
6299
6300 LLVMContext &Ctx = CLI->getFunction()->getContext();
6301 Value *IV = CLI->getIndVar();
6302 Value *OrigTripCount = CLI->getTripCount();
6303 Type *IVTy = IV->getType();
6304 assert(IVTy->getIntegerBitWidth() <= 64 &&
6305 "Max supported tripcount bitwidth is 64 bits");
6306 Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(C&: Ctx)
6307 : Type::getInt64Ty(C&: Ctx);
6308 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6309 Constant *Zero = ConstantInt::get(Ty: InternalIVTy, V: 0);
6310 Constant *One = ConstantInt::get(Ty: InternalIVTy, V: 1);
6311
6312 Function *F = CLI->getFunction();
6313 // Blocks must have terminators.
6314 // FIXME: Don't run analyses on incomplete/invalid IR.
6315 SmallVector<Instruction *> UIs;
6316 for (BasicBlock &BB : *F)
6317 if (!BB.hasTerminator())
6318 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
6319 FunctionAnalysisManager FAM;
6320 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
6321 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
6322 LoopAnalysis LIA;
6323 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
6324 for (Instruction *I : UIs)
6325 I->eraseFromParent();
6326 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
6327 SmallVector<Metadata *> LoopMDList;
6328 if (ChunkSize || DistScheduleChunkSize)
6329 applyParallelAccessesMetadata(CLI, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
6330 addLoopMetadata(Loop: CLI, Properties: LoopMDList);
6331
6332 // Declare useful OpenMP runtime functions.
6333 FunctionCallee StaticInit =
6334 getKmpcForStaticInitForType(Ty: InternalIVTy, M, OMPBuilder&: *this);
6335 FunctionCallee StaticFini =
6336 getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
6337
6338 // Allocate space for computed loop bounds as expected by the "init" function.
6339 Builder.restoreIP(IP: AllocaIP);
6340 Builder.SetCurrentDebugLocation(DL);
6341 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6342 Value *PLowerBound =
6343 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.lowerbound");
6344 Value *PUpperBound =
6345 Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.upperbound");
6346 Value *PStride = Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.stride");
6347 CLI->setLastIter(PLastIter);
6348
6349 // Set up the source location value for the OpenMP runtime.
6350 Builder.restoreIP(IP: CLI->getPreheaderIP());
6351 Builder.SetCurrentDebugLocation(DL);
6352
6353 // TODO: Detect overflow in ubsan or max-out with current tripcount.
6354 Value *CastedChunkSize = Builder.CreateZExtOrTrunc(
6355 V: ChunkSize ? ChunkSize : Zero, DestTy: InternalIVTy, Name: "chunksize");
6356 Value *CastedDistScheduleChunkSize = Builder.CreateZExtOrTrunc(
6357 V: DistScheduleChunkSize ? DistScheduleChunkSize : Zero, DestTy: InternalIVTy,
6358 Name: "distschedulechunksize");
6359 Value *CastedTripCount =
6360 Builder.CreateZExt(V: OrigTripCount, DestTy: InternalIVTy, Name: "tripcount");
6361
6362 Constant *SchedulingType =
6363 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6364 Constant *DistSchedulingType =
6365 ConstantInt::get(Ty: I32Type, V: static_cast<int>(DistScheduleSchedType));
6366 Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
6367 Value *OrigUpperBound = Builder.CreateSub(LHS: CastedTripCount, RHS: One);
6368 Value *IsTripCountZero = Builder.CreateICmpEQ(LHS: CastedTripCount, RHS: Zero);
6369 Value *UpperBound =
6370 Builder.CreateSelect(C: IsTripCountZero, True: Zero, False: OrigUpperBound);
6371 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6372 Builder.CreateStore(Val: One, Ptr: PStride);
6373
6374 // Call the "init" function and update the trip count of the loop with the
6375 // value it produced.
6376 uint32_t SrcLocStrSize;
6377 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6378 IdentFlag Flag = OMP_IDENT_FLAG_WORK_LOOP;
6379 if (DistScheduleSchedType != OMPScheduleType::None) {
6380 Flag |= OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6381 }
6382 Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6383 Value *ThreadNum =
6384 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6385 auto BuildInitCall = [StaticInit, SrcLoc, ThreadNum, PLastIter, PLowerBound,
6386 PUpperBound, PStride, One,
6387 this](Value *SchedulingType, Value *ChunkSize,
6388 auto &Builder) {
6389 createRuntimeFunctionCall(
6390 Callee: StaticInit, Args: {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
6391 /*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
6392 /*plower=*/PLowerBound, /*pupper=*/PUpperBound,
6393 /*pstride=*/PStride, /*incr=*/One,
6394 /*chunk=*/ChunkSize});
6395 };
6396 BuildInitCall(SchedulingType, CastedChunkSize, Builder);
6397 if (DistScheduleSchedType != OMPScheduleType::None &&
6398 SchedType != OMPScheduleType::OrderedDistributeChunked &&
6399 SchedType != OMPScheduleType::OrderedDistribute) {
6400 // We want to emit a second init function call for the dist_schedule clause
6401 // to the Distribute construct. This should only be done however if a
6402 // Workshare Loop is nested within a Distribute Construct
6403 BuildInitCall(DistSchedulingType, CastedDistScheduleChunkSize, Builder);
6404 }
6405
6406 // Load values written by the "init" function.
6407 Value *FirstChunkStart =
6408 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PLowerBound, Name: "omp_firstchunk.lb");
6409 Value *FirstChunkStop =
6410 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PUpperBound, Name: "omp_firstchunk.ub");
6411 Value *FirstChunkEnd = Builder.CreateAdd(LHS: FirstChunkStop, RHS: One);
6412 Value *ChunkRange =
6413 Builder.CreateSub(LHS: FirstChunkEnd, RHS: FirstChunkStart, Name: "omp_chunk.range");
6414 Value *NextChunkStride =
6415 Builder.CreateLoad(Ty: InternalIVTy, Ptr: PStride, Name: "omp_dispatch.stride");
6416
6417 // Create outer "dispatch" loop for enumerating the chunks.
6418 BasicBlock *DispatchEnter = splitBB(Builder, CreateBranch: true);
6419 Value *DispatchCounter;
6420
6421 // It is safe to assume this didn't return an error because the callback
6422 // passed into createCanonicalLoop is the only possible error source, and it
6423 // always returns success.
6424 CanonicalLoopInfo *DispatchCLI = cantFail(ValOrErr: createCanonicalLoop(
6425 Loc: {Builder.saveIP(), DL},
6426 BodyGenCB: [&](InsertPointTy BodyIP, Value *Counter) {
6427 DispatchCounter = Counter;
6428 return Error::success();
6429 },
6430 Start: FirstChunkStart, Stop: CastedTripCount, Step: NextChunkStride,
6431 /*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
6432 Name: "dispatch"));
6433
6434 // Remember the BasicBlocks of the dispatch loop we need, then invalidate to
6435 // not have to preserve the canonical invariant.
6436 BasicBlock *DispatchBody = DispatchCLI->getBody();
6437 BasicBlock *DispatchLatch = DispatchCLI->getLatch();
6438 BasicBlock *DispatchExit = DispatchCLI->getExit();
6439 BasicBlock *DispatchAfter = DispatchCLI->getAfter();
6440 DispatchCLI->invalidate();
6441
6442 // Rewire the original loop to become the chunk loop inside the dispatch loop.
6443 redirectTo(Source: DispatchAfter, Target: CLI->getAfter(), DL);
6444 redirectTo(Source: CLI->getExit(), Target: DispatchLatch, DL);
6445 redirectTo(Source: DispatchBody, Target: DispatchEnter, DL);
6446
6447 // Prepare the prolog of the chunk loop.
6448 Builder.restoreIP(IP: CLI->getPreheaderIP());
6449 Builder.SetCurrentDebugLocation(DL);
6450
6451 // Compute the number of iterations of the chunk loop.
6452 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6453 Value *ChunkEnd = Builder.CreateAdd(LHS: DispatchCounter, RHS: ChunkRange);
6454 Value *IsLastChunk =
6455 Builder.CreateICmpUGE(LHS: ChunkEnd, RHS: CastedTripCount, Name: "omp_chunk.is_last");
6456 Value *CountUntilOrigTripCount =
6457 Builder.CreateSub(LHS: CastedTripCount, RHS: DispatchCounter);
6458 Value *ChunkTripCount = Builder.CreateSelect(
6459 C: IsLastChunk, True: CountUntilOrigTripCount, False: ChunkRange, Name: "omp_chunk.tripcount");
6460 Value *BackcastedChunkTC =
6461 Builder.CreateTrunc(V: ChunkTripCount, DestTy: IVTy, Name: "omp_chunk.tripcount.trunc");
6462 CLI->setTripCount(BackcastedChunkTC);
6463
6464 // Update all uses of the induction variable except the one in the condition
6465 // block that compares it with the actual upper bound, and the increment in
6466 // the latch block.
6467 Value *BackcastedDispatchCounter =
6468 Builder.CreateTrunc(V: DispatchCounter, DestTy: IVTy, Name: "omp_dispatch.iv.trunc");
6469 CLI->mapIndVar(Updater: [&](Instruction *) -> Value * {
6470 Builder.restoreIP(IP: CLI->getBodyIP());
6471 return Builder.CreateAdd(LHS: IV, RHS: BackcastedDispatchCounter);
6472 });
6473
6474 // In the "exit" block, call the "fini" function.
6475 Builder.SetInsertPoint(DispatchExit->getFirstInsertionPt());
6476 createRuntimeFunctionCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
6477
6478 // Add the barrier if requested.
6479 if (NeedsBarrier) {
6480 InsertPointOrErrorTy AfterIP =
6481 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL), Kind: OMPD_for,
6482 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
6483 if (!AfterIP)
6484 return AfterIP.takeError();
6485 }
6486
6487#ifndef NDEBUG
6488 // Even though we currently do not support applying additional methods to it,
6489 // the chunk loop should remain a canonical loop.
6490 CLI->assertOK();
6491#endif
6492
6493 return DispatchAfter->getFirstInsertionPt();
6494}
6495
6496// Returns an LLVM function to call for executing an OpenMP static worksharing
6497// for loop depending on `type`. Only i32 and i64 are supported by the runtime.
6498// Always interpret integers as unsigned similarly to CanonicalLoopInfo.
6499static FunctionCallee
6500getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
6501 WorksharingLoopType LoopType) {
6502 unsigned Bitwidth = Ty->getIntegerBitWidth();
6503 Module &M = OMPBuilder->M;
6504 switch (LoopType) {
6505 case WorksharingLoopType::ForStaticLoop:
6506 if (Bitwidth == 32)
6507 return OMPBuilder->getOrCreateRuntimeFunction(
6508 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
6509 if (Bitwidth == 64)
6510 return OMPBuilder->getOrCreateRuntimeFunction(
6511 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
6512 break;
6513 case WorksharingLoopType::DistributeStaticLoop:
6514 if (Bitwidth == 32)
6515 return OMPBuilder->getOrCreateRuntimeFunction(
6516 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
6517 if (Bitwidth == 64)
6518 return OMPBuilder->getOrCreateRuntimeFunction(
6519 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
6520 break;
6521 case WorksharingLoopType::DistributeForStaticLoop:
6522 if (Bitwidth == 32)
6523 return OMPBuilder->getOrCreateRuntimeFunction(
6524 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
6525 if (Bitwidth == 64)
6526 return OMPBuilder->getOrCreateRuntimeFunction(
6527 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
6528 break;
6529 }
6530 if (Bitwidth != 32 && Bitwidth != 64) {
6531 llvm_unreachable("Unknown OpenMP loop iterator bitwidth");
6532 }
6533 llvm_unreachable("Unknown type of OpenMP worksharing loop");
6534}
6535
6536// Inserts a call to proper OpenMP Device RTL function which handles
6537// loop worksharing.
6538static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,
6539 WorksharingLoopType LoopType,
6540 BasicBlock *InsertBlock, Value *Ident,
6541 Value *LoopBodyArg, Value *TripCount,
6542 Function &LoopBodyFn, bool NoLoop) {
6543 Type *TripCountTy = TripCount->getType();
6544 Module &M = OMPBuilder->M;
6545 IRBuilder<> &Builder = OMPBuilder->Builder;
6546 FunctionCallee RTLFn =
6547 getKmpcForStaticLoopForType(Ty: TripCountTy, OMPBuilder, LoopType);
6548 SmallVector<Value *, 8> RealArgs;
6549 RealArgs.push_back(Elt: Ident);
6550 RealArgs.push_back(Elt: &LoopBodyFn);
6551 RealArgs.push_back(Elt: LoopBodyArg);
6552 RealArgs.push_back(Elt: TripCount);
6553 if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
6554 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6555 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6556 Builder.restoreIP(IP: std::prev(x: InsertBlock->end()));
6557 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6558 return;
6559 }
6560 FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(
6561 M, FnID: omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
6562 Builder.restoreIP(IP: std::prev(x: InsertBlock->end()));
6563 Value *NumThreads = OMPBuilder->createRuntimeFunctionCall(Callee: RTLNumThreads, Args: {});
6564
6565 RealArgs.push_back(
6566 Elt: Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: TripCountTy, Name: "num.threads.cast"));
6567 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6568 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
6569 RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
6570 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: NoLoop));
6571 } else {
6572 RealArgs.push_back(Elt: ConstantInt::get(Ty: Builder.getInt8Ty(), V: 0));
6573 }
6574
6575 OMPBuilder->createRuntimeFunctionCall(Callee: RTLFn, Args: RealArgs);
6576}
6577
6578static void workshareLoopTargetCallback(
6579 OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident,
6580 Function &OutlinedFn, const SmallVector<Instruction *, 4> &ToBeDeleted,
6581 WorksharingLoopType LoopType, bool NoLoop) {
6582 IRBuilder<> &Builder = OMPIRBuilder->Builder;
6583 BasicBlock *Preheader = CLI->getPreheader();
6584 Value *TripCount = CLI->getTripCount();
6585
6586 // After loop body outling, the loop body contains only set up
6587 // of loop body argument structure and the call to the outlined
6588 // loop body function. Firstly, we need to move setup of loop body args
6589 // into loop preheader.
6590 Preheader->splice(ToIt: std::prev(x: Preheader->end()), FromBB: CLI->getBody(),
6591 FromBeginIt: CLI->getBody()->begin(), FromEndIt: std::prev(x: CLI->getBody()->end()));
6592
6593 // The next step is to remove the whole loop. We do not it need anymore.
6594 // That's why make an unconditional branch from loop preheader to loop
6595 // exit block
6596 Builder.restoreIP(IP: Preheader->end());
6597 Builder.SetCurrentDebugLocation(Preheader->getTerminator()->getDebugLoc());
6598 Preheader->getTerminator()->eraseFromParent();
6599 Builder.CreateBr(Dest: CLI->getExit());
6600
6601 // Delete dead loop blocks
6602 OpenMPIRBuilder::OutlineInfo CleanUpInfo;
6603 SmallPtrSet<BasicBlock *, 32> RegionBlockSet;
6604 SmallVector<BasicBlock *, 32> BlocksToBeRemoved;
6605 CleanUpInfo.EntryBB = CLI->getHeader();
6606 CleanUpInfo.ExitBB = CLI->getExit();
6607 CleanUpInfo.collectBlocks(BlockSet&: RegionBlockSet, BlockVector&: BlocksToBeRemoved);
6608 DeleteDeadBlocks(BBs: BlocksToBeRemoved);
6609
6610 // Find the instruction which corresponds to loop body argument structure
6611 // and remove the call to loop body function instruction.
6612 Value *LoopBodyArg;
6613 User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();
6614 assert(OutlinedFnUser &&
6615 "Expected unique undroppable user of outlined function");
6616 CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(Val: OutlinedFnUser);
6617 assert(OutlinedFnCallInstruction && "Expected outlined function call");
6618 assert((OutlinedFnCallInstruction->getParent() == Preheader) &&
6619 "Expected outlined function call to be located in loop preheader");
6620 // Check in case no argument structure has been passed.
6621 if (OutlinedFnCallInstruction->arg_size() > 1)
6622 LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(i: 1);
6623 else
6624 LoopBodyArg = Constant::getNullValue(Ty: Builder.getPtrTy());
6625 OutlinedFnCallInstruction->eraseFromParent();
6626
6627 createTargetLoopWorkshareCall(OMPBuilder: OMPIRBuilder, LoopType, InsertBlock: Preheader, Ident,
6628 LoopBodyArg, TripCount, LoopBodyFn&: OutlinedFn, NoLoop);
6629
6630 for (auto &ToBeDeletedItem : ToBeDeleted)
6631 ToBeDeletedItem->eraseFromParent();
6632 CLI->invalidate();
6633}
6634
6635OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoopTarget(
6636 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6637 WorksharingLoopType LoopType, bool NeedsBarrier, bool NoLoop,
6638 bool NeedsLastIter) {
6639 uint32_t SrcLocStrSize;
6640 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6641
6642 // Mirrors host runtime reporting of last iteration by in-body computation.
6643 if (NeedsLastIter) {
6644 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6645 Builder.restoreIP(IP: AllocaIP);
6646 AllocaInst *PLastIter =
6647 Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6648 CLI->setLastIter(PLastIter);
6649
6650 Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
6651 Builder.CreateStore(Val: ConstantInt::get(Ty: I32Type, V: 0), Ptr: PLastIter);
6652
6653 Builder.SetInsertPoint(CLI->getBody()->getFirstInsertionPt());
6654 Value *TripCount = CLI->getTripCount();
6655 Value *LastIter =
6656 Builder.CreateSub(LHS: TripCount, RHS: ConstantInt::get(Ty: TripCount->getType(), V: 1));
6657 Value *IsLast =
6658 Builder.CreateICmpEQ(LHS: CLI->getIndVar(), RHS: LastIter, Name: "omp.is_last_iter");
6659 Builder.CreateStore(Val: Builder.CreateZExt(V: IsLast, DestTy: I32Type), Ptr: PLastIter);
6660 }
6661
6662 IdentFlag Flag = IdentFlag(0);
6663 switch (LoopType) {
6664 case WorksharingLoopType::ForStaticLoop:
6665 Flag = OMP_IDENT_FLAG_WORK_LOOP;
6666 break;
6667 case WorksharingLoopType::DistributeStaticLoop:
6668 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE;
6669 break;
6670 case WorksharingLoopType::DistributeForStaticLoop:
6671 Flag = OMP_IDENT_FLAG_WORK_DISTRIBUTE | OMP_IDENT_FLAG_WORK_LOOP;
6672 break;
6673 }
6674 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: Flag);
6675
6676 auto OI = std::make_unique<OutlineInfo>();
6677 OI->OuterAllocBB = CLI->getPreheader();
6678 Function *OuterFn = CLI->getPreheader()->getParent();
6679
6680 // Instructions which need to be deleted at the end of code generation
6681 SmallVector<Instruction *, 4> ToBeDeleted;
6682
6683 OI->OuterAllocBB = AllocaIP.getNodeParent();
6684
6685 // Mark the body loop as region which needs to be extracted
6686 OI->EntryBB = CLI->getBody();
6687 OI->ExitBB = CLI->getLatch()->splitBasicBlockBefore(I: CLI->getLatch()->begin(),
6688 BBName: "omp.prelatch");
6689
6690 // Prepare loop body for extraction
6691 Builder.restoreIP(IP: CLI->getPreheader()->begin());
6692
6693 // Insert new loop counter variable which will be used only in loop
6694 // body.
6695 AllocaInst *NewLoopCnt = Builder.CreateAlloca(Ty: CLI->getIndVarType(), ArraySize: 0, Name: "");
6696 Instruction *NewLoopCntLoad =
6697 Builder.CreateLoad(Ty: CLI->getIndVarType(), Ptr: NewLoopCnt);
6698 // New loop counter instructions are redundant in the loop preheader when
6699 // code generation for workshare loop is finshed. That's why mark them as
6700 // ready for deletion.
6701 ToBeDeleted.push_back(Elt: NewLoopCntLoad);
6702 ToBeDeleted.push_back(Elt: NewLoopCnt);
6703
6704 // Analyse loop body region. Find all input variables which are used inside
6705 // loop body region.
6706 SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
6707 SmallVector<BasicBlock *, 32> Blocks;
6708 OI->collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
6709
6710 CodeExtractorAnalysisCache CEAC(*OuterFn);
6711 CodeExtractor Extractor(Blocks,
6712 /* DominatorTree */ nullptr,
6713 /* AggregateArgs */ true,
6714 /* BlockFrequencyInfo */ nullptr,
6715 /* BranchProbabilityInfo */ nullptr,
6716 /* AssumptionCache */ nullptr,
6717 /* AllowVarArgs */ true,
6718 /* AllowAlloca */ true,
6719 /* AllocationBlock */ CLI->getPreheader(),
6720 /* DeallocationBlocks */ {},
6721 /* Suffix */ ".omp_wsloop",
6722 /* AggrArgsIn0AddrSpace */ true);
6723
6724 BasicBlock *CommonExit = nullptr;
6725 SetVector<Value *> SinkingCands, HoistingCands;
6726
6727 // Find allocas outside the loop body region which are used inside loop
6728 // body
6729 Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
6730
6731 // We need to model loop body region as the function f(cnt, loop_arg).
6732 // That's why we replace loop induction variable by the new counter
6733 // which will be one of loop body function argument
6734 SmallVector<User *> Users(CLI->getIndVar()->user_begin(),
6735 CLI->getIndVar()->user_end());
6736 for (auto Use : Users) {
6737 if (Instruction *Inst = dyn_cast<Instruction>(Val: Use)) {
6738 if (ParallelRegionBlockSet.count(Ptr: Inst->getParent())) {
6739 Inst->replaceUsesOfWith(From: CLI->getIndVar(), To: NewLoopCntLoad);
6740 }
6741 }
6742 }
6743 // Make sure that loop counter variable is not merged into loop body
6744 // function argument structure and it is passed as separate variable
6745 OI->ExcludeArgsFromAggregate.push_back(Elt: NewLoopCntLoad);
6746
6747 // PostOutline CB is invoked when loop body function is outlined and
6748 // loop body is replaced by call to outlined function. We need to add
6749 // call to OpenMP device rtl inside loop preheader. OpenMP device rtl
6750 // function will handle loop control logic.
6751 //
6752 OI->PostOutlineCB = [=, ToBeDeletedVec =
6753 std::move(ToBeDeleted)](Function &OutlinedFn) {
6754 workshareLoopTargetCallback(OMPIRBuilder: this, CLI, Ident, OutlinedFn, ToBeDeleted: ToBeDeletedVec,
6755 LoopType, NoLoop);
6756 };
6757 addOutlineInfo(OI: std::move(OI));
6758
6759 // Keep the barrier outside the outlined loop body so that every thread
6760 // encounters it, including threads that execute no iterations.
6761 if (NeedsBarrier) {
6762 Builder.SetInsertPoint(CLI->getExit()->getTerminator());
6763 // Standalone distribute loops never request a barrier. For both regular
6764 // worksharing loops and combined distribute/for loops, the barrier is
6765 // associated with the worksharing loop, hence OMPD_for.
6766 InsertPointOrErrorTy BarrierIP =
6767 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL), Kind: OMPD_for,
6768 /*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
6769 if (!BarrierIP)
6770 return BarrierIP.takeError();
6771 }
6772 return CLI->getAfterIP();
6773}
6774
6775OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::applyWorkshareLoop(
6776 DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
6777 bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
6778 bool HasSimdModifier, bool HasMonotonicModifier,
6779 bool HasNonmonotonicModifier, bool HasOrderedClause,
6780 WorksharingLoopType LoopType, bool NoLoop, bool HasDistSchedule,
6781 Value *DistScheduleChunkSize, bool NeedsLastIter) {
6782 if (Config.isTargetDevice())
6783 return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType, NeedsBarrier,
6784 NoLoop, NeedsLastIter);
6785 OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
6786 ClauseKind: SchedKind, HasChunks: ChunkSize, HasSimdModifier, HasMonotonicModifier,
6787 HasNonmonotonicModifier, HasOrderedClause, HasDistScheduleChunks: DistScheduleChunkSize);
6788
6789 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
6790 OMPScheduleType::ModifierOrdered;
6791 OMPScheduleType DistScheduleSchedType = OMPScheduleType::None;
6792 if (HasDistSchedule) {
6793 DistScheduleSchedType = DistScheduleChunkSize
6794 ? OMPScheduleType::OrderedDistributeChunked
6795 : OMPScheduleType::OrderedDistribute;
6796 }
6797 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
6798 case OMPScheduleType::BaseStatic:
6799 case OMPScheduleType::BaseDistribute:
6800 assert((!ChunkSize || !DistScheduleChunkSize) &&
6801 "No chunk size with static-chunked schedule");
6802 if (IsOrdered && !HasDistSchedule)
6803 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6804 NeedsBarrier, Chunk: ChunkSize);
6805 // FIXME: Monotonicity ignored?
6806 if (DistScheduleChunkSize)
6807 return applyStaticChunkedWorkshareLoop(
6808 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6809 DistScheduleChunkSize, DistScheduleSchedType);
6810 return applyStaticWorkshareLoop(DL, CLI, AllocaIP, LoopType, NeedsBarrier,
6811 HasDistSchedule);
6812
6813 case OMPScheduleType::BaseStaticChunked:
6814 case OMPScheduleType::BaseDistributeChunked:
6815 if (IsOrdered && !HasDistSchedule)
6816 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6817 NeedsBarrier, Chunk: ChunkSize);
6818 // FIXME: Monotonicity ignored?
6819 return applyStaticChunkedWorkshareLoop(
6820 DL, CLI, AllocaIP, NeedsBarrier, ChunkSize, SchedType: EffectiveScheduleType,
6821 DistScheduleChunkSize, DistScheduleSchedType);
6822
6823 case OMPScheduleType::BaseRuntime:
6824 case OMPScheduleType::BaseAuto:
6825 case OMPScheduleType::BaseGreedy:
6826 case OMPScheduleType::BaseBalanced:
6827 case OMPScheduleType::BaseSteal:
6828 case OMPScheduleType::BaseRuntimeSimd:
6829 assert(!ChunkSize &&
6830 "schedule type does not support user-defined chunk sizes");
6831 [[fallthrough]];
6832 case OMPScheduleType::BaseGuidedSimd:
6833 case OMPScheduleType::BaseDynamicChunked:
6834 case OMPScheduleType::BaseGuidedChunked:
6835 case OMPScheduleType::BaseGuidedIterativeChunked:
6836 case OMPScheduleType::BaseGuidedAnalyticalChunked:
6837 case OMPScheduleType::BaseStaticBalancedChunked:
6838 return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
6839 NeedsBarrier, Chunk: ChunkSize);
6840
6841 default:
6842 llvm_unreachable("Unknown/unimplemented schedule kind");
6843 }
6844}
6845
6846/// Returns an LLVM function to call for initializing loop bounds using OpenMP
6847/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6848/// the runtime. Always interpret integers as unsigned similarly to
6849/// CanonicalLoopInfo.
6850static FunctionCallee
6851getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6852 unsigned Bitwidth = Ty->getIntegerBitWidth();
6853 if (Bitwidth == 32)
6854 return OMPBuilder.getOrCreateRuntimeFunction(
6855 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
6856 if (Bitwidth == 64)
6857 return OMPBuilder.getOrCreateRuntimeFunction(
6858 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
6859 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6860}
6861
6862/// Returns an LLVM function to call for updating the next loop using OpenMP
6863/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
6864/// the runtime. Always interpret integers as unsigned similarly to
6865/// CanonicalLoopInfo.
6866static FunctionCallee
6867getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6868 unsigned Bitwidth = Ty->getIntegerBitWidth();
6869 if (Bitwidth == 32)
6870 return OMPBuilder.getOrCreateRuntimeFunction(
6871 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
6872 if (Bitwidth == 64)
6873 return OMPBuilder.getOrCreateRuntimeFunction(
6874 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
6875 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6876}
6877
6878/// Returns an LLVM function to call for finalizing the dynamic loop using
6879/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
6880/// interpret integers as unsigned similarly to CanonicalLoopInfo.
6881static FunctionCallee
6882getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
6883 unsigned Bitwidth = Ty->getIntegerBitWidth();
6884 if (Bitwidth == 32)
6885 return OMPBuilder.getOrCreateRuntimeFunction(
6886 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
6887 if (Bitwidth == 64)
6888 return OMPBuilder.getOrCreateRuntimeFunction(
6889 M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
6890 llvm_unreachable("unknown OpenMP loop iterator bitwidth");
6891}
6892
6893OpenMPIRBuilder::InsertPointOrErrorTy
6894OpenMPIRBuilder::applyDynamicWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
6895 InsertPointTy AllocaIP,
6896 OMPScheduleType SchedType,
6897 bool NeedsBarrier, Value *Chunk) {
6898 assert(CLI->isValid() && "Requires a valid canonical loop");
6899 assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
6900 "Require dedicated allocate IP");
6901 assert(isValidWorkshareLoopScheduleType(SchedType) &&
6902 "Require valid schedule type");
6903
6904 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
6905 OMPScheduleType::ModifierOrdered;
6906
6907 // Set up the source location value for OpenMP runtime.
6908 Builder.SetCurrentDebugLocation(DL);
6909
6910 uint32_t SrcLocStrSize;
6911 Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
6912 Value *SrcLoc =
6913 getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: OMP_IDENT_FLAG_WORK_LOOP);
6914
6915 // Declare useful OpenMP runtime functions.
6916 Value *IV = CLI->getIndVar();
6917 Type *IVTy = IV->getType();
6918 FunctionCallee DynamicInit = getKmpcForDynamicInitForType(Ty: IVTy, M, OMPBuilder&: *this);
6919 FunctionCallee DynamicNext = getKmpcForDynamicNextForType(Ty: IVTy, M, OMPBuilder&: *this);
6920
6921 // Allocate space for computed loop bounds as expected by the "init" function.
6922 Builder.SetInsertPoint(AllocaIP.getNodeParent()->getFirstNonPHIOrAlloca());
6923 Type *I32Type = Type::getInt32Ty(C&: M.getContext());
6924 Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
6925 Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
6926 Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
6927 Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
6928 CLI->setLastIter(PLastIter);
6929
6930 // At the end of the preheader, prepare for calling the "init" function by
6931 // storing the current loop bounds into the allocated space. A canonical loop
6932 // always iterates from 0 to trip-count with step 1. Note that "init" expects
6933 // and produces an inclusive upper bound.
6934 BasicBlock *PreHeader = CLI->getPreheader();
6935 Builder.SetInsertPoint(PreHeader->getTerminator());
6936 Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
6937 Builder.CreateStore(Val: One, Ptr: PLowerBound);
6938 Value *UpperBound = CLI->getTripCount();
6939 Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
6940 Builder.CreateStore(Val: One, Ptr: PStride);
6941
6942 BasicBlock *Header = CLI->getHeader();
6943 BasicBlock *Exit = CLI->getExit();
6944 BasicBlock *Cond = CLI->getCond();
6945 BasicBlock *Latch = CLI->getLatch();
6946 InsertPointTy AfterIP = CLI->getAfterIP();
6947
6948 // The CLI will be "broken" in the code below, as the loop is no longer
6949 // a valid canonical loop.
6950
6951 if (!Chunk)
6952 Chunk = One;
6953
6954 Value *ThreadNum =
6955 getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize));
6956
6957 Constant *SchedulingType =
6958 ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
6959
6960 // Call the "init" function.
6961 createRuntimeFunctionCall(Callee: DynamicInit, Args: {SrcLoc, ThreadNum, SchedulingType,
6962 /* LowerBound */ One, UpperBound,
6963 /* step */ One, Chunk});
6964
6965 // An outer loop around the existing one.
6966 BasicBlock *OuterCond = BasicBlock::Create(
6967 Context&: PreHeader->getContext(), Name: Twine(PreHeader->getName()) + ".outer.cond",
6968 Parent: PreHeader->getParent());
6969 // This needs to be 32-bit always, so can't use the IVTy Zero above.
6970 Builder.SetInsertPoint(OuterCond->getFirstInsertionPt());
6971 Value *Res = createRuntimeFunctionCall(
6972 Callee: DynamicNext,
6973 Args: {SrcLoc, ThreadNum, PLastIter, PLowerBound, PUpperBound, PStride});
6974 Constant *Zero32 = ConstantInt::get(Ty: I32Type, V: 0);
6975 Value *MoreWork = Builder.CreateCmp(Pred: CmpInst::ICMP_NE, LHS: Res, RHS: Zero32);
6976 Value *LowerBound =
6977 Builder.CreateSub(LHS: Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound), RHS: One, Name: "lb");
6978 Builder.CreateCondBr(Cond: MoreWork, True: Header, False: Exit);
6979
6980 // Change PHI-node in loop header to use outer cond rather than preheader,
6981 // and set IV to the LowerBound.
6982 Instruction *Phi = &Header->front();
6983 auto *PI = cast<PHINode>(Val: Phi);
6984 PI->setIncomingBlock(i: 0, BB: OuterCond);
6985 PI->setIncomingValue(i: 0, V: LowerBound);
6986
6987 // Then set the pre-header to jump to the OuterCond
6988 Instruction *Term = PreHeader->getTerminator();
6989 auto *Br = cast<UncondBrInst>(Val: Term);
6990 Br->setSuccessor(OuterCond);
6991
6992 // Modify the inner condition:
6993 // * Use the UpperBound returned from the DynamicNext call.
6994 // * jump to the loop outer loop when done with one of the inner loops.
6995 Builder.SetInsertPoint(Cond->getFirstInsertionPt());
6996 UpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound, Name: "ub");
6997 Instruction *Comp = &*Builder.GetInsertPoint();
6998 auto *CI = cast<CmpInst>(Val: Comp);
6999 CI->setOperand(i_nocapture: 1, Val_nocapture: UpperBound);
7000 // Redirect the inner exit to branch to outer condition.
7001 Instruction *Branch = &Cond->back();
7002 auto *BI = cast<CondBrInst>(Val: Branch);
7003 assert(BI->getSuccessor(1) == Exit);
7004 BI->setSuccessor(idx: 1, NewSucc: OuterCond);
7005
7006 // Call the "fini" function if "ordered" is present in wsloop directive.
7007 if (Ordered) {
7008 Builder.SetInsertPoint(&Latch->back());
7009 FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(Ty: IVTy, M, OMPBuilder&: *this);
7010 createRuntimeFunctionCall(Callee: DynamicFini, Args: {SrcLoc, ThreadNum});
7011 }
7012
7013 // Add the barrier if requested.
7014 if (NeedsBarrier) {
7015 Builder.SetInsertPoint(&Exit->back());
7016 InsertPointOrErrorTy BarrierIP =
7017 createBarrier(Loc: LocationDescription(Builder.saveIP(), DL),
7018 Kind: omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
7019 /* CheckCancelFlag */ false);
7020 if (!BarrierIP)
7021 return BarrierIP.takeError();
7022 }
7023
7024 CLI->invalidate();
7025 return AfterIP;
7026}
7027
7028/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
7029/// after this \p OldTarget will be orphaned.
7030static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
7031 BasicBlock *NewTarget, DebugLoc DL) {
7032 for (BasicBlock *Pred : make_early_inc_range(Range: predecessors(BB: OldTarget)))
7033 redirectTo(Source: Pred, Target: NewTarget, DL);
7034}
7035
7036static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
7037 SmallPtrSet<BasicBlock *, 8> InternalBBs(from_range, BBs);
7038 // We add a block to BBsToKeep iff we have proven it has an external use.
7039 SmallPtrSet<BasicBlock *, 8> BBsToKeep;
7040
7041 while (true) {
7042 bool Changed = false;
7043
7044 for (BasicBlock *BB : BBs) {
7045 if (BBsToKeep.contains(Ptr: BB))
7046 continue;
7047
7048 for (Use &U : BB->uses()) {
7049 auto *UseInst = dyn_cast<Instruction>(Val: U.getUser());
7050 if (!UseInst)
7051 continue;
7052 BasicBlock *UseBB = UseInst->getParent();
7053 if (!InternalBBs.contains(Ptr: UseBB) || BBsToKeep.contains(Ptr: UseBB)) {
7054 BBsToKeep.insert(Ptr: BB);
7055 Changed = true;
7056 break;
7057 }
7058 }
7059 }
7060
7061 if (!Changed)
7062 break;
7063 }
7064
7065 SmallVector<BasicBlock *> BBsToDelete = filter_to_vector(
7066 C&: BBs, Pred: [&BBsToKeep](BasicBlock *BB) { return !BBsToKeep.contains(Ptr: BB); });
7067 DeleteDeadBlocks(BBs: BBsToDelete);
7068}
7069
7070CanonicalLoopInfo *
7071OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7072 InsertPointTy ComputeIP) {
7073 assert(Loops.size() >= 1 && "At least one loop required");
7074 size_t NumLoops = Loops.size();
7075
7076 // Nothing to do if there is already just one loop.
7077 if (NumLoops == 1)
7078 return Loops.front();
7079
7080 CanonicalLoopInfo *Outermost = Loops.front();
7081 CanonicalLoopInfo *Innermost = Loops.back();
7082 BasicBlock *OrigPreheader = Outermost->getPreheader();
7083 BasicBlock *OrigAfter = Outermost->getAfter();
7084 Function *F = OrigPreheader->getParent();
7085
7086 // Loop control blocks that may become orphaned later.
7087 SmallVector<BasicBlock *, 12> OldControlBBs;
7088 OldControlBBs.reserve(N: 6 * Loops.size());
7089 for (CanonicalLoopInfo *Loop : Loops)
7090 Loop->collectControlBlocks(BBs&: OldControlBBs);
7091
7092 // Setup the IRBuilder for inserting the trip count computation.
7093 Builder.SetCurrentDebugLocation(DL);
7094 if (ComputeIP.isValid())
7095 Builder.restoreIP(IP: ComputeIP);
7096 else
7097 Builder.restoreIP(IP: Outermost->getPreheaderIP());
7098
7099 // Derive the collapsed' loop trip count.
7100 // TODO: Find common/largest indvar type.
7101 Value *CollapsedTripCount = nullptr;
7102 for (CanonicalLoopInfo *L : Loops) {
7103 assert(L->isValid() &&
7104 "All loops to collapse must be valid canonical loops");
7105 Value *OrigTripCount = L->getTripCount();
7106 if (!CollapsedTripCount) {
7107 CollapsedTripCount = OrigTripCount;
7108 continue;
7109 }
7110
7111 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
7112 CollapsedTripCount =
7113 Builder.CreateNUWMul(LHS: CollapsedTripCount, RHS: OrigTripCount);
7114 }
7115
7116 // Create the collapsed loop control flow.
7117 CanonicalLoopInfo *Result =
7118 createLoopSkeleton(DL, TripCount: CollapsedTripCount, F,
7119 PreInsertBefore: OrigPreheader->getNextNode(), PostInsertBefore: OrigAfter, Name: "collapsed",
7120 /*IsCollapsed=*/true);
7121
7122 // Build the collapsed loop body code.
7123 // Start with deriving the input loop induction variables from the collapsed
7124 // one, using a divmod scheme. To preserve the original loops' order, the
7125 // innermost loop use the least significant bits.
7126 Builder.restoreIP(IP: Result->getBodyIP());
7127
7128 Value *Leftover = Result->getIndVar();
7129 SmallVector<Value *> NewIndVars;
7130 NewIndVars.resize(N: NumLoops);
7131 for (int i = NumLoops - 1; i >= 1; --i) {
7132 Value *OrigTripCount = Loops[i]->getTripCount();
7133
7134 Value *NewIndVar = Builder.CreateURem(LHS: Leftover, RHS: OrigTripCount);
7135 NewIndVars[i] = NewIndVar;
7136
7137 Leftover = Builder.CreateUDiv(LHS: Leftover, RHS: OrigTripCount);
7138 }
7139 // Outermost loop gets all the remaining bits.
7140 NewIndVars[0] = Leftover;
7141
7142 // Construct the loop body control flow.
7143 // We progressively construct the branch structure following in direction of
7144 // the control flow, from the leading in-between code, the loop nest body, the
7145 // trailing in-between code, and rejoining the collapsed loop's latch.
7146 // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
7147 // the ContinueBlock is set, continue with that block. If ContinuePred, use
7148 // its predecessors as sources.
7149 BasicBlock *ContinueBlock = Result->getBody();
7150 BasicBlock *ContinuePred = nullptr;
7151 auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
7152 BasicBlock *NextSrc) {
7153 if (ContinueBlock)
7154 redirectTo(Source: ContinueBlock, Target: Dest, DL);
7155 else
7156 redirectAllPredecessorsTo(OldTarget: ContinuePred, NewTarget: Dest, DL);
7157
7158 ContinueBlock = nullptr;
7159 ContinuePred = NextSrc;
7160 };
7161
7162 // The code before the nested loop of each level.
7163 // Because we are sinking it into the nest, it will be executed more often
7164 // that the original loop. More sophisticated schemes could keep track of what
7165 // the in-between code is and instantiate it only once per thread.
7166 for (size_t i = 0; i < NumLoops - 1; ++i)
7167 ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
7168
7169 // Connect the loop nest body.
7170 ContinueWith(Innermost->getBody(), Innermost->getLatch());
7171
7172 // The code after the nested loop at each level.
7173 for (size_t i = NumLoops - 1; i > 0; --i)
7174 ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
7175
7176 // Connect the finished loop to the collapsed loop latch.
7177 ContinueWith(Result->getLatch(), nullptr);
7178
7179 // Replace the input loops with the new collapsed loop.
7180 redirectTo(Source: Outermost->getPreheader(), Target: Result->getPreheader(), DL);
7181 redirectTo(Source: Result->getAfter(), Target: Outermost->getAfter(), DL);
7182
7183 // Replace the input loop indvars with the derived ones.
7184 for (size_t i = 0; i < NumLoops; ++i)
7185 Loops[i]->getIndVar()->replaceAllUsesWith(V: NewIndVars[i]);
7186
7187 // Remove unused parts of the input loops.
7188 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7189
7190 for (CanonicalLoopInfo *L : Loops)
7191 L->invalidate();
7192
7193#ifndef NDEBUG
7194 Result->assertOK();
7195#endif
7196 return Result;
7197}
7198
7199std::vector<CanonicalLoopInfo *>
7200OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
7201 ArrayRef<Value *> TileSizes) {
7202 assert(TileSizes.size() == Loops.size() &&
7203 "Must pass as many tile sizes as there are loops");
7204 int NumLoops = Loops.size();
7205 assert(NumLoops >= 1 && "At least one loop to tile required");
7206
7207 CanonicalLoopInfo *OutermostLoop = Loops.front();
7208 CanonicalLoopInfo *InnermostLoop = Loops.back();
7209 Function *F = OutermostLoop->getBody()->getParent();
7210 BasicBlock *InnerEnter = InnermostLoop->getBody();
7211 BasicBlock *InnerLatch = InnermostLoop->getLatch();
7212
7213 // Loop control blocks that may become orphaned later.
7214 SmallVector<BasicBlock *, 12> OldControlBBs;
7215 OldControlBBs.reserve(N: 6 * Loops.size());
7216 for (CanonicalLoopInfo *Loop : Loops)
7217 Loop->collectControlBlocks(BBs&: OldControlBBs);
7218
7219 // Collect original trip counts and induction variable to be accessible by
7220 // index. Also, the structure of the original loops is not preserved during
7221 // the construction of the tiled loops, so do it before we scavenge the BBs of
7222 // any original CanonicalLoopInfo.
7223 SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
7224 for (CanonicalLoopInfo *L : Loops) {
7225 assert(L->isValid() && "All input loops must be valid canonical loops");
7226 OrigTripCounts.push_back(Elt: L->getTripCount());
7227 OrigIndVars.push_back(Elt: L->getIndVar());
7228 }
7229
7230 // Collect the code between loop headers. These may contain SSA definitions
7231 // that are used in the loop nest body. To be usable with in the innermost
7232 // body, these BasicBlocks will be sunk into the loop nest body. That is,
7233 // these instructions may be executed more often than before the tiling.
7234 // TODO: It would be sufficient to only sink them into body of the
7235 // corresponding tile loop.
7236 SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
7237 for (int i = 0; i < NumLoops - 1; ++i) {
7238 CanonicalLoopInfo *Surrounding = Loops[i];
7239 CanonicalLoopInfo *Nested = Loops[i + 1];
7240
7241 BasicBlock *EnterBB = Surrounding->getBody();
7242 BasicBlock *ExitBB = Nested->getHeader();
7243 InbetweenCode.emplace_back(Args&: EnterBB, Args&: ExitBB);
7244 }
7245
7246 // Compute the trip counts of the floor loops.
7247 Builder.SetCurrentDebugLocation(DL);
7248 Builder.restoreIP(IP: OutermostLoop->getPreheaderIP());
7249 SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
7250 for (int i = 0; i < NumLoops; ++i) {
7251 Value *TileSize = TileSizes[i];
7252 Value *OrigTripCount = OrigTripCounts[i];
7253 Type *IVType = OrigTripCount->getType();
7254
7255 Value *FloorCompleteTripCount = Builder.CreateUDiv(LHS: OrigTripCount, RHS: TileSize);
7256 Value *FloorTripRem = Builder.CreateURem(LHS: OrigTripCount, RHS: TileSize);
7257
7258 // 0 if tripcount divides the tilesize, 1 otherwise.
7259 // 1 means we need an additional iteration for a partial tile.
7260 //
7261 // Unfortunately we cannot just use the roundup-formula
7262 // (tripcount + tilesize - 1)/tilesize
7263 // because the summation might overflow. We do not want introduce undefined
7264 // behavior when the untiled loop nest did not.
7265 Value *FloorTripOverflow =
7266 Builder.CreateICmpNE(LHS: FloorTripRem, RHS: ConstantInt::get(Ty: IVType, V: 0));
7267
7268 FloorTripOverflow = Builder.CreateZExt(V: FloorTripOverflow, DestTy: IVType);
7269 Value *FloorTripCount =
7270 Builder.CreateAdd(LHS: FloorCompleteTripCount, RHS: FloorTripOverflow,
7271 Name: "omp_floor" + Twine(i) + ".tripcount", HasNUW: true);
7272
7273 // Remember some values for later use.
7274 FloorCompleteCount.push_back(Elt: FloorCompleteTripCount);
7275 FloorCount.push_back(Elt: FloorTripCount);
7276 FloorRems.push_back(Elt: FloorTripRem);
7277 }
7278
7279 // Generate the new loop nest, from the outermost to the innermost.
7280 std::vector<CanonicalLoopInfo *> Result;
7281 Result.reserve(n: NumLoops * 2);
7282
7283 // The basic block of the surrounding loop that enters the nest generated
7284 // loop.
7285 BasicBlock *Enter = OutermostLoop->getPreheader();
7286
7287 // The basic block of the surrounding loop where the inner code should
7288 // continue.
7289 BasicBlock *Continue = OutermostLoop->getAfter();
7290
7291 // Where the next loop basic block should be inserted.
7292 BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
7293
7294 auto EmbeddNewLoop =
7295 [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
7296 Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
7297 CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
7298 DL, TripCount, F, PreInsertBefore: InnerEnter, PostInsertBefore: OutroInsertBefore, Name);
7299 redirectTo(Source: Enter, Target: EmbeddedLoop->getPreheader(), DL);
7300 redirectTo(Source: EmbeddedLoop->getAfter(), Target: Continue, DL);
7301
7302 // Setup the position where the next embedded loop connects to this loop.
7303 Enter = EmbeddedLoop->getBody();
7304 Continue = EmbeddedLoop->getLatch();
7305 OutroInsertBefore = EmbeddedLoop->getLatch();
7306 return EmbeddedLoop;
7307 };
7308
7309 auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
7310 const Twine &NameBase) {
7311 for (auto P : enumerate(First&: TripCounts)) {
7312 CanonicalLoopInfo *EmbeddedLoop =
7313 EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
7314 Result.push_back(x: EmbeddedLoop);
7315 }
7316 };
7317
7318 EmbeddNewLoops(FloorCount, "floor");
7319
7320 // Within the innermost floor loop, emit the code that computes the tile
7321 // sizes.
7322 Builder.SetInsertPoint(Enter->getTerminator());
7323 SmallVector<Value *, 4> TileCounts;
7324 for (int i = 0; i < NumLoops; ++i) {
7325 CanonicalLoopInfo *FloorLoop = Result[i];
7326 Value *TileSize = TileSizes[i];
7327
7328 Value *FloorIsEpilogue =
7329 Builder.CreateICmpEQ(LHS: FloorLoop->getIndVar(), RHS: FloorCompleteCount[i]);
7330 Value *TileTripCount =
7331 Builder.CreateSelect(C: FloorIsEpilogue, True: FloorRems[i], False: TileSize);
7332
7333 TileCounts.push_back(Elt: TileTripCount);
7334 }
7335
7336 // Create the tile loops.
7337 EmbeddNewLoops(TileCounts, "tile");
7338
7339 // Insert the inbetween code into the body.
7340 BasicBlock *BodyEnter = Enter;
7341 BasicBlock *BodyEntered = nullptr;
7342 for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
7343 BasicBlock *EnterBB = P.first;
7344 BasicBlock *ExitBB = P.second;
7345
7346 if (BodyEnter)
7347 redirectTo(Source: BodyEnter, Target: EnterBB, DL);
7348 else
7349 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: EnterBB, DL);
7350
7351 BodyEnter = nullptr;
7352 BodyEntered = ExitBB;
7353 }
7354
7355 // Append the original loop nest body into the generated loop nest body.
7356 if (BodyEnter)
7357 redirectTo(Source: BodyEnter, Target: InnerEnter, DL);
7358 else
7359 redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: InnerEnter, DL);
7360 redirectAllPredecessorsTo(OldTarget: InnerLatch, NewTarget: Continue, DL);
7361
7362 // Replace the original induction variable with an induction variable computed
7363 // from the tile and floor induction variables.
7364 Builder.restoreIP(IP: Result.back()->getBodyIP());
7365 for (int i = 0; i < NumLoops; ++i) {
7366 CanonicalLoopInfo *FloorLoop = Result[i];
7367 CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
7368 Value *OrigIndVar = OrigIndVars[i];
7369 Value *Size = TileSizes[i];
7370
7371 Value *Scale =
7372 Builder.CreateMul(LHS: Size, RHS: FloorLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7373 Value *Shift =
7374 Builder.CreateAdd(LHS: Scale, RHS: TileLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
7375 OrigIndVar->replaceAllUsesWith(V: Shift);
7376 }
7377
7378 // Remove unused parts of the original loops.
7379 removeUnusedBlocksFromParent(BBs: OldControlBBs);
7380
7381 for (CanonicalLoopInfo *L : Loops)
7382 L->invalidate();
7383
7384#ifndef NDEBUG
7385 for (CanonicalLoopInfo *GenL : Result)
7386 GenL->assertOK();
7387#endif
7388 return Result;
7389}
7390
7391/// Attach metadata \p Properties to the basic block described by \p BB. If the
7392/// basic block already has metadata, the basic block properties are appended.
7393static void addBasicBlockMetadata(BasicBlock *BB,
7394 ArrayRef<Metadata *> Properties) {
7395 // Nothing to do if no property to attach.
7396 if (Properties.empty())
7397 return;
7398
7399 LLVMContext &Ctx = BB->getContext();
7400 SmallVector<Metadata *> NewProperties;
7401 NewProperties.push_back(Elt: nullptr);
7402
7403 // If the basic block already has metadata, prepend it to the new metadata.
7404 MDNode *Existing = BB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
7405 if (Existing)
7406 append_range(C&: NewProperties, R: drop_begin(RangeOrContainer: Existing->operands(), N: 1));
7407
7408 append_range(C&: NewProperties, R&: Properties);
7409 MDNode *BasicBlockID = MDNode::getDistinct(Context&: Ctx, MDs: NewProperties);
7410 BasicBlockID->replaceOperandWith(I: 0, New: BasicBlockID);
7411
7412 BB->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: BasicBlockID);
7413}
7414
7415/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
7416/// loop already has metadata, the loop properties are appended.
7417static void addLoopMetadata(CanonicalLoopInfo *Loop,
7418 ArrayRef<Metadata *> Properties) {
7419 assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");
7420
7421 // Attach metadata to the loop's latch
7422 BasicBlock *Latch = Loop->getLatch();
7423 assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");
7424 addBasicBlockMetadata(BB: Latch, Properties);
7425}
7426
7427/// Attach llvm.access.group metadata to the memref instructions of \p Block
7428static void addAccessGroupMetadata(BasicBlock *Block, MDNode *AccessGroup,
7429 LoopInfo &LI) {
7430 for (Instruction &I : *Block) {
7431 if (I.mayReadOrWriteMemory()) {
7432 // TODO: This instruction may already have access group from
7433 // other pragmas e.g. #pragma clang loop vectorize. Append
7434 // so that the existing metadata is not overwritten.
7435 I.setMetadata(KindID: LLVMContext::MD_access_group, Node: AccessGroup);
7436 }
7437 }
7438}
7439
7440CanonicalLoopInfo *
7441OpenMPIRBuilder::fuseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops) {
7442 CanonicalLoopInfo *firstLoop = Loops.front();
7443 CanonicalLoopInfo *lastLoop = Loops.back();
7444 Function *F = firstLoop->getPreheader()->getParent();
7445
7446 // Loop control blocks that will become orphaned later
7447 SmallVector<BasicBlock *> oldControlBBs;
7448 for (CanonicalLoopInfo *Loop : Loops)
7449 Loop->collectControlBlocks(BBs&: oldControlBBs);
7450
7451 // Collect original trip counts
7452 SmallVector<Value *> origTripCounts;
7453 for (CanonicalLoopInfo *L : Loops) {
7454 assert(L->isValid() && "All input loops must be valid canonical loops");
7455 origTripCounts.push_back(Elt: L->getTripCount());
7456 }
7457
7458 Builder.SetCurrentDebugLocation(DL);
7459
7460 // Compute max trip count.
7461 // The fused loop will be from 0 to max(origTripCounts)
7462 BasicBlock *TCBlock = BasicBlock::Create(Context&: F->getContext(), Name: "omp.fuse.comp.tc",
7463 Parent: F, InsertBefore: firstLoop->getHeader());
7464 Builder.SetInsertPoint(TCBlock);
7465 Value *fusedTripCount = nullptr;
7466 for (CanonicalLoopInfo *L : Loops) {
7467 assert(L->isValid() && "All loops to fuse must be valid canonical loops");
7468 Value *origTripCount = L->getTripCount();
7469 if (!fusedTripCount) {
7470 fusedTripCount = origTripCount;
7471 continue;
7472 }
7473 Value *condTP = Builder.CreateICmpSGT(LHS: fusedTripCount, RHS: origTripCount);
7474 fusedTripCount = Builder.CreateSelect(C: condTP, True: fusedTripCount, False: origTripCount,
7475 Name: ".omp.fuse.tc");
7476 }
7477
7478 // Generate new loop
7479 CanonicalLoopInfo *fused =
7480 createLoopSkeleton(DL, TripCount: fusedTripCount, F, PreInsertBefore: firstLoop->getBody(),
7481 PostInsertBefore: lastLoop->getLatch(), Name: "fused");
7482
7483 // Replace original loops with the fused loop
7484 // Preheader and After are not considered inside the CLI.
7485 // These are used to compute the individual TCs of the loops
7486 // so they have to be put before the resulting fused loop.
7487 // Moving them up for readability.
7488 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7489 Loops[i]->getPreheader()->moveBefore(MovePos: TCBlock);
7490 Loops[i]->getAfter()->moveBefore(MovePos: TCBlock);
7491 }
7492 lastLoop->getPreheader()->moveBefore(MovePos: TCBlock);
7493
7494 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7495 redirectTo(Source: Loops[i]->getPreheader(), Target: Loops[i]->getAfter(), DL);
7496 redirectTo(Source: Loops[i]->getAfter(), Target: Loops[i + 1]->getPreheader(), DL);
7497 }
7498 redirectTo(Source: lastLoop->getPreheader(), Target: TCBlock, DL);
7499 redirectTo(Source: TCBlock, Target: fused->getPreheader(), DL);
7500 redirectTo(Source: fused->getAfter(), Target: lastLoop->getAfter(), DL);
7501
7502 // Build the fused body
7503 // Create new Blocks with conditions that jump to the original loop bodies
7504 SmallVector<BasicBlock *> condBBs;
7505 SmallVector<Value *> condValues;
7506 for (size_t i = 0; i < Loops.size(); ++i) {
7507 BasicBlock *condBlock = BasicBlock::Create(
7508 Context&: F->getContext(), Name: "omp.fused.inner.cond", Parent: F, InsertBefore: Loops[i]->getBody());
7509 Builder.SetInsertPoint(condBlock);
7510 Value *condValue =
7511 Builder.CreateICmpSLT(LHS: fused->getIndVar(), RHS: origTripCounts[i]);
7512 condBBs.push_back(Elt: condBlock);
7513 condValues.push_back(Elt: condValue);
7514 }
7515 // Join the condition blocks with the bodies of the original loops
7516 redirectTo(Source: fused->getBody(), Target: condBBs[0], DL);
7517 for (size_t i = 0; i < Loops.size() - 1; ++i) {
7518 Builder.SetInsertPoint(condBBs[i]);
7519 Builder.CreateCondBr(Cond: condValues[i], True: Loops[i]->getBody(), False: condBBs[i + 1]);
7520 redirectAllPredecessorsTo(OldTarget: Loops[i]->getLatch(), NewTarget: condBBs[i + 1], DL);
7521 // Replace the IV with the fused IV
7522 Loops[i]->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7523 }
7524 // Last body jumps to the created end body block
7525 Builder.SetInsertPoint(condBBs.back());
7526 Builder.CreateCondBr(Cond: condValues.back(), True: lastLoop->getBody(),
7527 False: fused->getLatch());
7528 redirectAllPredecessorsTo(OldTarget: lastLoop->getLatch(), NewTarget: fused->getLatch(), DL);
7529 // Replace the IV with the fused IV
7530 lastLoop->getIndVar()->replaceAllUsesWith(V: fused->getIndVar());
7531
7532 // The loop latch must have only one predecessor. Currently it is branched to
7533 // from both the last condition block and the last loop body
7534 fused->getLatch()->splitBasicBlockBefore(I: fused->getLatch()->begin(),
7535 BBName: "omp.fused.pre_latch");
7536
7537 // Remove unused parts
7538 removeUnusedBlocksFromParent(BBs: oldControlBBs);
7539
7540 // Invalidate old CLIs
7541 for (CanonicalLoopInfo *L : Loops)
7542 L->invalidate();
7543
7544#ifndef NDEBUG
7545 fused->assertOK();
7546#endif
7547 return fused;
7548}
7549
7550void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
7551 LLVMContext &Ctx = Builder.getContext();
7552 addLoopMetadata(
7553 Loop, Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7554 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full"))});
7555}
7556
7557void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
7558 LLVMContext &Ctx = Builder.getContext();
7559 addLoopMetadata(
7560 Loop, Properties: {
7561 MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
7562 });
7563}
7564
7565void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
7566 Value *IfCond, ValueToValueMapTy &VMap,
7567 LoopAnalysis &LIA, LoopInfo &LI, Loop *L,
7568 const Twine &NamePrefix) {
7569 Function *F = CanonicalLoop->getFunction();
7570
7571 // We can't do
7572 // if (cond) {
7573 // simd_loop;
7574 // } else {
7575 // non_simd_loop;
7576 // }
7577 // because then the CanonicalLoopInfo would only point to one of the loops:
7578 // leading to other constructs operating on the same loop to malfunction.
7579 // Instead generate
7580 // while (...) {
7581 // if (cond) {
7582 // simd_body;
7583 // } else {
7584 // not_simd_body;
7585 // }
7586 // }
7587 // At least for simple loops, LLVM seems able to hoist the if out of the loop
7588 // body at -O3
7589
7590 // Define where if branch should be inserted
7591 auto SplitBeforeIt = CanonicalLoop->getBody()->getFirstNonPHIIt();
7592
7593 // Create additional blocks for the if statement
7594 BasicBlock *Cond = SplitBeforeIt->getParent();
7595 llvm::LLVMContext &C = Cond->getContext();
7596 llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
7597 Context&: C, Name: NamePrefix + ".if.then", Parent: Cond->getParent(), InsertBefore: Cond->getNextNode());
7598 llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
7599 Context&: C, Name: NamePrefix + ".if.else", Parent: Cond->getParent(), InsertBefore: CanonicalLoop->getExit());
7600
7601 // Create if condition branch.
7602 Builder.SetInsertPoint(SplitBeforeIt);
7603 Instruction *BrInstr =
7604 Builder.CreateCondBr(Cond: IfCond, True: ThenBlock, /*ifFalse*/ False: ElseBlock);
7605 InsertPointTy IP(++BrInstr->getIterator());
7606 // Then block contains branch to omp loop body which needs to be vectorized
7607 spliceBB(IP, New: ThenBlock, CreateBranch: false, DL: Builder.getCurrentDebugLocation());
7608 ThenBlock->replaceSuccessorsPhiUsesWith(Old: Cond, New: ThenBlock);
7609
7610 Builder.SetInsertPoint(ElseBlock);
7611
7612 // Clone loop for the else branch
7613 SmallVector<BasicBlock *, 8> NewBlocks;
7614
7615 SmallVector<BasicBlock *, 8> ExistingBlocks;
7616 ExistingBlocks.reserve(N: L->getNumBlocks() + 1);
7617 ExistingBlocks.push_back(Elt: ThenBlock);
7618 ExistingBlocks.append(in_start: L->block_begin(), in_end: L->block_end());
7619 // Cond is the block that has the if clause condition
7620 // LoopCond is omp_loop.cond
7621 // LoopHeader is omp_loop.header
7622 BasicBlock *LoopCond = Cond->getUniquePredecessor();
7623 BasicBlock *LoopHeader = LoopCond->getUniquePredecessor();
7624 assert(LoopCond && LoopHeader && "Invalid loop structure");
7625 for (BasicBlock *Block : ExistingBlocks) {
7626 if (Block == L->getLoopPreheader() || Block == L->getLoopLatch() ||
7627 Block == LoopHeader || Block == LoopCond || Block == Cond) {
7628 continue;
7629 }
7630 BasicBlock *NewBB = CloneBasicBlock(BB: Block, VMap, NameSuffix: "", F);
7631
7632 // fix name not to be omp.if.then
7633 if (Block == ThenBlock)
7634 NewBB->setName(NamePrefix + ".if.else");
7635
7636 NewBB->moveBefore(MovePos: CanonicalLoop->getExit());
7637 VMap[Block] = NewBB;
7638 NewBlocks.push_back(Elt: NewBB);
7639 }
7640 remapInstructionsInBlocks(Blocks: NewBlocks, VMap);
7641 Builder.CreateBr(Dest: NewBlocks.front());
7642
7643 // The loop latch must have only one predecessor. Currently it is branched to
7644 // from both the 'then' and 'else' branches.
7645 L->getLoopLatch()->splitBasicBlockBefore(I: L->getLoopLatch()->begin(),
7646 BBName: NamePrefix + ".pre_latch");
7647
7648 // Ensure that the then block is added to the loop so we add the attributes in
7649 // the next step
7650 L->addBasicBlockToLoop(NewBB: ThenBlock, LI);
7651}
7652
7653unsigned
7654OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
7655 const StringMap<bool> &Features) {
7656 if (TargetTriple.isX86()) {
7657 if (Features.lookup(Key: "avx512f"))
7658 return 512;
7659 else if (Features.lookup(Key: "avx"))
7660 return 256;
7661 return 128;
7662 }
7663 if (TargetTriple.isPPC())
7664 return 128;
7665 if (TargetTriple.isWasm())
7666 return 128;
7667 if (TargetTriple.isSystemZ())
7668 return 64;
7669 return 0;
7670}
7671
7672void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
7673 MapVector<Value *, Value *> AlignedVars,
7674 Value *IfCond, OrderKind Order,
7675 ConstantInt *Simdlen, ConstantInt *Safelen) {
7676 LLVMContext &Ctx = Builder.getContext();
7677
7678 Function *F = CanonicalLoop->getFunction();
7679
7680 // Blocks must have terminators.
7681 // FIXME: Don't run analyses on incomplete/invalid IR.
7682 SmallVector<Instruction *> UIs;
7683 for (BasicBlock &BB : *F)
7684 if (!BB.hasTerminator())
7685 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7686
7687 // TODO: We should not rely on pass manager. Currently we use pass manager
7688 // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
7689 // object. We should have a method which returns all blocks between
7690 // CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
7691 FunctionAnalysisManager FAM;
7692 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7693 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7694 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7695
7696 LoopAnalysis LIA;
7697 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7698
7699 for (Instruction *I : UIs)
7700 I->eraseFromParent();
7701
7702 Loop *L = LI.getLoopFor(BB: CanonicalLoop->getHeader());
7703 if (AlignedVars.size()) {
7704 InsertPointTy IP = Builder.saveIP();
7705 for (auto &AlignedItem : AlignedVars) {
7706 Value *AlignedPtr = AlignedItem.first;
7707 Value *Alignment = AlignedItem.second;
7708 Instruction *loadInst = dyn_cast<Instruction>(Val: AlignedPtr);
7709 Builder.SetInsertPoint(loadInst->getNextNode());
7710 Builder.CreateAlignmentAssumption(DL: F->getDataLayout(), PtrValue: AlignedPtr,
7711 Alignment);
7712 }
7713 Builder.restoreIP(IP);
7714 }
7715
7716 if (IfCond) {
7717 ValueToValueMapTy VMap;
7718 createIfVersion(CanonicalLoop, IfCond, VMap, LIA, LI, L, NamePrefix: "simd");
7719 }
7720
7721 SmallPtrSet<BasicBlock *, 8> Reachable;
7722
7723 // Get the basic blocks from the loop in which memref instructions
7724 // can be found.
7725 // TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
7726 // preferably without running any passes.
7727 for (BasicBlock *Block : L->getBlocks()) {
7728 if (Block == CanonicalLoop->getCond() ||
7729 Block == CanonicalLoop->getHeader())
7730 continue;
7731 Reachable.insert(Ptr: Block);
7732 }
7733
7734 SmallVector<Metadata *> LoopMDList;
7735
7736 // In presence of finite 'safelen', it may be unsafe to mark all
7737 // the memory instructions parallel, because loop-carried
7738 // dependences of 'safelen' iterations are possible.
7739 // If clause order(concurrent) is specified then the memory instructions
7740 // are marked parallel even if 'safelen' is finite.
7741 if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent))
7742 applyParallelAccessesMetadata(CLI: CanonicalLoop, Ctx, Loop: L, LoopInfo&: LI, LoopMDList);
7743
7744 // FIXME: the IF clause shares a loop backedge for the SIMD and non-SIMD
7745 // versions so we can't add the loop attributes in that case.
7746 if (IfCond) {
7747 // we can still add llvm.loop.parallel_access
7748 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7749 return;
7750 }
7751
7752 // Use the above access group metadata to create loop level
7753 // metadata, which should be distinct for each loop.
7754 LoopMDList.push_back(
7755 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable")}));
7756
7757 if (Simdlen || Safelen) {
7758 // If both simdlen and safelen clauses are specified, the value of the
7759 // simdlen parameter must be less than or equal to the value of the safelen
7760 // parameter. Therefore, use safelen only in the absence of simdlen.
7761 ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
7762 LoopMDList.push_back(
7763 Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
7764 ConstantAsMetadata::get(C: VectorizeWidth)}));
7765 }
7766
7767 addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
7768}
7769
7770/// Create the TargetMachine object to query the backend for optimization
7771/// preferences.
7772///
7773/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
7774/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
7775/// needed for the LLVM pass pipline. We use some default options to avoid
7776/// having to pass too many settings from the frontend that probably do not
7777/// matter.
7778///
7779/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
7780/// method. If we are going to use TargetMachine for more purposes, especially
7781/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
7782/// might become be worth requiring front-ends to pass on their TargetMachine,
7783/// or at least cache it between methods. Note that while fontends such as Clang
7784/// have just a single main TargetMachine per translation unit, "target-cpu" and
7785/// "target-features" that determine the TargetMachine are per-function and can
7786/// be overrided using __attribute__((target("OPTIONS"))).
7787static std::unique_ptr<TargetMachine>
7788createTargetMachine(Function *F, CodeGenOptLevel OptLevel) {
7789 Module *M = F->getParent();
7790
7791 StringRef CPU = F->getFnAttribute(Kind: "target-cpu").getValueAsString();
7792 StringRef Features = F->getFnAttribute(Kind: "target-features").getValueAsString();
7793 const llvm::Triple &Triple = M->getTargetTriple();
7794
7795 std::string Error;
7796 const llvm::Target *TheTarget = TargetRegistry::lookupTarget(TheTriple: Triple, Error);
7797 if (!TheTarget)
7798 return {};
7799
7800 llvm::TargetOptions Options;
7801 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
7802 TT: Triple, CPU, Features, Options, /*RelocModel=*/RM: std::nullopt,
7803 /*CodeModel=*/CM: std::nullopt, OL: OptLevel));
7804}
7805
7806/// Heuristically determine the best-performant unroll factor for \p CLI. This
7807/// depends on the target processor. We are re-using the same heuristics as the
7808/// LoopUnrollPass.
7809static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
7810 Function *F = CLI->getFunction();
7811
7812 // Assume the user requests the most aggressive unrolling, even if the rest of
7813 // the code is optimized using a lower setting.
7814 CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive;
7815 std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
7816
7817 // Blocks must have terminators.
7818 // FIXME: Don't run analyses on incomplete/invalid IR.
7819 SmallVector<Instruction *> UIs;
7820 for (BasicBlock &BB : *F)
7821 if (!BB.hasTerminator())
7822 UIs.push_back(Elt: new UnreachableInst(F->getContext(), &BB));
7823
7824 FunctionAnalysisManager FAM;
7825 FAM.registerPass(PassBuilder: []() { return TargetLibraryAnalysis(); });
7826 FAM.registerPass(PassBuilder: []() { return AssumptionAnalysis(); });
7827 FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
7828 FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
7829 FAM.registerPass(PassBuilder: []() { return ScalarEvolutionAnalysis(); });
7830 FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
7831 TargetIRAnalysis TIRA;
7832 if (TM)
7833 TIRA = TargetIRAnalysis(
7834 [&](const Function &F) { return TM->getTargetTransformInfo(F); });
7835 FAM.registerPass(PassBuilder: [&]() { return TIRA; });
7836
7837 TargetIRAnalysis::Result &&TTI = TIRA.run(F: *F, FAM);
7838 ScalarEvolutionAnalysis SEA;
7839 ScalarEvolution &&SE = SEA.run(F&: *F, AM&: FAM);
7840 DominatorTreeAnalysis DTA;
7841 DominatorTree &&DT = DTA.run(F&: *F, FAM);
7842 LoopAnalysis LIA;
7843 LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
7844 AssumptionAnalysis ACT;
7845 AssumptionCache &&AC = ACT.run(F&: *F, FAM);
7846 OptimizationRemarkEmitter ORE{F};
7847
7848 for (Instruction *I : UIs)
7849 I->eraseFromParent();
7850
7851 Loop *L = LI.getLoopFor(BB: CLI->getHeader());
7852 assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop");
7853
7854 TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
7855 L, SE, TTI,
7856 /*BlockFrequencyInfo=*/BFI: nullptr,
7857 /*ProfileSummaryInfo=*/PSI: nullptr, ORE, OptLevel: static_cast<int>(OptLevel),
7858 /*UserThreshold=*/std::nullopt,
7859 /*UserAllowPartial=*/true,
7860 /*UserAllowRuntime=*/UserRuntime: true,
7861 /*UserUpperBound=*/std::nullopt,
7862 /*UserFullUnrollMaxCount=*/std::nullopt);
7863
7864 UP.Force = true;
7865
7866 // Account for additional optimizations taking place before the LoopUnrollPass
7867 // would unroll the loop.
7868 UP.Threshold *= UnrollThresholdFactor;
7869 UP.PartialThreshold *= UnrollThresholdFactor;
7870
7871 // Use normal unroll factors even if the rest of the code is optimized for
7872 // size.
7873 UP.OptSizeThreshold = UP.Threshold;
7874 UP.PartialOptSizeThreshold = UP.PartialThreshold;
7875
7876 LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"
7877 << " Threshold=" << UP.Threshold << "\n"
7878 << " PartialThreshold=" << UP.PartialThreshold << "\n"
7879 << " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"
7880 << " PartialOptSizeThreshold="
7881 << UP.PartialOptSizeThreshold << "\n");
7882
7883 // Disable peeling.
7884 TargetTransformInfo::PeelingPreferences PP =
7885 gatherPeelingPreferences(L, SE, TTI,
7886 /*UserAllowPeeling=*/false,
7887 /*UserAllowProfileBasedPeeling=*/false,
7888 /*UnrollingSpecficValues=*/false);
7889
7890 SmallPtrSet<const Value *, 32> EphValues;
7891 CodeMetrics::collectEphemeralValues(L, AC: &AC, EphValues);
7892
7893 // Assume that reads and writes to stack variables can be eliminated by
7894 // Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
7895 // size.
7896 for (BasicBlock *BB : L->blocks()) {
7897 for (Instruction &I : *BB) {
7898 Value *Ptr;
7899 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
7900 Ptr = Load->getPointerOperand();
7901 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
7902 Ptr = Store->getPointerOperand();
7903 } else
7904 continue;
7905
7906 Ptr = Ptr->stripPointerCasts();
7907
7908 if (auto *Alloca = dyn_cast<AllocaInst>(Val: Ptr)) {
7909 if (Alloca->getParent() == &F->getEntryBlock())
7910 EphValues.insert(Ptr: &I);
7911 }
7912 }
7913 }
7914
7915 UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
7916
7917 // Loop is not unrollable if the loop contains certain instructions.
7918 if (!UCE.canUnroll()) {
7919 LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");
7920 return 1;
7921 }
7922
7923 LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize()
7924 << "\n");
7925
7926 // TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
7927 // be able to use it.
7928 int TripCount = 0;
7929 int MaxTripCount = 0;
7930 bool MaxOrZero = false;
7931 unsigned TripMultiple = 0;
7932
7933 unsigned Factor =
7934 computeUnrollCount(L, TTI, DT, LI: &LI, AC: &AC, SE, EphValues, ORE: &ORE, TripCount,
7935 MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP);
7936 LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n");
7937
7938 // This function returns 1 to signal to not unroll a loop.
7939 if (Factor == 0)
7940 return 1;
7941 return Factor;
7942}
7943
7944void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
7945 int32_t Factor,
7946 CanonicalLoopInfo **UnrolledCLI) {
7947 assert(Factor >= 0 && "Unroll factor must not be negative");
7948
7949 Function *F = Loop->getFunction();
7950 LLVMContext &Ctx = F->getContext();
7951
7952 // If the unrolled loop is not used for another loop-associated directive, it
7953 // is sufficient to add metadata for the LoopUnrollPass.
7954 if (!UnrolledCLI) {
7955 SmallVector<Metadata *, 2> LoopMetadata;
7956 LoopMetadata.push_back(
7957 Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")));
7958
7959 if (Factor >= 1) {
7960 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
7961 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
7962 LoopMetadata.push_back(Elt: MDNode::get(
7963 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst}));
7964 }
7965
7966 addLoopMetadata(Loop, Properties: LoopMetadata);
7967 return;
7968 }
7969
7970 // Heuristically determine the unroll factor.
7971 if (Factor == 0)
7972 Factor = computeHeuristicUnrollFactor(CLI: Loop);
7973
7974 // No change required with unroll factor 1.
7975 if (Factor == 1) {
7976 *UnrolledCLI = Loop;
7977 return;
7978 }
7979
7980 assert(Factor >= 2 &&
7981 "unrolling only makes sense with a factor of 2 or larger");
7982
7983 Type *IndVarTy = Loop->getIndVarType();
7984
7985 // Apply partial unrolling by tiling the loop by the unroll-factor, then fully
7986 // unroll the inner loop.
7987 Value *FactorVal =
7988 ConstantInt::get(Ty: IndVarTy, V: APInt(IndVarTy->getIntegerBitWidth(), Factor,
7989 /*isSigned=*/false));
7990 std::vector<CanonicalLoopInfo *> LoopNest =
7991 tileLoops(DL, Loops: {Loop}, TileSizes: {FactorVal});
7992 assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");
7993 *UnrolledCLI = LoopNest[0];
7994 CanonicalLoopInfo *InnerLoop = LoopNest[1];
7995
7996 // LoopUnrollPass can only fully unroll loops with constant trip count.
7997 // Unroll by the unroll factor with a fallback epilog for the remainder
7998 // iterations if necessary.
7999 ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
8000 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
8001 addLoopMetadata(
8002 Loop: InnerLoop,
8003 Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
8004 MDNode::get(
8005 Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst})});
8006
8007#ifndef NDEBUG
8008 (*UnrolledCLI)->assertOK();
8009#endif
8010}
8011
8012OpenMPIRBuilder::InsertPointTy
8013OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
8014 llvm::Value *BufSize, llvm::Value *CpyBuf,
8015 llvm::Value *CpyFn, llvm::Value *DidIt) {
8016 if (!updateToLocation(Loc))
8017 return Loc.IP;
8018
8019 uint32_t SrcLocStrSize;
8020 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8021 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8022 Value *ThreadId = getOrCreateThreadID(Ident);
8023
8024 llvm::Value *DidItLD = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: DidIt);
8025
8026 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
8027
8028 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_copyprivate);
8029 createRuntimeFunctionCall(Callee: Fn, Args);
8030
8031 return Builder.saveIP();
8032}
8033
8034OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createSingle(
8035 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8036 FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef<llvm::Value *> CPVars,
8037 ArrayRef<llvm::Function *> CPFuncs) {
8038
8039 if (!updateToLocation(Loc))
8040 return Loc.IP;
8041
8042 // If needed allocate and initialize `DidIt` with 0.
8043 // DidIt: flag variable: 1=single thread; 0=not single thread.
8044 llvm::Value *DidIt = nullptr;
8045 if (!CPVars.empty()) {
8046 DidIt = Builder.CreateAlloca(Ty: llvm::Type::getInt32Ty(C&: Builder.getContext()));
8047 Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: DidIt);
8048 }
8049
8050 Directive OMPD = Directive::OMPD_single;
8051 uint32_t SrcLocStrSize;
8052 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8053 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8054 Value *ThreadId = getOrCreateThreadID(Ident);
8055 Value *Args[] = {Ident, ThreadId};
8056
8057 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_single);
8058 Instruction *EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
8059
8060 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_single);
8061 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8062
8063 auto FiniCBWrapper = [&](InsertPointTy IP) -> Error {
8064 if (Error Err = FiniCB(IP))
8065 return Err;
8066
8067 // The thread that executes the single region must set `DidIt` to 1.
8068 // This is used by __kmpc_copyprivate, to know if the caller is the
8069 // single thread or not.
8070 if (DidIt)
8071 Builder.CreateStore(Val: Builder.getInt32(C: 1), Ptr: DidIt);
8072
8073 return Error::success();
8074 };
8075
8076 // generates the following:
8077 // if (__kmpc_single()) {
8078 // .... single region ...
8079 // __kmpc_end_single
8080 // }
8081 // __kmpc_copyprivate
8082 // __kmpc_barrier
8083
8084 InsertPointOrErrorTy AfterIP =
8085 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB: FiniCBWrapper,
8086 /*Conditional*/ true,
8087 /*hasFinalize*/ HasFinalize: true);
8088 if (!AfterIP)
8089 return AfterIP.takeError();
8090
8091 if (DidIt) {
8092 for (size_t I = 0, E = CPVars.size(); I < E; ++I)
8093 // NOTE BufSize is currently unused, so just pass 0.
8094 createCopyPrivate(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8095 /*BufSize=*/ConstantInt::get(Ty: Int64, V: 0), CpyBuf: CPVars[I],
8096 CpyFn: CPFuncs[I], DidIt);
8097 // NOTE __kmpc_copyprivate already inserts a barrier
8098 } else if (!IsNowait) {
8099 InsertPointOrErrorTy AfterIP =
8100 createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8101 Kind: omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
8102 /* CheckCancelFlag */ false);
8103 if (!AfterIP)
8104 return AfterIP.takeError();
8105 }
8106 return Builder.saveIP();
8107}
8108
8109OpenMPIRBuilder::InsertPointOrErrorTy
8110OpenMPIRBuilder::createScope(const LocationDescription &Loc,
8111 BodyGenCallbackTy BodyGenCB,
8112 FinalizeCallbackTy FiniCB, bool IsNowait) {
8113
8114 if (!updateToLocation(Loc))
8115 return Loc.IP;
8116
8117 // All threads execute the scope body — no conditional entry.
8118 InsertPointOrErrorTy AfterIP = EmitOMPInlinedRegion(
8119 OMPD: Directive::OMPD_scope, /*EntryCall=*/nullptr, /*ExitCall=*/nullptr,
8120 BodyGenCB, FiniCB, /*Conditional=*/false, /*HasFinalize=*/true,
8121 /*IsCancellable=*/false);
8122 if (!AfterIP)
8123 return AfterIP.takeError();
8124
8125 Builder.restoreIP(IP: *AfterIP);
8126 if (!IsNowait) {
8127 AfterIP = createBarrier(Loc: LocationDescription(Builder.saveIP(), Loc.DL),
8128 Kind: omp::Directive::OMPD_unknown,
8129 /*ForceSimpleCall=*/false,
8130 /*CheckCancelFlag=*/false);
8131 if (!AfterIP)
8132 return AfterIP.takeError();
8133 }
8134 return Builder.saveIP();
8135}
8136
8137OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createCritical(
8138 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8139 FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
8140
8141 if (!updateToLocation(Loc))
8142 return Loc.IP;
8143
8144 Directive OMPD = Directive::OMPD_critical;
8145 uint32_t SrcLocStrSize;
8146 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8147 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8148 Value *ThreadId = getOrCreateThreadID(Ident);
8149 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
8150 Value *Args[] = {Ident, ThreadId, LockVar};
8151
8152 SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args), std::end(arr&: Args));
8153 Function *RTFn = nullptr;
8154 if (HintInst) {
8155 // Add Hint to entry Args and create call
8156 EnterArgs.push_back(Elt: HintInst);
8157 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical_with_hint);
8158 } else {
8159 RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical);
8160 }
8161 Instruction *EntryCall = createRuntimeFunctionCall(Callee: RTFn, Args: EnterArgs);
8162
8163 Function *ExitRTLFn =
8164 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_critical);
8165 Instruction *ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8166
8167 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8168 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8169}
8170
8171OpenMPIRBuilder::InsertPointTy
8172OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
8173 InsertPointTy AllocaIP, unsigned NumLoops,
8174 ArrayRef<llvm::Value *> StoreValues,
8175 const Twine &Name, bool IsDependSource) {
8176 assert(
8177 llvm::all_of(StoreValues,
8178 [](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&
8179 "OpenMP runtime requires depend vec with i64 type");
8180
8181 if (!updateToLocation(Loc))
8182 return Loc.IP;
8183
8184 // Allocate space for vector and generate alloc instruction.
8185 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumLoops);
8186 Builder.restoreIP(IP: AllocaIP);
8187 AllocaInst *ArgsBase = Builder.CreateAlloca(Ty: ArrI64Ty, ArraySize: nullptr, Name);
8188 ArgsBase->setAlignment(Align(8));
8189 updateToLocation(Loc);
8190
8191 // Store the index value with offset in depend vector.
8192 for (unsigned I = 0; I < NumLoops; ++I) {
8193 Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
8194 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: I)});
8195 StoreInst *STInst = Builder.CreateStore(Val: StoreValues[I], Ptr: DependAddrGEPIter);
8196 STInst->setAlignment(Align(8));
8197 }
8198
8199 Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
8200 Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: 0)});
8201
8202 uint32_t SrcLocStrSize;
8203 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8204 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8205 Value *ThreadId = getOrCreateThreadID(Ident);
8206 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
8207
8208 Function *RTLFn = nullptr;
8209 if (IsDependSource)
8210 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_post);
8211 else
8212 RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_wait);
8213 createRuntimeFunctionCall(Callee: RTLFn, Args);
8214
8215 return Builder.saveIP();
8216}
8217
8218OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createOrderedThreadsSimd(
8219 const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
8220 FinalizeCallbackTy FiniCB, bool IsThreads) {
8221 if (!updateToLocation(Loc))
8222 return Loc.IP;
8223
8224 Directive OMPD = Directive::OMPD_ordered_blockassoc;
8225 Instruction *EntryCall = nullptr;
8226 Instruction *ExitCall = nullptr;
8227
8228 if (IsThreads) {
8229 uint32_t SrcLocStrSize;
8230 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8231 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8232 Value *ThreadId = getOrCreateThreadID(Ident);
8233 Value *Args[] = {Ident, ThreadId};
8234
8235 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_ordered);
8236 EntryCall = createRuntimeFunctionCall(Callee: EntryRTLFn, Args);
8237
8238 Function *ExitRTLFn =
8239 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_ordered);
8240 ExitCall = createRuntimeFunctionCall(Callee: ExitRTLFn, Args);
8241 }
8242
8243 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
8244 /*Conditional*/ false, /*hasFinalize*/ HasFinalize: true);
8245}
8246
8247OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::EmitOMPInlinedRegion(
8248 Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
8249 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
8250 bool HasFinalize, bool IsCancellable) {
8251
8252 if (HasFinalize)
8253 FinalizationStack.push_back(Elt: {FiniCB, OMPD, IsCancellable});
8254
8255 // Create inlined region's entry and body blocks, in preparation
8256 // for conditional creation
8257 BasicBlock *EntryBB = Builder.GetInsertBlock();
8258 Instruction *SplitPos = EntryBB->getTerminatorOrNull();
8259 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8260 SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
8261 BasicBlock *ExitBB = EntryBB->splitBasicBlock(I: SplitPos, BBName: "omp_region.end");
8262 BasicBlock *FiniBB =
8263 EntryBB->splitBasicBlock(I: EntryBB->getTerminator(), BBName: "omp_region.finalize");
8264
8265 Builder.SetInsertPoint(EntryBB->getTerminator());
8266 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
8267
8268 // generate body
8269 if (Error Err =
8270 BodyGenCB(/* AllocaIP */ InsertPointTy(),
8271 /* CodeGenIP */ Builder.saveIP(), /* DeallocBlocks */ {}))
8272 return Err;
8273
8274 // emit exit call and do any needed finalization.
8275 auto FinIP = FiniBB->getFirstInsertionPt();
8276 assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
8277 FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
8278 "Unexpected control flow graph state!!");
8279 InsertPointOrErrorTy AfterIP =
8280 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
8281 if (!AfterIP)
8282 return AfterIP.takeError();
8283
8284 // If we are skipping the region of a non conditional, remove the exit
8285 // block, and clear the builder's insertion point.
8286 assert(SplitPos->getParent() == ExitBB &&
8287 "Unexpected Insertion point location!");
8288 auto merged = MergeBlockIntoPredecessor(BB: ExitBB);
8289 BasicBlock *ExitPredBB = SplitPos->getParent();
8290 auto InsertBB = merged ? ExitPredBB : ExitBB;
8291 if (!isa_and_nonnull<UncondBrInst, CondBrInst>(Val: SplitPos))
8292 SplitPos->eraseFromParent();
8293 Builder.SetInsertPoint(InsertBB);
8294
8295 return Builder.saveIP();
8296}
8297
8298OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
8299 Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
8300 // if nothing to do, Return current insertion point.
8301 if (!Conditional || !EntryCall)
8302 return Builder.saveIP();
8303
8304 BasicBlock *EntryBB = Builder.GetInsertBlock();
8305 Value *CallBool = Builder.CreateIsNotNull(Arg: EntryCall);
8306 auto *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp_region.body");
8307 auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
8308
8309 // Emit thenBB and set the Builder's insertion point there for
8310 // body generation next. Place the block after the current block.
8311 Function *CurFn = EntryBB->getParent();
8312 CurFn->insert(Position: std::next(x: EntryBB->getIterator()), BB: ThenBB);
8313
8314 // Move Entry branch to end of ThenBB, and replace with conditional
8315 // branch (If-stmt)
8316 Instruction *EntryBBTI = EntryBB->getTerminator();
8317 Builder.CreateCondBr(Cond: CallBool, True: ThenBB, False: ExitBB);
8318 EntryBBTI->removeFromParent();
8319 Builder.SetInsertPoint(UI);
8320 Builder.Insert(I: EntryBBTI);
8321 UI->eraseFromParent();
8322 Builder.SetInsertPoint(ThenBB->getTerminator());
8323
8324 // return an insertion point to ExitBB.
8325 return ExitBB->getFirstInsertionPt();
8326}
8327
8328OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitCommonDirectiveExit(
8329 omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
8330 bool HasFinalize) {
8331
8332 Builder.restoreIP(IP: FinIP);
8333
8334 // If there is finalization to do, emit it before the exit call
8335 if (HasFinalize) {
8336 assert(!FinalizationStack.empty() &&
8337 "Unexpected finalization stack state!");
8338
8339 FinalizationInfo Fi = FinalizationStack.pop_back_val();
8340 assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
8341
8342 BasicBlock *FinBB = FinIP.getNodeParent();
8343 if (Error Err = Fi.mergeFiniBB(Builder, OtherFiniBB: FinBB))
8344 return std::move(Err);
8345
8346 // Exit condition: insertion point is before the terminator of the new Fini
8347 // block
8348 Builder.SetInsertPoint(FinBB->getTerminator());
8349 }
8350
8351 if (!ExitCall)
8352 return Builder.saveIP();
8353
8354 // place the Exitcall as last instruction before Finalization block terminator
8355 ExitCall->removeFromParent();
8356 Builder.Insert(I: ExitCall);
8357
8358 return ExitCall->getIterator();
8359}
8360
8361OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
8362 InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
8363 llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
8364 if (!IP.isValid())
8365 return IP;
8366
8367 IRBuilder<>::InsertPointGuard IPG(Builder);
8368
8369 // creates the following CFG structure
8370 // OMP_Entry : (MasterAddr != PrivateAddr)?
8371 // F T
8372 // | \
8373 // | copin.not.master
8374 // | /
8375 // v /
8376 // copyin.not.master.end
8377 // |
8378 // v
8379 // OMP.Entry.Next
8380
8381 BasicBlock *OMP_Entry = IP.getNodeParent();
8382 Function *CurFn = OMP_Entry->getParent();
8383 BasicBlock *CopyBegin =
8384 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master", Parent: CurFn);
8385 BasicBlock *CopyEnd = nullptr;
8386
8387 // If entry block is terminated, split to preserve the branch to following
8388 // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
8389 if (isa_and_nonnull<CondBrInst>(Val: OMP_Entry->getTerminatorOrNull())) {
8390 CopyEnd = OMP_Entry->splitBasicBlock(I: OMP_Entry->getTerminator(),
8391 BBName: "copyin.not.master.end");
8392 OMP_Entry->getTerminator()->eraseFromParent();
8393 } else {
8394 CopyEnd =
8395 BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master.end", Parent: CurFn);
8396 }
8397
8398 Builder.SetInsertPoint(OMP_Entry);
8399 Value *MasterPtr = Builder.CreatePtrToInt(V: MasterAddr, DestTy: IntPtrTy);
8400 Value *PrivatePtr = Builder.CreatePtrToInt(V: PrivateAddr, DestTy: IntPtrTy);
8401 Value *cmp = Builder.CreateICmpNE(LHS: MasterPtr, RHS: PrivatePtr);
8402 Builder.CreateCondBr(Cond: cmp, True: CopyBegin, False: CopyEnd);
8403
8404 Builder.SetInsertPoint(CopyBegin);
8405 if (BranchtoEnd)
8406 Builder.SetInsertPoint(Builder.CreateBr(Dest: CopyEnd));
8407
8408 return Builder.saveIP();
8409}
8410
8411CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
8412 Value *Size, Value *Allocator,
8413 std::string Name) {
8414 IRBuilder<>::InsertPointGuard IPG(Builder);
8415 if (!updateToLocation(Loc))
8416 return nullptr;
8417
8418 uint32_t SrcLocStrSize;
8419 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8420 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8421 Value *ThreadId = getOrCreateThreadID(Ident);
8422 Value *Args[] = {ThreadId, Size, Allocator};
8423
8424 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc);
8425
8426 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8427}
8428
8429CallInst *OpenMPIRBuilder::createOMPAlignedAlloc(const LocationDescription &Loc,
8430 Value *Align, Value *Size,
8431 Value *Allocator,
8432 std::string Name) {
8433 IRBuilder<>::InsertPointGuard IPG(Builder);
8434 if (!updateToLocation(Loc))
8435 return nullptr;
8436
8437 uint32_t SrcLocStrSize;
8438 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8439 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8440 Value *ThreadId = getOrCreateThreadID(Ident);
8441 Value *Args[] = {ThreadId, Align, Size, Allocator};
8442
8443 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_aligned_alloc);
8444
8445 return Builder.CreateCall(Callee: Fn, Args, Name);
8446}
8447
8448CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
8449 Value *Addr, Value *Allocator,
8450 std::string Name) {
8451 IRBuilder<>::InsertPointGuard IPG(Builder);
8452 if (!updateToLocation(Loc))
8453 return nullptr;
8454
8455 uint32_t SrcLocStrSize;
8456 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8457 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8458 Value *ThreadId = getOrCreateThreadID(Ident);
8459 Value *Args[] = {ThreadId, Addr, Allocator};
8460 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free);
8461 return createRuntimeFunctionCall(Callee: Fn, Args, Name);
8462}
8463
8464CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8465 Value *Size,
8466 const Twine &Name) {
8467 IRBuilder<>::InsertPointGuard IPG(Builder);
8468 updateToLocation(Loc);
8469
8470 Value *Args[] = {Size};
8471 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc_shared);
8472 CallInst *Call = Builder.CreateCall(Callee: Fn, Args, Name);
8473 Call->addRetAttr(Attr: Attribute::getWithAlignment(
8474 Context&: M.getContext(), Alignment: M.getDataLayout().getPrefTypeAlign(Ty: Int64)));
8475 return Call;
8476}
8477
8478CallInst *OpenMPIRBuilder::createOMPAllocShared(const LocationDescription &Loc,
8479 Type *VarType,
8480 const Twine &Name) {
8481 return createOMPAllocShared(
8482 Loc, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)), Name);
8483}
8484
8485CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8486 Value *Addr, Value *Size,
8487 const Twine &Name) {
8488 IRBuilder<>::InsertPointGuard IPG(Builder);
8489 updateToLocation(Loc);
8490
8491 Value *Args[] = {Addr, Size};
8492 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free_shared);
8493 return Builder.CreateCall(Callee: Fn, Args, Name);
8494}
8495
8496CallInst *OpenMPIRBuilder::createOMPFreeShared(const LocationDescription &Loc,
8497 Value *Addr, Type *VarType,
8498 const Twine &Name) {
8499 return createOMPFreeShared(
8500 Loc, Addr, Size: Builder.getInt64(C: M.getDataLayout().getTypeAllocSize(Ty: VarType)),
8501 Name);
8502}
8503
8504CallInst *OpenMPIRBuilder::createOMPInteropInit(
8505 const LocationDescription &Loc, Value *InteropVar,
8506 omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
8507 Value *DependenceAddress, bool HaveNowaitClause) {
8508 IRBuilder<>::InsertPointGuard IPG(Builder);
8509 updateToLocation(Loc);
8510
8511 uint32_t SrcLocStrSize;
8512 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8513 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8514 Value *ThreadId = getOrCreateThreadID(Ident);
8515 if (Device == nullptr)
8516 Device = Constant::getAllOnesValue(Ty: Int32);
8517 else if (Device->getType() != Int32)
8518 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8519 Constant *InteropTypeVal = ConstantInt::get(Ty: Int32, V: (int)InteropType);
8520 if (NumDependences == nullptr) {
8521 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8522 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8523 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8524 }
8525 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8526 Value *Args[] = {
8527 Ident, ThreadId, InteropVar, InteropTypeVal,
8528 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
8529
8530 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_init);
8531
8532 return createRuntimeFunctionCall(Callee: Fn, Args);
8533}
8534
8535CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
8536 const LocationDescription &Loc, Value *InteropVar, Value *Device,
8537 Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
8538 IRBuilder<>::InsertPointGuard IPG(Builder);
8539 updateToLocation(Loc);
8540
8541 uint32_t SrcLocStrSize;
8542 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8543 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8544 Value *ThreadId = getOrCreateThreadID(Ident);
8545 if (Device == nullptr)
8546 Device = Constant::getAllOnesValue(Ty: Int32);
8547 else if (Device->getType() != Int32)
8548 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8549 if (NumDependences == nullptr) {
8550 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8551 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8552 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8553 }
8554 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8555 Value *Args[] = {
8556 Ident, ThreadId, InteropVar, Device,
8557 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8558
8559 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_destroy);
8560
8561 return createRuntimeFunctionCall(Callee: Fn, Args);
8562}
8563
8564CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
8565 Value *InteropVar, Value *Device,
8566 Value *NumDependences,
8567 Value *DependenceAddress,
8568 bool HaveNowaitClause) {
8569 IRBuilder<>::InsertPointGuard IPG(Builder);
8570 updateToLocation(Loc);
8571 uint32_t SrcLocStrSize;
8572 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8573 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8574 Value *ThreadId = getOrCreateThreadID(Ident);
8575 if (Device == nullptr)
8576 Device = Constant::getAllOnesValue(Ty: Int32);
8577 else if (Device->getType() != Int32)
8578 Device = Builder.CreateIntCast(V: Device, DestTy: Int32, /*isSigned=*/true);
8579 if (NumDependences == nullptr) {
8580 NumDependences = ConstantInt::get(Ty: Int32, V: 0);
8581 PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
8582 DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
8583 }
8584 Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
8585 Value *Args[] = {
8586 Ident, ThreadId, InteropVar, Device,
8587 NumDependences, DependenceAddress, HaveNowaitClauseVal};
8588
8589 Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_use);
8590
8591 return createRuntimeFunctionCall(Callee: Fn, Args);
8592}
8593
8594CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
8595 const LocationDescription &Loc, llvm::Value *Pointer,
8596 llvm::ConstantInt *Size, const llvm::Twine &Name) {
8597 IRBuilder<>::InsertPointGuard IPG(Builder);
8598 updateToLocation(Loc);
8599
8600 uint32_t SrcLocStrSize;
8601 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8602 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8603 Value *ThreadId = getOrCreateThreadID(Ident);
8604 Constant *ThreadPrivateCache =
8605 getOrCreateInternalVariable(Ty: Int8PtrPtr, Name: Name.str());
8606 llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
8607
8608 Function *Fn =
8609 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_threadprivate_cached);
8610
8611 return createRuntimeFunctionCall(Callee: Fn, Args);
8612}
8613
8614Constant *OpenMPIRBuilder::emitKernelEnvironment(
8615 const LocationDescription &Loc,
8616 const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
8617 assert(!Attrs.MaxThreads.empty() && !Attrs.MaxTeams.empty() &&
8618 "expected num_threads and num_teams to be specified");
8619
8620 if (!updateToLocation(Loc))
8621 return nullptr;
8622
8623 uint32_t SrcLocStrSize;
8624 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
8625 Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
8626 Constant *IsSPMDVal = ConstantInt::getSigned(Ty: Int8, V: Attrs.ExecFlags);
8627 Constant *UseGenericStateMachineVal = ConstantInt::getSigned(
8628 Ty: Int8, V: Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD &&
8629 Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP);
8630 Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Ty: Int8, V: true);
8631 Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Ty: Int16, V: 0);
8632
8633 Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
8634 Function *Kernel = DebugKernelWrapper;
8635
8636 // We need to strip the debug prefix to get the correct kernel name.
8637 StringRef KernelName = Kernel->getName();
8638 const std::string DebugPrefix = "_debug__";
8639 if (KernelName.ends_with(Suffix: DebugPrefix)) {
8640 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8641 Kernel = M.getFunction(Name: KernelName);
8642 assert(Kernel && "Expected the real kernel to exist");
8643 }
8644
8645 // Manifest the launch configuration in the metadata matching the kernel
8646 // environment.
8647 if (Attrs.MinTeams.front() > 1 || Attrs.MaxTeams.front() > 0)
8648 writeTeamsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinTeams.front(),
8649 UB: Attrs.MaxTeams.front());
8650
8651 // Don't derive or write thread bounds for Bare kernels.
8652 int32_t MaxThreadsVal = Attrs.MaxThreads.front();
8653 if (Attrs.ExecFlags != omp::OMP_TGT_EXEC_MODE_BARE) {
8654 // If MaxThreads is not set and needs adjustment, select the maximum
8655 // between the default workgroup size and the MinThreads value. This is
8656 // only meaningful for targets with a known grid value (i.e. GPUs); for
8657 // other targets (e.g. host kernels) leave it unset so the runtime falls
8658 // back to its own device-specific default.
8659 if (MaxThreadsVal < 0 && UseDefaultMaxThreads && hasGridValue(T))
8660 MaxThreadsVal =
8661 std::max(a: int32_t(getGridValue(T, Kernel).GV_Default_WG_Size),
8662 b: Attrs.MinThreads.front());
8663
8664 // Generic mode runs the main thread on a warp of its own, past
8665 // thread_limit. Reserve the widest warp any target has. Not on SPIR-V,
8666 // causes problems with Level Zero.
8667 if (MaxThreadsVal > 0 &&
8668 Attrs.ExecFlags == omp::OMP_TGT_EXEC_MODE_GENERIC && hasGridValue(T) &&
8669 !T.isSPIRV())
8670 MaxThreadsVal = int32_t(
8671 std::min<int64_t>(a: int64_t(MaxThreadsVal) + 64,
8672 b: int64_t(getGridValue(T, Kernel).GV_Max_WG_Size)));
8673
8674 if (MaxThreadsVal > 0)
8675 writeThreadBoundsForKernel(T, Kernel&: *Kernel, LB: Attrs.MinThreads.front(),
8676 UB: MaxThreadsVal);
8677 }
8678
8679 Constant *MinThreads =
8680 ConstantInt::getSigned(Ty: Int32, V: Attrs.MinThreads.front());
8681 Constant *MaxThreads = ConstantInt::getSigned(Ty: Int32, V: MaxThreadsVal);
8682 Constant *MinTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MinTeams.front());
8683 Constant *MaxTeams = ConstantInt::getSigned(Ty: Int32, V: Attrs.MaxTeams.front());
8684 Constant *ReductionDataSize =
8685 ConstantInt::getSigned(Ty: Int32, V: Attrs.ReductionDataSize);
8686
8687 const DataLayout &DL = M.getDataLayout();
8688
8689 Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";
8690 Constant *DynamicEnvironmentInitializer =
8691 ConstantStruct::get(T: DynamicEnvironment, V: {DebugIndentionLevelVal});
8692 GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(
8693 M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,
8694 DynamicEnvironmentInitializer, DynamicEnvironmentName,
8695 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8696 DL.getDefaultGlobalsAddressSpace());
8697 DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8698
8699 Constant *DynamicEnvironment =
8700 DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr
8701 ? DynamicEnvironmentGV
8702 : ConstantExpr::getAddrSpaceCast(C: DynamicEnvironmentGV,
8703 Ty: DynamicEnvironmentPtr);
8704
8705 Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(
8706 T: ConfigurationEnvironment, V: {
8707 UseGenericStateMachineVal,
8708 MayUseNestedParallelismVal,
8709 IsSPMDVal,
8710 MinThreads,
8711 MaxThreads,
8712 MinTeams,
8713 MaxTeams,
8714 ReductionDataSize,
8715 });
8716 Constant *KernelEnvironmentInitializer = ConstantStruct::get(
8717 T: KernelEnvironment, V: {
8718 ConfigurationEnvironmentInitializer,
8719 Ident,
8720 DynamicEnvironment,
8721 });
8722 std::string KernelEnvironmentName =
8723 (KernelName + "_kernel_environment").str();
8724 GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
8725 M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
8726 KernelEnvironmentInitializer, KernelEnvironmentName,
8727 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
8728 DL.getDefaultGlobalsAddressSpace());
8729 KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
8730
8731 return KernelEnvironmentGV->getType() == KernelEnvironmentPtr
8732 ? KernelEnvironmentGV
8733 : ConstantExpr::getAddrSpaceCast(C: KernelEnvironmentGV,
8734 Ty: KernelEnvironmentPtr);
8735}
8736
8737OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetInit(
8738 const LocationDescription &Loc,
8739 const llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
8740 Constant *KernelEnvironment = emitKernelEnvironment(Loc, Attrs);
8741 if (!KernelEnvironment)
8742 return Loc.IP;
8743
8744 if (!updateToLocation(Loc))
8745 return Loc.IP;
8746
8747 Function *DebugKernelWrapper = Builder.GetInsertBlock()->getParent();
8748 Function *Fn = getOrCreateRuntimeFunctionPtr(
8749 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_init);
8750
8751 Value *KernelLaunchEnvironment =
8752 DebugKernelWrapper->getArg(i: DebugKernelWrapper->arg_size() - 1);
8753 Type *KernelLaunchEnvParamTy = Fn->getFunctionType()->getParamType(i: 1);
8754 KernelLaunchEnvironment =
8755 KernelLaunchEnvironment->getType() == KernelLaunchEnvParamTy
8756 ? KernelLaunchEnvironment
8757 : Builder.CreateAddrSpaceCast(V: KernelLaunchEnvironment,
8758 DestTy: KernelLaunchEnvParamTy);
8759 CallInst *ThreadKind = createRuntimeFunctionCall(
8760 Callee: Fn, Args: {KernelEnvironment, KernelLaunchEnvironment});
8761
8762 Value *ExecUserCode = Builder.CreateICmpEQ(
8763 LHS: ThreadKind, RHS: Constant::getAllOnesValue(Ty: ThreadKind->getType()),
8764 Name: "exec_user_code");
8765
8766 // ThreadKind = __kmpc_target_init(...)
8767 // if (ThreadKind == -1)
8768 // user_code
8769 // else
8770 // return;
8771
8772 auto *UI = Builder.CreateUnreachable();
8773 BasicBlock *CheckBB = UI->getParent();
8774 BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(I: UI, BBName: "user_code.entry");
8775
8776 BasicBlock *WorkerExitBB = BasicBlock::Create(
8777 Context&: CheckBB->getContext(), Name: "worker.exit", Parent: CheckBB->getParent());
8778 Builder.SetInsertPoint(WorkerExitBB);
8779 Builder.CreateRetVoid();
8780
8781 auto *CheckBBTI = CheckBB->getTerminator();
8782 Builder.SetInsertPoint(CheckBBTI);
8783 Builder.CreateCondBr(Cond: ExecUserCode, True: UI->getParent(), False: WorkerExitBB);
8784
8785 CheckBBTI->eraseFromParent();
8786 UI->eraseFromParent();
8787
8788 // Continue in the "user_code" block, see diagram above and in
8789 // openmp/libomptarget/deviceRTLs/common/include/target.h .
8790 return UserCodeEntryBB->getFirstInsertionPt();
8791}
8792
8793void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
8794 int32_t TeamsReductionDataSize) {
8795 if (!updateToLocation(Loc))
8796 return;
8797
8798 Function *Fn = getOrCreateRuntimeFunctionPtr(
8799 FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
8800
8801 createRuntimeFunctionCall(Callee: Fn, Args: {});
8802
8803 if (!TeamsReductionDataSize)
8804 return;
8805
8806 Function *Kernel = Builder.GetInsertBlock()->getParent();
8807 // We need to strip the debug prefix to get the correct kernel name.
8808 StringRef KernelName = Kernel->getName();
8809 const std::string DebugPrefix = "_debug__";
8810 if (KernelName.ends_with(Suffix: DebugPrefix))
8811 KernelName = KernelName.drop_back(N: DebugPrefix.length());
8812 auto *KernelEnvironmentGV =
8813 M.getNamedGlobal(Name: (KernelName + "_kernel_environment").str());
8814 assert(KernelEnvironmentGV && "Expected kernel environment global\n");
8815 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
8816 auto *NewInitializer = ConstantFoldInsertValueInstruction(
8817 Agg: KernelEnvironmentInitializer,
8818 Val: ConstantInt::get(Ty: Int32, V: TeamsReductionDataSize), Idxs: {0, 7});
8819 KernelEnvironmentGV->setInitializer(NewInitializer);
8820}
8821
8822static void updateNVPTXAttr(Function &Kernel, StringRef Name, int32_t Value,
8823 bool Min) {
8824 if (Kernel.hasFnAttribute(Kind: Name)) {
8825 int32_t OldLimit = Kernel.getFnAttributeAsParsedInteger(Kind: Name);
8826 Value = Min ? std::min(a: OldLimit, b: Value) : std::max(a: OldLimit, b: Value);
8827 }
8828 Kernel.addFnAttr(Kind: Name, Val: llvm::utostr(X: Value));
8829}
8830
8831std::pair<int32_t, int32_t>
8832OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {
8833 int32_t ThreadLimit =
8834 Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_thread_limit");
8835
8836 if (T.isAMDGPU()) {
8837 const auto &Attr = Kernel.getFnAttribute(Kind: "amdgpu-flat-work-group-size");
8838 if (!Attr.isValid() || !Attr.isStringAttribute())
8839 return {0, ThreadLimit};
8840 auto [LBStr, UBStr] = Attr.getValueAsString().split(Separator: ',');
8841 int32_t LB, UB;
8842 if (!llvm::to_integer(S: UBStr, Num&: UB, Base: 10))
8843 return {0, ThreadLimit};
8844 UB = ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB;
8845 if (!llvm::to_integer(S: LBStr, Num&: LB, Base: 10))
8846 return {0, UB};
8847 return {LB, UB};
8848 }
8849
8850 if (Kernel.hasFnAttribute(Kind: NVVMAttr::MaxNTID)) {
8851 int32_t UB = Kernel.getFnAttributeAsParsedInteger(Kind: NVVMAttr::MaxNTID);
8852 return {0, ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB};
8853 }
8854 return {0, ThreadLimit};
8855}
8856
8857void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,
8858 Function &Kernel, int32_t LB,
8859 int32_t UB) {
8860 Kernel.addFnAttr(Kind: "omp_target_thread_limit", Val: std::to_string(val: UB));
8861
8862 if (T.isAMDGPU()) {
8863 Kernel.addFnAttr(Kind: "amdgpu-flat-work-group-size",
8864 Val: llvm::utostr(X: LB) + "," + llvm::utostr(X: UB));
8865 return;
8866 }
8867
8868 updateNVPTXAttr(Kernel, Name: NVVMAttr::MaxNTID, Value: UB, Min: true);
8869}
8870
8871std::pair<int32_t, int32_t>
8872OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
8873 // TODO: Read from backend annotations if available.
8874 return {0, Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_num_teams")};
8875}
8876
8877void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
8878 int32_t LB, int32_t UB) {
8879 if (UB > 0) {
8880 if (T.isNVPTX())
8881 Kernel.addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: llvm::utostr(X: UB));
8882 if (T.isAMDGPU())
8883 Kernel.addFnAttr(Kind: "amdgpu-max-num-workgroups", Val: llvm::utostr(X: UB) + ",1,1");
8884 }
8885
8886 Kernel.addFnAttr(Kind: "omp_target_num_teams", Val: std::to_string(val: LB));
8887}
8888
8889void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
8890 Function *OutlinedFn) {
8891 if (Config.isTargetDevice()) {
8892 OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);
8893 // TODO: Determine if DSO local can be set to true.
8894 OutlinedFn->setDSOLocal(false);
8895 OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
8896 if (T.isAMDGCN())
8897 OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
8898 else if (T.isNVPTX())
8899 OutlinedFn->setCallingConv(CallingConv::PTX_Kernel);
8900 else if (T.isSPIRV())
8901 OutlinedFn->setCallingConv(CallingConv::SPIR_KERNEL);
8902 }
8903}
8904
8905Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,
8906 StringRef EntryFnIDName) {
8907 if (Config.isTargetDevice()) {
8908 assert(OutlinedFn && "The outlined function must exist if embedded");
8909 return OutlinedFn;
8910 }
8911
8912 return new GlobalVariable(
8913 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
8914 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnIDName);
8915}
8916
8917Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
8918 StringRef EntryFnName) {
8919 if (OutlinedFn)
8920 return OutlinedFn;
8921
8922 assert(!M.getGlobalVariable(EntryFnName, true) &&
8923 "Named kernel already exists?");
8924 return new GlobalVariable(
8925 M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
8926 Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnName);
8927}
8928
8929Error OpenMPIRBuilder::emitTargetRegionFunction(
8930 TargetRegionEntryInfo &EntryInfo,
8931 FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
8932 Function *&OutlinedFn, Constant *&OutlinedFnID) {
8933
8934 SmallString<64> EntryFnName;
8935 OffloadInfoManager.getTargetRegionEntryFnName(Name&: EntryFnName, EntryInfo);
8936
8937 if (Config.isTargetDevice() || !Config.openMPOffloadMandatory()) {
8938 Expected<Function *> CBResult = GenerateFunctionCallback(EntryFnName);
8939 if (!CBResult)
8940 return CBResult.takeError();
8941 OutlinedFn = *CBResult;
8942 } else {
8943 OutlinedFn = nullptr;
8944 }
8945
8946 // If this target outline function is not an offload entry, we don't need to
8947 // register it. This may be in the case of a false if clause, or if there are
8948 // no OpenMP targets.
8949 if (!IsOffloadEntry)
8950 return Error::success();
8951
8952 std::string EntryFnIDName =
8953 Config.isTargetDevice()
8954 ? std::string(EntryFnName)
8955 : createPlatformSpecificName(Parts: {EntryFnName, "region_id"});
8956
8957 OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFunction: OutlinedFn,
8958 EntryFnName, EntryFnIDName);
8959 return Error::success();
8960}
8961
8962Constant *OpenMPIRBuilder::registerTargetRegionFunction(
8963 TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,
8964 StringRef EntryFnName, StringRef EntryFnIDName) {
8965 if (OutlinedFn)
8966 setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
8967 auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
8968 auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
8969 OffloadInfoManager.registerTargetRegionEntryInfo(
8970 EntryInfo, Addr: EntryAddr, ID: OutlinedFnID,
8971 Flags: OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
8972 return OutlinedFnID;
8973}
8974
8975OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
8976 const LocationDescription &Loc, InsertPointTy AllocaIP,
8977 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
8978 Value *DeviceID, Value *IfCond, TargetDataInfo &Info,
8979 GenMapInfoCallbackTy GenMapInfoCB, CustomMapperCallbackTy CustomMapperCB,
8980 omp::RuntimeFunction *MapperFunc,
8981 function_ref<InsertPointOrErrorTy(InsertPointTy CodeGenIP,
8982 BodyGenTy BodyGenType)>
8983 BodyGenCB,
8984 function_ref<void(unsigned int, Value *)> DeviceAddrCB, Value *SrcLocInfo) {
8985 if (!updateToLocation(Loc))
8986 return InsertPointTy();
8987
8988 Builder.restoreIP(IP: CodeGenIP);
8989
8990 bool IsStandAlone = !BodyGenCB;
8991 MapInfosTy *MapInfo;
8992 // Generate the code for the opening of the data environment. Capture all the
8993 // arguments of the runtime call by reference because they are used in the
8994 // closing of the region.
8995 auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
8996 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
8997 MapInfo = &GenMapInfoCB(Builder.saveIP());
8998 if (Error Err = emitOffloadingArrays(
8999 AllocaIP, CodeGenIP: Builder.saveIP(), CombinedInfo&: *MapInfo, Info, CustomMapperCB,
9000 /*IsNonContiguous=*/true, DeviceAddrCB))
9001 return Err;
9002
9003 TargetDataRTArgs RTArgs;
9004 emitOffloadingArraysArgument(Builder, RTArgs, Info);
9005
9006 // Emit the number of elements in the offloading arrays.
9007 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
9008
9009 // Source location for the ident struct
9010 if (!SrcLocInfo) {
9011 uint32_t SrcLocStrSize;
9012 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
9013 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9014 }
9015
9016 SmallVector<llvm::Value *, 13> OffloadingArgs = {
9017 SrcLocInfo, DeviceID,
9018 PointerNum, RTArgs.BasePointersArray,
9019 RTArgs.PointersArray, RTArgs.SizesArray,
9020 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
9021 RTArgs.MappersArray};
9022
9023 if (IsStandAlone) {
9024 assert(MapperFunc && "MapperFunc missing for standalone target data");
9025
9026 auto TaskBodyCB = [&](Value *, Value *,
9027 IRBuilderBase::InsertPoint) -> Error {
9028 if (Info.HasNoWait) {
9029 OffloadingArgs.append(IL: {llvm::Constant::getNullValue(Ty: Int32),
9030 llvm::Constant::getNullValue(Ty: VoidPtr),
9031 llvm::Constant::getNullValue(Ty: Int32),
9032 llvm::Constant::getNullValue(Ty: VoidPtr)});
9033 }
9034
9035 createRuntimeFunctionCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: *MapperFunc),
9036 Args: OffloadingArgs);
9037
9038 if (Info.HasNoWait) {
9039 BasicBlock *OffloadContBlock =
9040 BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
9041 Function *CurFn = Builder.GetInsertBlock()->getParent();
9042 emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
9043 Builder.restoreIP(IP: Builder.saveIP());
9044 }
9045 return Error::success();
9046 };
9047
9048 bool RequiresOuterTargetTask = Info.HasNoWait;
9049 if (!RequiresOuterTargetTask)
9050 cantFail(Err: TaskBodyCB(/*DeviceID=*/nullptr, /*RTLoc=*/nullptr,
9051 /*TargetTaskAllocaIP=*/{}));
9052 else
9053 cantFail(ValOrErr: emitTargetTask(TaskBodyCB, DeviceID, RTLoc: SrcLocInfo, AllocaIP,
9054 /*Dependencies=*/{}, RTArgs, HasNoWait: Info.HasNoWait));
9055 } else {
9056 Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
9057 FnID: omp::OMPRTL___tgt_target_data_begin_mapper);
9058
9059 createRuntimeFunctionCall(Callee: BeginMapperFunc, Args: OffloadingArgs);
9060
9061 for (auto DeviceMap : Info.DevicePtrInfoMap) {
9062 if (isa<AllocaInst>(Val: DeviceMap.second.second)) {
9063 auto *LI =
9064 Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DeviceMap.second.first);
9065 Builder.CreateStore(Val: LI, Ptr: DeviceMap.second.second);
9066 }
9067 }
9068
9069 // If device pointer privatization is required, emit the body of the
9070 // region here. It will have to be duplicated: with and without
9071 // privatization.
9072 InsertPointOrErrorTy AfterIP =
9073 BodyGenCB(Builder.saveIP(), BodyGenTy::Priv);
9074 if (!AfterIP)
9075 return AfterIP.takeError();
9076 Builder.restoreIP(IP: *AfterIP);
9077 }
9078 return Error::success();
9079 };
9080
9081 // If we need device pointer privatization, we need to emit the body of the
9082 // region with no privatization in the 'else' branch of the conditional.
9083 // Otherwise, we don't have to do anything.
9084 auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9085 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
9086 InsertPointOrErrorTy AfterIP =
9087 BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv);
9088 if (!AfterIP)
9089 return AfterIP.takeError();
9090 Builder.restoreIP(IP: *AfterIP);
9091 return Error::success();
9092 };
9093
9094 // Generate code for the closing of the data region.
9095 auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9096 ArrayRef<BasicBlock *> DeallocBlocks) {
9097 TargetDataRTArgs RTArgs;
9098 Info.EmitDebug = !MapInfo->Names.empty();
9099 emitOffloadingArraysArgument(Builder, RTArgs, Info, /*ForEndCall=*/true);
9100
9101 // Emit the number of elements in the offloading arrays.
9102 Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
9103
9104 // Source location for the ident struct
9105 if (!SrcLocInfo) {
9106 uint32_t SrcLocStrSize;
9107 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
9108 SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9109 }
9110
9111 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
9112 PointerNum, RTArgs.BasePointersArray,
9113 RTArgs.PointersArray, RTArgs.SizesArray,
9114 RTArgs.MapTypesArray, RTArgs.MapNamesArray,
9115 RTArgs.MappersArray};
9116 Function *EndMapperFunc =
9117 getOrCreateRuntimeFunctionPtr(FnID: omp::OMPRTL___tgt_target_data_end_mapper);
9118
9119 createRuntimeFunctionCall(Callee: EndMapperFunc, Args: OffloadingArgs);
9120 return Error::success();
9121 };
9122
9123 // We don't have to do anything to close the region if the if clause evaluates
9124 // to false.
9125 auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
9126 ArrayRef<BasicBlock *> DeallocBlocks) {
9127 return Error::success();
9128 };
9129
9130 Error Err = [&]() -> Error {
9131 if (BodyGenCB) {
9132 Error Err = [&]() {
9133 if (IfCond)
9134 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: BeginElseGen, AllocaIP);
9135 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9136 }();
9137
9138 if (Err)
9139 return Err;
9140
9141 // If we don't require privatization of device pointers, we emit the body
9142 // in between the runtime calls. This avoids duplicating the body code.
9143 InsertPointOrErrorTy AfterIP =
9144 BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
9145 if (!AfterIP)
9146 return AfterIP.takeError();
9147 restoreIPandDebugLoc(Builder, IP: *AfterIP);
9148
9149 if (IfCond)
9150 return emitIfClause(Cond: IfCond, ThenGen: EndThenGen, ElseGen: EndElseGen, AllocaIP);
9151 return EndThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9152 }
9153 if (IfCond)
9154 return emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: EndElseGen, AllocaIP);
9155 return BeginThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
9156 }();
9157
9158 if (Err)
9159 return Err;
9160
9161 return Builder.saveIP();
9162}
9163
9164FunctionCallee
9165OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,
9166 bool IsGPUDistribute) {
9167 assert((IVSize == 32 || IVSize == 64) &&
9168 "IV size is not compatible with the omp runtime");
9169 RuntimeFunction Name;
9170 if (IsGPUDistribute)
9171 Name = IVSize == 32
9172 ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
9173 : omp::OMPRTL___kmpc_distribute_static_init_4u)
9174 : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
9175 : omp::OMPRTL___kmpc_distribute_static_init_8u);
9176 else
9177 Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
9178 : omp::OMPRTL___kmpc_for_static_init_4u)
9179 : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
9180 : omp::OMPRTL___kmpc_for_static_init_8u);
9181
9182 return getOrCreateRuntimeFunction(M, FnID: Name);
9183}
9184
9185FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,
9186 bool IVSigned) {
9187 assert((IVSize == 32 || IVSize == 64) &&
9188 "IV size is not compatible with the omp runtime");
9189 RuntimeFunction Name = IVSize == 32
9190 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
9191 : omp::OMPRTL___kmpc_dispatch_init_4u)
9192 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
9193 : omp::OMPRTL___kmpc_dispatch_init_8u);
9194
9195 return getOrCreateRuntimeFunction(M, FnID: Name);
9196}
9197
9198FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,
9199 bool IVSigned) {
9200 assert((IVSize == 32 || IVSize == 64) &&
9201 "IV size is not compatible with the omp runtime");
9202 RuntimeFunction Name = IVSize == 32
9203 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
9204 : omp::OMPRTL___kmpc_dispatch_next_4u)
9205 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
9206 : omp::OMPRTL___kmpc_dispatch_next_8u);
9207
9208 return getOrCreateRuntimeFunction(M, FnID: Name);
9209}
9210
9211FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,
9212 bool IVSigned) {
9213 assert((IVSize == 32 || IVSize == 64) &&
9214 "IV size is not compatible with the omp runtime");
9215 RuntimeFunction Name = IVSize == 32
9216 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
9217 : omp::OMPRTL___kmpc_dispatch_fini_4u)
9218 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
9219 : omp::OMPRTL___kmpc_dispatch_fini_8u);
9220
9221 return getOrCreateRuntimeFunction(M, FnID: Name);
9222}
9223
9224FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() {
9225 return getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_dispatch_deinit);
9226}
9227
9228static void FixupDebugInfoForOutlinedFunction(
9229 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Function *Func,
9230 DenseMap<Value *, std::tuple<Value *, unsigned>> &ValueReplacementMap) {
9231
9232 DISubprogram *NewSP = Func->getSubprogram();
9233 if (!NewSP)
9234 return;
9235
9236 SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
9237
9238 auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
9239 DILocalVariable *&NewVar = RemappedVariables[OldVar];
9240 // Only use cached variable if the arg number matches. This is important
9241 // so that DIVariable created for privatized variables are not discarded.
9242 if (NewVar && (arg == NewVar->getArg()))
9243 return NewVar;
9244
9245 NewVar = llvm::DILocalVariable::get(
9246 Context&: Builder.getContext(), Scope: OldVar->getScope(), Name: OldVar->getName(),
9247 File: OldVar->getFile(), Line: OldVar->getLine(), Type: OldVar->getType(), Arg: arg,
9248 Flags: OldVar->getFlags(), AlignInBits: OldVar->getAlignInBits(), Annotations: OldVar->getAnnotations());
9249 return NewVar;
9250 };
9251
9252 auto UpdateDebugRecord = [&](auto *DR) {
9253 DILocalVariable *OldVar = DR->getVariable();
9254 unsigned ArgNo = 0;
9255 for (auto Loc : DR->location_ops()) {
9256 auto Iter = ValueReplacementMap.find(Loc);
9257 if (Iter != ValueReplacementMap.end()) {
9258 DR->replaceVariableLocationOp(Loc, std::get<0>(Iter->second));
9259 ArgNo = std::get<1>(Iter->second) + 1;
9260 }
9261 }
9262 if (ArgNo != 0)
9263 DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
9264 };
9265
9266 SmallVector<DbgVariableRecord *, 4> DVRsToDelete;
9267 auto MoveDebugRecordToCorrectBlock = [&](DbgVariableRecord *DVR) {
9268 if (DVR->getNumVariableLocationOps() != 1u) {
9269 DVR->setKillLocation();
9270 return;
9271 }
9272 Value *Loc = DVR->getVariableLocationOp(OpIdx: 0u);
9273 BasicBlock *CurBB = DVR->getParent();
9274 BasicBlock *RequiredBB = nullptr;
9275
9276 if (Instruction *LocInst = dyn_cast<Instruction>(Val: Loc))
9277 RequiredBB = LocInst->getParent();
9278 else if (isa<llvm::Argument>(Val: Loc))
9279 RequiredBB = &DVR->getFunction()->getEntryBlock();
9280
9281 if (RequiredBB && RequiredBB != CurBB) {
9282 assert(!RequiredBB->empty());
9283 RequiredBB->insertDbgRecordBefore(DR: DVR->clone(),
9284 Here: RequiredBB->back().getIterator());
9285 DVRsToDelete.push_back(Elt: DVR);
9286 }
9287 };
9288
9289 // The location and scope of variable records still point to
9290 // the parent function of the target region. Update them.
9291 for (Instruction &I : instructions(F: Func)) {
9292 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
9293 UpdateDebugRecord(&DVR);
9294 MoveDebugRecordToCorrectBlock(&DVR);
9295 }
9296 }
9297 for (auto *DVR : DVRsToDelete)
9298 DVR->getMarker()->MarkedInstr->dropOneDbgRecord(I: DVR);
9299 // An extra argument is passed to the device. Create the debug data for it.
9300 if (OMPBuilder.Config.isTargetDevice()) {
9301 DICompileUnit *CU = NewSP->getUnit();
9302 Module *M = Func->getParent();
9303 DIBuilder DB(*M, true, CU);
9304 DIType *VoidPtrTy =
9305 DB.createQualifiedType(Tag: dwarf::DW_TAG_pointer_type, FromTy: nullptr);
9306 unsigned ArgNo = Func->arg_size();
9307 DILocalVariable *Var = DB.createParameterVariable(
9308 Scope: NewSP, Name: "dyn_ptr", ArgNo, File: NewSP->getFile(), /*LineNo=*/0, Ty: VoidPtrTy,
9309 /*AlwaysPreserve=*/false, Flags: DINode::DIFlags::FlagArtificial);
9310 auto Loc = DILocation::get(Context&: Func->getContext(), Line: 0, Column: 0, Scope: NewSP, InlinedAt: 0);
9311 Argument *LastArg = Func->getArg(i: Func->arg_size() - 1);
9312 DB.insertDeclare(Storage: LastArg, VarInfo: Var, Expr: DB.createExpression(), DL: Loc,
9313 InsertAtEnd: &(*Func->begin()));
9314 }
9315}
9316
9317static Value *removeASCastIfPresent(Value *V) {
9318 if (Operator::getOpcode(V) == Instruction::AddrSpaceCast)
9319 return cast<Operator>(Val: V)->getOperand(i: 0);
9320 return V;
9321}
9322
9323static Expected<Function *> createOutlinedFunction(
9324 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
9325 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9326 StringRef FuncName, SmallVectorImpl<Value *> &Inputs,
9327 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9328 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB,
9329 DebugLoc OutlinedFnLoc) {
9330 SmallVector<Type *> ParameterTypes;
9331 if (OMPBuilder.Config.isTargetDevice()) {
9332 // All parameters to target devices are passed as pointers
9333 // or i64. This assumes 64-bit address spaces/pointers.
9334 for (auto &Arg : Inputs)
9335 ParameterTypes.push_back(Elt: Arg->getType()->isPointerTy()
9336 ? Arg->getType()
9337 : Type::getInt64Ty(C&: Builder.getContext()));
9338 } else {
9339 for (auto &Arg : Inputs)
9340 ParameterTypes.push_back(Elt: Arg->getType());
9341 }
9342
9343 // The implicit dyn_ptr argument is always the last parameter on both host
9344 // and device so the argument counts match without runtime manipulation.
9345 auto *PtrTy = PointerType::getUnqual(C&: Builder.getContext());
9346 ParameterTypes.push_back(Elt: PtrTy);
9347
9348 auto BB = Builder.GetInsertBlock();
9349 auto M = BB->getModule();
9350 auto FuncType = FunctionType::get(Result: Builder.getVoidTy(), Params: ParameterTypes,
9351 /*isVarArg*/ false);
9352 auto Func =
9353 Function::Create(Ty: FuncType, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
9354
9355 // Forward target-cpu and target-features function attributes from the
9356 // original function to the new outlined function.
9357 Function *ParentFn = Builder.GetInsertBlock()->getParent();
9358
9359 auto TargetCpuAttr = ParentFn->getFnAttribute(Kind: "target-cpu");
9360 if (TargetCpuAttr.isStringAttribute())
9361 Func->addFnAttr(Attr: TargetCpuAttr);
9362
9363 auto TargetFeaturesAttr = ParentFn->getFnAttribute(Kind: "target-features");
9364 if (TargetFeaturesAttr.isStringAttribute())
9365 Func->addFnAttr(Attr: TargetFeaturesAttr);
9366
9367 if (OMPBuilder.Config.isTargetDevice()) {
9368 Value *ExecMode =
9369 OMPBuilder.emitKernelExecutionMode(KernelName: FuncName, Mode: DefaultAttrs.ExecFlags);
9370 OMPBuilder.emitUsed(Name: "llvm.compiler.used", List: {ExecMode});
9371 }
9372
9373 // Save insert point.
9374 IRBuilder<>::InsertPointGuard IPG(Builder);
9375 // We will generate the entries in the outlined function but the debug
9376 // location is still pointing to the parent function, which is the wrong
9377 // scope. OutlinedFnLoc, when the caller provides one, is the same source
9378 // position scoped to the subprogram that will be attached to the outlined
9379 // function, so it is what everything emitted below needs.
9380 Builder.SetCurrentDebugLocation(OutlinedFnLoc);
9381
9382 // Generate the region into the function.
9383 BasicBlock *EntryBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: Func);
9384 Builder.SetInsertPoint(EntryBB);
9385
9386 // Insert target init call in the device compilation pass. On the host
9387 // (e.g. a non-GPU offload target), there is no runtime init/deinit
9388 // sequence, but the runtime still needs a '<kernel>_kernel_environment'
9389 // global to know how the kernel was configured, so emit it directly.
9390 if (OMPBuilder.Config.isTargetDevice())
9391 Builder.restoreIP(IP: OMPBuilder.createTargetInit(Loc: Builder, Attrs: DefaultAttrs));
9392 else
9393 OMPBuilder.emitKernelEnvironment(Loc: Builder, Attrs: DefaultAttrs);
9394
9395 BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();
9396
9397 // As we embed the user code in the middle of our target region after we
9398 // generate entry code, we must move what allocas we can into the entry
9399 // block to avoid possible breaking optimisations
9400 OMPBuilder.ConstantAllocaRaiseCandidates.emplace_back(Args&: Func);
9401
9402 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "target.exit");
9403 BasicBlock *OutlinedBodyBB =
9404 splitBB(Builder, /*CreateBranch=*/true, Name: "outlined.body");
9405 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
9406 CBFunc(Builder.saveIP(), OutlinedBodyBB->begin(), ExitBB);
9407 if (!AfterIP)
9408 return AfterIP.takeError();
9409 Builder.SetInsertPoint(ExitBB);
9410 // The body callback builds the body with its own IRBuilder and cannot reach
9411 // this one directly. But a body holding another OpenMP construct, a nested
9412 // parallel say, calls OpenMPIRBuilder::createParallel, and that can leave
9413 // this Builder pointing at the wrong debug location, or at none at all. The
9414 // epilogue below belongs to the target construct rather than to whatever the
9415 // body emitted last, so re-establish the location the prologue was emitted
9416 // with.
9417 Builder.SetCurrentDebugLocation(OutlinedFnLoc);
9418
9419 // Insert target deinit call in the device compilation pass.
9420 if (OMPBuilder.Config.isTargetDevice())
9421 OMPBuilder.createTargetDeinit(Loc: Builder);
9422
9423 // Insert return instruction.
9424 Builder.CreateRetVoid();
9425
9426 // New Alloca IP at entry point of created device function.
9427 Builder.SetInsertPoint(EntryBB->getFirstNonPHIIt());
9428 auto AllocaIP = Builder.saveIP();
9429
9430 Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrPseudoProbe());
9431
9432 // Do not include the artificial dyn_ptr argument.
9433 const auto &ArgRange = make_range(x: Func->arg_begin(), y: Func->arg_end() - 1);
9434
9435 DenseMap<Value *, std::tuple<Value *, unsigned>> ValueReplacementMap;
9436
9437 auto ReplaceValue = [](Value *Input, Value *InputCopy, Function *Func) {
9438 // Things like GEP's can come in the form of Constants. Constants and
9439 // ConstantExpr's do not have access to the knowledge of what they're
9440 // contained in, so we must dig a little to find an instruction so we
9441 // can tell if they're used inside of the function we're outlining. We
9442 // also replace the original constant expression with a new instruction
9443 // equivalent; an instruction as it allows easy modification in the
9444 // following loop, as we can now know the constant (instruction) is
9445 // owned by our target function and replaceUsesOfWith can now be invoked
9446 // on it (cannot do this with constants it seems). A brand new one also
9447 // allows us to be cautious as it is perhaps possible the old expression
9448 // was used inside of the function but exists and is used externally
9449 // (unlikely by the nature of a Constant, but still).
9450 // NOTE: We cannot remove dead constants that have been rewritten to
9451 // instructions at this stage, we run the risk of breaking later lowering
9452 // by doing so as we could still be in the process of lowering the module
9453 // from MLIR to LLVM-IR and the MLIR lowering may still require the original
9454 // constants we have created rewritten versions of.
9455 if (auto *Const = dyn_cast<Constant>(Val: Input))
9456 convertUsersOfConstantsToInstructions(Consts: Const, RestrictToFunc: Func, RemoveDeadConstants: false);
9457
9458 // Collect users before iterating over them to avoid invalidating the
9459 // iteration in case a user uses Input more than once (e.g. a call
9460 // instruction).
9461 SetVector<User *> Users(Input->users().begin(), Input->users().end());
9462 // Collect all the instructions
9463 for (User *User : make_early_inc_range(Range&: Users))
9464 if (auto *Instr = dyn_cast<Instruction>(Val: User))
9465 if (Instr->getFunction() == Func)
9466 Instr->replaceUsesOfWith(From: Input, To: InputCopy);
9467 };
9468
9469 SmallVector<std::pair<Value *, Value *>> DeferredReplacement;
9470
9471 // Rewrite uses of input valus to parameters.
9472 for (auto InArg : zip(t&: Inputs, u: ArgRange)) {
9473 Value *Input = std::get<0>(t&: InArg);
9474 Argument &Arg = std::get<1>(t&: InArg);
9475 Value *InputCopy = nullptr;
9476
9477 llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP = ArgAccessorFuncCB(
9478 Arg, Input, InputCopy, AllocaIP, Builder.saveIP(), ExitBB->begin());
9479 if (!AfterIP)
9480 return AfterIP.takeError();
9481 Builder.restoreIP(IP: *AfterIP);
9482 ValueReplacementMap[Input] = std::make_tuple(args&: InputCopy, args: Arg.getArgNo());
9483
9484 // In certain cases a Global may be set up for replacement, however, this
9485 // Global may be used in multiple arguments to the kernel, just segmented
9486 // apart, for example, if we have a global array, that is sectioned into
9487 // multiple mappings (technically not legal in OpenMP, but there is a case
9488 // in Fortran for Common Blocks where this is neccesary), we will end up
9489 // with GEP's into this array inside the kernel, that refer to the Global
9490 // but are technically separate arguments to the kernel for all intents and
9491 // purposes. If we have mapped a segment that requires a GEP into the 0-th
9492 // index, it will fold into an referal to the Global, if we then encounter
9493 // this folded GEP during replacement all of the references to the
9494 // Global in the kernel will be replaced with the argument we have generated
9495 // that corresponds to it, including any other GEP's that refer to the
9496 // Global that may be other arguments. This will invalidate all of the other
9497 // preceding mapped arguments that refer to the same global that may be
9498 // separate segments. To prevent this, we defer global processing until all
9499 // other processing has been performed.
9500 if (llvm::isa<llvm::GlobalValue, llvm::GlobalObject, llvm::GlobalVariable>(
9501 Val: removeASCastIfPresent(V: Input))) {
9502 DeferredReplacement.push_back(Elt: std::make_pair(x&: Input, y&: InputCopy));
9503 continue;
9504 }
9505
9506 if (isa<ConstantData>(Val: Input))
9507 continue;
9508
9509 ReplaceValue(Input, InputCopy, Func);
9510 }
9511
9512 // Replace all of our deferred Input values, currently just Globals.
9513 for (auto Deferred : DeferredReplacement)
9514 ReplaceValue(std::get<0>(in&: Deferred), std::get<1>(in&: Deferred), Func);
9515
9516 FixupDebugInfoForOutlinedFunction(OMPBuilder, Builder, Func,
9517 ValueReplacementMap);
9518 return Func;
9519}
9520/// Given a task descriptor, TaskWithPrivates, return the pointer to the block
9521/// of pointers containing shared data between the parent task and the created
9522/// task.
9523static LoadInst *loadSharedDataFromTaskDescriptor(OpenMPIRBuilder &OMPIRBuilder,
9524 IRBuilderBase &Builder,
9525 Value *TaskWithPrivates,
9526 Type *TaskWithPrivatesTy) {
9527
9528 Type *TaskTy = OMPIRBuilder.Task;
9529 LLVMContext &Ctx = Builder.getContext();
9530 Value *TaskT =
9531 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 0);
9532 Value *Shareds = TaskT;
9533 // TaskWithPrivatesTy can be one of the following
9534 // 1. %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9535 // %struct.privates }
9536 // 2. %struct.kmp_task_ompbuilder_t ;; This is simply TaskTy
9537 //
9538 // In the former case, that is when TaskWithPrivatesTy != TaskTy,
9539 // its first member has to be the task descriptor. TaskTy is the type of the
9540 // task descriptor. TaskT is the pointer to the task descriptor. Loading the
9541 // first member of TaskT, gives us the pointer to shared data.
9542 if (TaskWithPrivatesTy != TaskTy)
9543 Shareds = Builder.CreateStructGEP(Ty: TaskTy, Ptr: TaskT, Idx: 0);
9544 return Builder.CreateLoad(Ty: PointerType::getUnqual(C&: Ctx), Ptr: Shareds);
9545}
9546/// Create an entry point for a target task with the following.
9547/// It'll have the following signature
9548/// void @.omp_target_task_proxy_func(i32 %thread.id, ptr %task)
9549/// This function is called from emitTargetTask once the
9550/// code to launch the target kernel has been outlined already.
9551/// NumOffloadingArrays is the number of offloading arrays that we need to copy
9552/// into the task structure so that the deferred target task can access this
9553/// data even after the stack frame of the generating task has been rolled
9554/// back. Offloading arrays contain base pointers, pointers, sizes etc
9555/// of the data that the target kernel will access. These in effect are the
9556/// non-empty arrays of pointers held by OpenMPIRBuilder::TargetDataRTArgs.
9557static Function *emitTargetTaskProxyFunction(
9558 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI,
9559 StructType *PrivatesTy, StructType *TaskWithPrivatesTy,
9560 const size_t NumOffloadingArrays, const int SharedArgsOperandNo) {
9561
9562 // If NumOffloadingArrays is non-zero, PrivatesTy better not be nullptr.
9563 // This is because PrivatesTy is the type of the structure in which
9564 // we pass the offloading arrays to the deferred target task.
9565 assert((!NumOffloadingArrays || PrivatesTy) &&
9566 "PrivatesTy cannot be nullptr when there are offloadingArrays"
9567 "to privatize");
9568
9569 Module &M = OMPBuilder.M;
9570 // KernelLaunchFunction is the target launch function, i.e.
9571 // the function that sets up kernel arguments and calls
9572 // __tgt_target_kernel to launch the kernel on the device.
9573 //
9574 Function *KernelLaunchFunction = StaleCI->getCalledFunction();
9575
9576 // StaleCI is the CallInst which is the call to the outlined
9577 // target kernel launch function. If there are local live-in values
9578 // that the outlined function uses then these are aggregated into a structure
9579 // which is passed as the second argument. If there are no local live-in
9580 // values or if all values used by the outlined kernel are global variables,
9581 // then there's only one argument, the threadID. So, StaleCI can be
9582 //
9583 // %structArg = alloca { ptr, ptr }, align 8
9584 // %gep_ = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 0
9585 // store ptr %20, ptr %gep_, align 8
9586 // %gep_8 = getelementptr { ptr, ptr }, ptr %structArg, i32 0, i32 1
9587 // store ptr %21, ptr %gep_8, align 8
9588 // call void @_QQmain..omp_par.1(i32 %global.tid.val6, ptr %structArg)
9589 //
9590 // OR
9591 //
9592 // call void @_QQmain..omp_par.1(i32 %global.tid.val6)
9593 LLVMContext &Ctx = StaleCI->getParent()->getContext();
9594
9595 Type *ThreadIDTy = Type::getInt32Ty(C&: Ctx);
9596 Type *TaskPtrTy = OMPBuilder.TaskPtr;
9597 [[maybe_unused]] Type *TaskTy = OMPBuilder.Task;
9598
9599 auto ProxyFnTy =
9600 FunctionType::get(Result: Builder.getVoidTy(), Params: {ThreadIDTy, TaskPtrTy},
9601 /* isVarArg */ false);
9602 auto ProxyFn = Function::Create(Ty: ProxyFnTy, Linkage: GlobalValue::InternalLinkage,
9603 N: ".omp_target_task_proxy_func", M);
9604 Value *ThreadId = ProxyFn->getArg(i: 0);
9605 Value *TaskWithPrivates = ProxyFn->getArg(i: 1);
9606 ThreadId->setName("thread.id");
9607 TaskWithPrivates->setName("task");
9608
9609 bool HasShareds = SharedArgsOperandNo > 0;
9610 bool HasOffloadingArrays = NumOffloadingArrays > 0;
9611 IRBuilder<>::InsertPointGuard IPG(Builder);
9612 BasicBlock *EntryBB =
9613 BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: ProxyFn);
9614 Builder.SetInsertPoint(EntryBB);
9615 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
9616
9617 SmallVector<Value *> KernelLaunchArgs;
9618 KernelLaunchArgs.reserve(N: StaleCI->arg_size());
9619 KernelLaunchArgs.push_back(Elt: ThreadId);
9620
9621 if (HasOffloadingArrays) {
9622 assert(TaskTy != TaskWithPrivatesTy &&
9623 "If there are offloading arrays to pass to the target"
9624 "TaskTy cannot be the same as TaskWithPrivatesTy");
9625 (void)TaskTy;
9626 Value *Privates =
9627 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskWithPrivates, Idx: 1);
9628 for (unsigned int i = 0; i < NumOffloadingArrays; ++i)
9629 KernelLaunchArgs.push_back(
9630 Elt: Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i));
9631 }
9632
9633 if (HasShareds) {
9634 auto *ArgStructAlloca =
9635 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgsOperandNo));
9636 assert(ArgStructAlloca &&
9637 "Unable to find the alloca instruction corresponding to arguments "
9638 "for extracted function");
9639 auto *ArgStructType = cast<StructType>(Val: ArgStructAlloca->getAllocatedType());
9640 std::optional<TypeSize> ArgAllocSize =
9641 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9642 assert(ArgStructType && ArgAllocSize &&
9643 "Unable to determine size of arguments for extracted function");
9644 uint64_t StructSize = ArgAllocSize->getFixedValue();
9645
9646 AllocaInst *NewArgStructAlloca =
9647 Builder.CreateAlloca(Ty: ArgStructType, ArraySize: nullptr, Name: "structArg");
9648
9649 Value *SharedsSize = Builder.getInt64(C: StructSize);
9650
9651 LoadInst *LoadShared = loadSharedDataFromTaskDescriptor(
9652 OMPIRBuilder&: OMPBuilder, Builder, TaskWithPrivates, TaskWithPrivatesTy);
9653
9654 Builder.CreateMemCpy(
9655 Dst: NewArgStructAlloca, DstAlign: NewArgStructAlloca->getAlign(), Src: LoadShared,
9656 SrcAlign: LoadShared->getPointerAlignment(DL: M.getDataLayout()), Size: SharedsSize);
9657 KernelLaunchArgs.push_back(Elt: NewArgStructAlloca);
9658 }
9659 OMPBuilder.createRuntimeFunctionCall(Callee: KernelLaunchFunction, Args: KernelLaunchArgs);
9660 Builder.CreateRetVoid();
9661 return ProxyFn;
9662}
9663static Type *getOffloadingArrayType(Value *V) {
9664
9665 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: V))
9666 return GEP->getSourceElementType();
9667 if (auto *Alloca = dyn_cast<AllocaInst>(Val: V))
9668 return Alloca->getAllocatedType();
9669
9670 llvm_unreachable("Unhandled Instruction type");
9671 return nullptr;
9672}
9673// This function returns a struct that has at most two members.
9674// The first member is always %struct.kmp_task_ompbuilder_t, that is the task
9675// descriptor. The second member, if needed, is a struct containing arrays
9676// that need to be passed to the offloaded target kernel. For example,
9677// if .offload_baseptrs, .offload_ptrs and .offload_sizes have to be passed to
9678// the target kernel and their types are [3 x ptr], [3 x ptr] and [3 x i64]
9679// respectively, then the types created by this function are
9680//
9681// %struct.privates = type { [3 x ptr], [3 x ptr], [3 x i64] }
9682// %struct.task_with_privates = type { %struct.kmp_task_ompbuilder_t,
9683// %struct.privates }
9684// %struct.task_with_privates is returned by this function.
9685// If there aren't any offloading arrays to pass to the target kernel,
9686// %struct.kmp_task_ompbuilder_t is returned.
9687static StructType *
9688createTaskWithPrivatesTy(OpenMPIRBuilder &OMPIRBuilder,
9689 ArrayRef<Value *> OffloadingArraysToPrivatize) {
9690
9691 if (OffloadingArraysToPrivatize.empty())
9692 return OMPIRBuilder.Task;
9693
9694 SmallVector<Type *, 4> StructFieldTypes;
9695 for (Value *V : OffloadingArraysToPrivatize) {
9696 assert(V->getType()->isPointerTy() &&
9697 "Expected pointer to array to privatize. Got a non-pointer value "
9698 "instead");
9699 Type *ArrayTy = getOffloadingArrayType(V);
9700 assert(ArrayTy && "ArrayType cannot be nullptr");
9701 StructFieldTypes.push_back(Elt: ArrayTy);
9702 }
9703 StructType *PrivatesStructTy =
9704 StructType::create(Elements: StructFieldTypes, Name: "struct.privates");
9705 return StructType::create(Elements: {OMPIRBuilder.Task, PrivatesStructTy},
9706 Name: "struct.task_with_privates");
9707}
9708static Error emitTargetOutlinedFunction(
9709 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry,
9710 TargetRegionEntryInfo &EntryInfo,
9711 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
9712 Function *&OutlinedFn, Constant *&OutlinedFnID,
9713 SmallVectorImpl<Value *> &Inputs,
9714 OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
9715 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB,
9716 DebugLoc OutlinedFnLoc) {
9717
9718 OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
9719 [&](StringRef EntryFnName) {
9720 return createOutlinedFunction(OMPBuilder, Builder, DefaultAttrs,
9721 FuncName: EntryFnName, Inputs, CBFunc,
9722 ArgAccessorFuncCB, OutlinedFnLoc);
9723 };
9724
9725 return OMPBuilder.emitTargetRegionFunction(
9726 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
9727 OutlinedFnID);
9728}
9729
9730OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
9731 TargetTaskBodyCallbackTy TaskBodyCB, Value *DeviceID, Value *RTLoc,
9732 OpenMPIRBuilder::InsertPointTy AllocaIP,
9733 const DependenciesInfo &Dependencies, const TargetDataRTArgs &RTArgs,
9734 bool HasNoWait) {
9735
9736 // The following explains the code-gen scenario for the `target` directive. A
9737 // similar scneario is followed for other device-related directives (e.g.
9738 // `target enter data`) but in similar fashion since we only need to emit task
9739 // that encapsulates the proper runtime call.
9740 //
9741 // When we arrive at this function, the target region itself has been
9742 // outlined into the function OutlinedFn.
9743 // So at ths point, for
9744 // --------------------------------------------------------------
9745 // void user_code_that_offloads(...) {
9746 // omp target depend(..) map(from:a) map(to:b) private(i)
9747 // do i = 1, 10
9748 // a(i) = b(i) + n
9749 // }
9750 //
9751 // --------------------------------------------------------------
9752 //
9753 // we have
9754 //
9755 // --------------------------------------------------------------
9756 //
9757 // void user_code_that_offloads(...) {
9758 // %.offload_baseptrs = alloca [2 x ptr], align 8
9759 // %.offload_ptrs = alloca [2 x ptr], align 8
9760 // %.offload_mappers = alloca [2 x ptr], align 8
9761 // ;; target region has been outlined and now we need to
9762 // ;; offload to it via a target task.
9763 // }
9764 // void outlined_device_function(ptr a, ptr b, ptr n) {
9765 // n = *n_ptr;
9766 // do i = 1, 10
9767 // a(i) = b(i) + n
9768 // }
9769 //
9770 // We have to now do the following
9771 // (i) Make an offloading call to outlined_device_function using the OpenMP
9772 // RTL. See 'kernel_launch_function' in the pseudo code below. This is
9773 // emitted by emitKernelLaunch
9774 // (ii) Create a task entry point function that calls kernel_launch_function
9775 // and is the entry point for the target task. See
9776 // '@.omp_target_task_proxy_func in the pseudocode below.
9777 // (iii) Create a task with the task entry point created in (ii)
9778 //
9779 // That is we create the following
9780 // struct task_with_privates {
9781 // struct kmp_task_ompbuilder_t task_struct;
9782 // struct privates {
9783 // [2 x ptr] ; baseptrs
9784 // [2 x ptr] ; ptrs
9785 // [2 x i64] ; sizes
9786 // }
9787 // }
9788 // void user_code_that_offloads(...) {
9789 // %.offload_baseptrs = alloca [2 x ptr], align 8
9790 // %.offload_ptrs = alloca [2 x ptr], align 8
9791 // %.offload_sizes = alloca [2 x i64], align 8
9792 //
9793 // %structArg = alloca { ptr, ptr, ptr }, align 8
9794 // %strucArg[0] = a
9795 // %strucArg[1] = b
9796 // %strucArg[2] = &n
9797 //
9798 // target_task_with_privates = @__kmpc_omp_target_task_alloc(...,
9799 // sizeof(kmp_task_ompbuilder_t),
9800 // sizeof(structArg),
9801 // @.omp_target_task_proxy_func,
9802 // ...)
9803 // memcpy(target_task_with_privates->task_struct->shareds, %structArg,
9804 // sizeof(structArg))
9805 // memcpy(target_task_with_privates->privates->baseptrs,
9806 // offload_baseptrs, sizeof(offload_baseptrs)
9807 // memcpy(target_task_with_privates->privates->ptrs,
9808 // offload_ptrs, sizeof(offload_ptrs)
9809 // memcpy(target_task_with_privates->privates->sizes,
9810 // offload_sizes, sizeof(offload_sizes)
9811 // dependencies_array = ...
9812 // ;; if nowait not present
9813 // call @__kmpc_omp_wait_deps(..., dependencies_array)
9814 // call @__kmpc_omp_task_begin_if0(...)
9815 // call @ @.omp_target_task_proxy_func(i32 thread_id, ptr
9816 // %target_task_with_privates)
9817 // call @__kmpc_omp_task_complete_if0(...)
9818 // }
9819 //
9820 // define internal void @.omp_target_task_proxy_func(i32 %thread.id,
9821 // ptr %task) {
9822 // %structArg = alloca {ptr, ptr, ptr}
9823 // %task_ptr = getelementptr(%task, 0, 0)
9824 // %shared_data = load (getelementptr %task_ptr, 0, 0)
9825 // mempcy(%structArg, %shared_data, sizeof(%structArg))
9826 //
9827 // %offloading_arrays = getelementptr(%task, 0, 1)
9828 // %offload_baseptrs = getelementptr(%offloading_arrays, 0, 0)
9829 // %offload_ptrs = getelementptr(%offloading_arrays, 0, 1)
9830 // %offload_sizes = getelementptr(%offloading_arrays, 0, 2)
9831 // kernel_launch_function(%thread.id, %offload_baseptrs, %offload_ptrs,
9832 // %offload_sizes, %structArg)
9833 // }
9834 //
9835 // We need the proxy function because the signature of the task entry point
9836 // expected by kmpc_omp_task is always the same and will be different from
9837 // that of the kernel_launch function.
9838 //
9839 // kernel_launch_function is generated by emitKernelLaunch and has the
9840 // always_inline attribute. For this example, it'll look like so:
9841 // void kernel_launch_function(%thread_id, %offload_baseptrs, %offload_ptrs,
9842 // %offload_sizes, %structArg) alwaysinline {
9843 // %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
9844 // ; load aggregated data from %structArg
9845 // ; setup kernel_args using offload_baseptrs, offload_ptrs and
9846 // ; offload_sizes
9847 // call i32 @__tgt_target_kernel(...,
9848 // outlined_device_function,
9849 // ptr %kernel_args)
9850 // }
9851 // void outlined_device_function(ptr a, ptr b, ptr n) {
9852 // n = *n_ptr;
9853 // do i = 1, 10
9854 // a(i) = b(i) + n
9855 // }
9856 //
9857 BasicBlock *TargetTaskBodyBB =
9858 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.body");
9859 BasicBlock *TargetTaskAllocaBB =
9860 splitBB(Builder, /*CreateBranch=*/true, Name: "target.task.alloca");
9861
9862 InsertPointTy TargetTaskAllocaIP(TargetTaskAllocaBB->begin());
9863 InsertPointTy TargetTaskBodyIP(TargetTaskBodyBB->begin());
9864
9865 auto OI = std::make_unique<OutlineInfo>();
9866 OI->EntryBB = TargetTaskAllocaBB;
9867 OI->OuterAllocBB = AllocaIP.getNodeParent();
9868
9869 // Add the thread ID argument.
9870 SmallVector<Instruction *, 4> ToBeDeleted;
9871 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
9872 Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TargetTaskAllocaIP, Name: "global.tid", AsPtr: false));
9873
9874 // Generate the task body which will subsequently be outlined.
9875 Builder.restoreIP(IP: TargetTaskBodyIP);
9876 if (Error Err = TaskBodyCB(DeviceID, RTLoc, TargetTaskAllocaIP))
9877 return Err;
9878
9879 // The outliner (CodeExtractor) extract a sequence or vector of blocks that
9880 // it is given. These blocks are enumerated by
9881 // OpenMPIRBuilder::OutlineInfo::collectBlocks which expects the OI.ExitBlock
9882 // to be outside the region. In other words, OI.ExitBlock is expected to be
9883 // the start of the region after the outlining. We used to set OI.ExitBlock
9884 // to the InsertBlock after TaskBodyCB is done. This is fine in most cases
9885 // except when the task body is a single basic block. In that case,
9886 // OI.ExitBlock is set to the single task body block and will get left out of
9887 // the outlining process. So, simply create a new empty block to which we
9888 // uncoditionally branch from where TaskBodyCB left off
9889 OI->ExitBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "target.task.cont");
9890 emitBlock(BB: OI->ExitBB, CurFn: Builder.GetInsertBlock()->getParent(),
9891 /*IsFinished=*/true);
9892
9893 SmallVector<Value *, 2> OffloadingArraysToPrivatize;
9894 bool NeedsTargetTask = HasNoWait && DeviceID;
9895 if (NeedsTargetTask) {
9896 for (auto *V :
9897 {RTArgs.BasePointersArray, RTArgs.PointersArray, RTArgs.MappersArray,
9898 RTArgs.MapNamesArray, RTArgs.MapTypesArray, RTArgs.MapTypesArrayEnd,
9899 RTArgs.SizesArray}) {
9900 if (V && !isa<ConstantPointerNull, GlobalVariable>(Val: V)) {
9901 OffloadingArraysToPrivatize.push_back(Elt: V);
9902 OI->ExcludeArgsFromAggregate.push_back(Elt: V);
9903 }
9904 }
9905 }
9906 OI->PostOutlineCB = [this, ToBeDeleted, Dependencies, NeedsTargetTask,
9907 DeviceID, OffloadingArraysToPrivatize](
9908 Function &OutlinedFn) mutable {
9909 assert(OutlinedFn.hasOneUse() &&
9910 "there must be a single user for the outlined function");
9911
9912 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
9913
9914 // The first argument of StaleCI is always the thread id.
9915 // The next few arguments are the pointers to offloading arrays
9916 // if any. (see OffloadingArraysToPrivatize)
9917 // Finally, all other local values that are live-in into the outlined region
9918 // end up in a structure whose pointer is passed as the last argument. This
9919 // piece of data is passed in the "shared" field of the task structure. So,
9920 // we know we have to pass shareds to the task if the number of arguments is
9921 // greater than OffloadingArraysToPrivatize.size() + 1 The 1 is for the
9922 // thread id. Further, for safety, we assert that the number of arguments of
9923 // StaleCI is exactly OffloadingArraysToPrivatize.size() + 2
9924 const unsigned int NumStaleCIArgs = StaleCI->arg_size();
9925 bool HasShareds = NumStaleCIArgs > OffloadingArraysToPrivatize.size() + 1;
9926 assert((!HasShareds ||
9927 NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2)) &&
9928 "Wrong number of arguments for StaleCI when shareds are present");
9929 int SharedArgOperandNo =
9930 HasShareds ? OffloadingArraysToPrivatize.size() + 1 : 0;
9931
9932 StructType *TaskWithPrivatesTy =
9933 createTaskWithPrivatesTy(OMPIRBuilder&: *this, OffloadingArraysToPrivatize);
9934 StructType *PrivatesTy = nullptr;
9935
9936 if (!OffloadingArraysToPrivatize.empty())
9937 PrivatesTy =
9938 static_cast<StructType *>(TaskWithPrivatesTy->getElementType(N: 1));
9939
9940 Function *ProxyFn = emitTargetTaskProxyFunction(
9941 OMPBuilder&: *this, Builder, StaleCI, PrivatesTy, TaskWithPrivatesTy,
9942 NumOffloadingArrays: OffloadingArraysToPrivatize.size(), SharedArgsOperandNo: SharedArgOperandNo);
9943
9944 LLVM_DEBUG(dbgs() << "Proxy task entry function created: " << *ProxyFn
9945 << "\n");
9946
9947 Builder.SetInsertPoint(StaleCI);
9948
9949 // Gather the arguments for emitting the runtime call.
9950 uint32_t SrcLocStrSize;
9951 Constant *SrcLocStr =
9952 getOrCreateSrcLocStr(Loc: LocationDescription(Builder), SrcLocStrSize);
9953 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
9954
9955 // @__kmpc_omp_task_alloc or @__kmpc_omp_target_task_alloc
9956 //
9957 // If `HasNoWait == true`, we call @__kmpc_omp_target_task_alloc to provide
9958 // the DeviceID to the deferred task and also since
9959 // @__kmpc_omp_target_task_alloc creates an untied/async task.
9960 Function *TaskAllocFn =
9961 !NeedsTargetTask
9962 ? getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
9963 : getOrCreateRuntimeFunctionPtr(
9964 FnID: OMPRTL___kmpc_omp_target_task_alloc);
9965
9966 // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
9967 // call.
9968 Value *ThreadID = getOrCreateThreadID(Ident);
9969
9970 // Argument - `sizeof_kmp_task_t` (TaskSize)
9971 // Tasksize refers to the size in bytes of kmp_task_t data structure
9972 // plus any other data to be passed to the target task, if any, which
9973 // is packed into a struct. kmp_task_t and the struct so created are
9974 // packed into a wrapper struct whose type is TaskWithPrivatesTy.
9975 Value *TaskSize = Builder.getInt64(
9976 C: M.getDataLayout().getTypeStoreSize(Ty: TaskWithPrivatesTy));
9977
9978 // Argument - `sizeof_shareds` (SharedsSize)
9979 // SharedsSize refers to the shareds array size in the kmp_task_t data
9980 // structure.
9981 Value *SharedsSize = Builder.getInt64(C: 0);
9982 if (HasShareds) {
9983 auto *ArgStructAlloca =
9984 dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: SharedArgOperandNo));
9985 assert(ArgStructAlloca &&
9986 "Unable to find the alloca instruction corresponding to arguments "
9987 "for extracted function");
9988 std::optional<TypeSize> ArgAllocSize =
9989 ArgStructAlloca->getAllocationSize(DL: M.getDataLayout());
9990 assert(ArgAllocSize &&
9991 "Unable to determine size of arguments for extracted function");
9992 SharedsSize = Builder.getInt64(C: ArgAllocSize->getFixedValue());
9993 }
9994
9995 // Argument - `flags`
9996 // Task is tied iff (Flags & 1) == 1.
9997 // Task is untied iff (Flags & 1) == 0.
9998 // Task is final iff (Flags & 2) == 2.
9999 // Task is not final iff (Flags & 2) == 0.
10000 // A target task is not final and is untied.
10001 Value *Flags = Builder.getInt32(C: 0);
10002
10003 // Emit the @__kmpc_omp_task_alloc runtime call
10004 // The runtime call returns a pointer to an area where the task captured
10005 // variables must be copied before the task is run (TaskData)
10006 CallInst *TaskData = nullptr;
10007
10008 SmallVector<llvm::Value *> TaskAllocArgs = {
10009 /*loc_ref=*/Ident, /*gtid=*/ThreadID,
10010 /*flags=*/Flags,
10011 /*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
10012 /*task_func=*/ProxyFn};
10013
10014 if (NeedsTargetTask) {
10015 assert(DeviceID && "Expected non-empty device ID.");
10016 TaskAllocArgs.push_back(Elt: DeviceID);
10017 }
10018
10019 TaskData = createRuntimeFunctionCall(Callee: TaskAllocFn, Args: TaskAllocArgs);
10020
10021 Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
10022 if (HasShareds) {
10023 Value *Shareds = StaleCI->getArgOperand(i: SharedArgOperandNo);
10024 Value *TaskShareds = loadSharedDataFromTaskDescriptor(
10025 OMPIRBuilder&: *this, Builder, TaskWithPrivates: TaskData, TaskWithPrivatesTy);
10026 Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
10027 Size: SharedsSize);
10028 }
10029 if (!OffloadingArraysToPrivatize.empty()) {
10030 Value *Privates =
10031 Builder.CreateStructGEP(Ty: TaskWithPrivatesTy, Ptr: TaskData, Idx: 1);
10032 for (unsigned int i = 0; i < OffloadingArraysToPrivatize.size(); ++i) {
10033 Value *PtrToPrivatize = OffloadingArraysToPrivatize[i];
10034 [[maybe_unused]] Type *ArrayType =
10035 getOffloadingArrayType(V: PtrToPrivatize);
10036 assert(ArrayType && "ArrayType cannot be nullptr");
10037
10038 Type *ElementType = PrivatesTy->getElementType(N: i);
10039 assert(ElementType == ArrayType &&
10040 "ElementType should match ArrayType");
10041 (void)ArrayType;
10042
10043 Value *Dst = Builder.CreateStructGEP(Ty: PrivatesTy, Ptr: Privates, Idx: i);
10044 Builder.CreateMemCpy(
10045 Dst, DstAlign: Alignment, Src: PtrToPrivatize, SrcAlign: Alignment,
10046 Size: Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: ElementType)));
10047 }
10048 }
10049
10050 Value *DepArray = nullptr;
10051 Value *NumDeps = nullptr;
10052 if (Dependencies.DepArray) {
10053 DepArray = Dependencies.DepArray;
10054 NumDeps = Dependencies.NumDeps;
10055 } else if (!Dependencies.Deps.empty()) {
10056 DepArray = emitTaskDependencies(OMPBuilder&: *this, Dependencies: Dependencies.Deps);
10057 NumDeps = Builder.getInt32(C: Dependencies.Deps.size());
10058 }
10059
10060 // ---------------------------------------------------------------
10061 // V5.2 13.8 target construct
10062 // If the nowait clause is present, execution of the target task
10063 // may be deferred. If the nowait clause is not present, the target task is
10064 // an included task.
10065 // ---------------------------------------------------------------
10066 // The above means that the lack of a nowait on the target construct
10067 // translates to '#pragma omp task if(0)'
10068 if (!NeedsTargetTask) {
10069 if (DepArray) {
10070 Function *TaskWaitFn =
10071 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_wait_deps);
10072 createRuntimeFunctionCall(
10073 Callee: TaskWaitFn,
10074 Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID,
10075 /*ndeps=*/NumDeps,
10076 /*dep_list=*/DepArray,
10077 /*ndeps_noalias=*/ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
10078 /*noalias_dep_list=*/
10079 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
10080 }
10081 // Included task.
10082 Function *TaskBeginFn =
10083 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
10084 Function *TaskCompleteFn =
10085 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
10086 createRuntimeFunctionCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
10087 CallInst *CI = createRuntimeFunctionCall(Callee: ProxyFn, Args: {ThreadID, TaskData});
10088 CI->setDebugLoc(StaleCI->getDebugLoc());
10089 createRuntimeFunctionCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
10090 } else if (DepArray) {
10091 // HasNoWait - meaning the task may be deferred. Call
10092 // __kmpc_omp_task_with_deps if there are dependencies,
10093 // else call __kmpc_omp_task
10094 Function *TaskFn =
10095 getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
10096 createRuntimeFunctionCall(
10097 Callee: TaskFn,
10098 Args: {Ident, ThreadID, TaskData, NumDeps, DepArray,
10099 ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
10100 ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
10101 } else {
10102 // Emit the @__kmpc_omp_task runtime call to spawn the task
10103 Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
10104 createRuntimeFunctionCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
10105 }
10106
10107 Builder.ClearInsertionPoint();
10108 StaleCI->eraseFromParent();
10109 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
10110 I->eraseFromParent();
10111 };
10112 addOutlineInfo(OI: std::move(OI));
10113
10114 LLVM_DEBUG(dbgs() << "Insert block after emitKernelLaunch = \n"
10115 << *(Builder.GetInsertBlock()) << "\n");
10116 LLVM_DEBUG(dbgs() << "Module after emitKernelLaunch = \n"
10117 << *(Builder.getModule()) << "\n");
10118 return Builder.saveIP();
10119}
10120
10121Error OpenMPIRBuilder::emitOffloadingArraysAndArgs(
10122 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info,
10123 TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo,
10124 CustomMapperCallbackTy CustomMapperCB, bool IsNonContiguous,
10125 bool ForEndCall, function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
10126 if (Error Err =
10127 emitOffloadingArrays(AllocaIP, CodeGenIP, CombinedInfo, Info,
10128 CustomMapperCB, IsNonContiguous, DeviceAddrCB))
10129 return Err;
10130 emitOffloadingArraysArgument(Builder, RTArgs, Info, ForEndCall);
10131 return Error::success();
10132}
10133
10134static void emitTargetCall(
10135 OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Value *RTLocOverride,
10136 OpenMPIRBuilder::InsertPointTy AllocaIP,
10137 ArrayRef<BasicBlock *> DeallocBlocks, OpenMPIRBuilder::TargetDataInfo &Info,
10138 const OpenMPIRBuilder::TargetKernelDefaultAttrs &DefaultAttrs,
10139 const OpenMPIRBuilder::TargetKernelRuntimeAttrs &RuntimeAttrs,
10140 Value *IfCond, Function *OutlinedFn, Constant *OutlinedFnID,
10141 SmallVectorImpl<Value *> &Args,
10142 OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
10143 OpenMPIRBuilder::CustomMapperCallbackTy CustomMapperCB,
10144 const OpenMPIRBuilder::DependenciesInfo &Dependencies, bool HasNoWait,
10145 Value *DynCGroupMem, OMPDynGroupprivateFallbackType DynCGroupMemFallback) {
10146 // Generate a function call to the host fallback implementation of the target
10147 // region. This is called by the host when no offload entry was generated for
10148 // the target region and when the offloading call fails at runtime.
10149 auto &&EmitTargetCallFallbackCB = [&](OpenMPIRBuilder::InsertPointTy IP)
10150 -> OpenMPIRBuilder::InsertPointOrErrorTy {
10151 Builder.restoreIP(IP);
10152 // Ensure the host fallback has the same dyn_ptr ABI as the device.
10153 SmallVector<Value *> FallbackArgs(Args.begin(), Args.end());
10154 FallbackArgs.push_back(
10155 Elt: Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext())));
10156 OMPBuilder.createRuntimeFunctionCall(Callee: OutlinedFn, Args: FallbackArgs);
10157 return Builder.saveIP();
10158 };
10159
10160 bool HasDependencies = !Dependencies.empty();
10161 bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
10162
10163 OpenMPIRBuilder::TargetKernelArgs KArgs;
10164
10165 auto TaskBodyCB =
10166 [&](Value *DeviceID, Value *RTLoc,
10167 IRBuilderBase::InsertPoint TargetTaskAllocaIP) -> Error {
10168 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10169 // produce any.
10170 llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10171 // emitKernelLaunch makes the necessary runtime call to offload the
10172 // kernel. We then outline all that code into a separate function
10173 // ('kernel_launch_function' in the pseudo code above). This function is
10174 // then called by the target task proxy function (see
10175 // '@.omp_target_task_proxy_func' in the pseudo code above)
10176 // "@.omp_target_task_proxy_func' is generated by
10177 // emitTargetTaskProxyFunction.
10178 if (OutlinedFnID && DeviceID)
10179 return OMPBuilder.emitKernelLaunch(Loc: Builder, OutlinedFnID,
10180 EmitTargetCallFallbackCB, Args&: KArgs,
10181 DeviceID, RTLoc, AllocaIP: TargetTaskAllocaIP);
10182
10183 // We only need to do the outlining if `DeviceID` is set to avoid calling
10184 // `emitKernelLaunch` if we want to code-gen for the host; e.g. if we are
10185 // generating the `else` branch of an `if` clause.
10186 //
10187 // When OutlinedFnID is set to nullptr, then it's not an offloading call.
10188 // In this case, we execute the host implementation directly.
10189 return EmitTargetCallFallbackCB(OMPBuilder.Builder.saveIP());
10190 }());
10191
10192 OMPBuilder.Builder.restoreIP(IP: AfterIP);
10193 return Error::success();
10194 };
10195
10196 auto &&EmitTargetCallElse =
10197 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10198 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10199 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10200 // Assume no error was returned because EmitTargetCallFallbackCB doesn't
10201 // produce any.
10202 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10203 if (RequiresOuterTargetTask) {
10204 // Arguments that are intended to be directly forwarded to an
10205 // emitKernelLaunch call are pased as nullptr, since
10206 // OutlinedFnID=nullptr results in that call not being done.
10207 OpenMPIRBuilder::TargetDataRTArgs EmptyRTArgs;
10208 return OMPBuilder.emitTargetTask(TaskBodyCB, /*DeviceID=*/nullptr,
10209 /*RTLoc=*/nullptr, AllocaIP,
10210 Dependencies, RTArgs: EmptyRTArgs, HasNoWait);
10211 }
10212 return EmitTargetCallFallbackCB(Builder.saveIP());
10213 }());
10214
10215 Builder.restoreIP(IP: AfterIP);
10216 return Error::success();
10217 };
10218
10219 auto &&EmitTargetCallThen =
10220 [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
10221 OpenMPIRBuilder::InsertPointTy CodeGenIP,
10222 ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
10223 Info.HasNoWait = HasNoWait;
10224 OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
10225
10226 OpenMPIRBuilder::TargetDataRTArgs RTArgs;
10227 if (Error Err = OMPBuilder.emitOffloadingArraysAndArgs(
10228 AllocaIP, CodeGenIP: Builder.saveIP(), Info, RTArgs, CombinedInfo&: MapInfo, CustomMapperCB,
10229 /*IsNonContiguous=*/true,
10230 /*ForEndCall=*/false))
10231 return Err;
10232
10233 SmallVector<Value *, 3> NumTeamsC;
10234 for (auto [DefaultVal, RuntimeVal] :
10235 zip_equal(t: DefaultAttrs.MaxTeams, u: RuntimeAttrs.MaxTeams))
10236 NumTeamsC.push_back(Elt: RuntimeVal ? RuntimeVal
10237 : Builder.getInt32(C: DefaultVal));
10238
10239 // Calculate number of threads: 0 if no clauses specified, otherwise it is
10240 // the minimum between optional THREAD_LIMIT and NUM_THREADS clauses.
10241 auto InitMaxThreadsClause = [&Builder](Value *Clause) {
10242 if (Clause)
10243 Clause = Builder.CreateIntCast(V: Clause, DestTy: Builder.getInt32Ty(),
10244 /*isSigned=*/false);
10245 return Clause;
10246 };
10247 auto CombineMaxThreadsClauses = [&Builder](Value *Clause, Value *&Result) {
10248 if (Clause)
10249 Result =
10250 Result ? Builder.CreateSelect(C: Builder.CreateICmpULT(LHS: Result, RHS: Clause),
10251 True: Result, False: Clause)
10252 : Clause;
10253 };
10254
10255 // If a multi-dimensional THREAD_LIMIT is set, it is the OMPX_BARE case, so
10256 // the NUM_THREADS clause is overriden by THREAD_LIMIT.
10257 SmallVector<Value *, 3> NumThreadsC;
10258 Value *MaxThreadsClause =
10259 RuntimeAttrs.TeamsThreadLimit.size() == 1
10260 ? InitMaxThreadsClause(RuntimeAttrs.MaxThreads.front())
10261 : nullptr;
10262
10263 for (auto [TeamsVal, TargetVal] : zip_equal(
10264 t: RuntimeAttrs.TeamsThreadLimit, u: RuntimeAttrs.TargetThreadLimit)) {
10265 Value *TeamsThreadLimitClause = InitMaxThreadsClause(TeamsVal);
10266 Value *NumThreads = InitMaxThreadsClause(TargetVal);
10267
10268 CombineMaxThreadsClauses(TeamsThreadLimitClause, NumThreads);
10269 CombineMaxThreadsClauses(MaxThreadsClause, NumThreads);
10270
10271 NumThreadsC.push_back(Elt: NumThreads ? NumThreads : Builder.getInt32(C: 0));
10272 }
10273
10274 unsigned NumTargetItems = Info.NumberOfPtrs;
10275 Value *RTLoc = RTLocOverride;
10276 if (!RTLoc) {
10277 uint32_t SrcLocStrSize;
10278 Constant *SrcLocStr =
10279 OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10280 RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
10281 LocFlags: llvm::omp::IdentFlag(0), Reserve2Flags: 0);
10282 }
10283
10284 Value *TripCount = RuntimeAttrs.LoopTripCount
10285 ? Builder.CreateIntCast(V: RuntimeAttrs.LoopTripCount,
10286 DestTy: Builder.getInt64Ty(),
10287 /*isSigned=*/false)
10288 : Builder.getInt64(C: 0);
10289
10290 // Request zero groupprivate bytes by default.
10291 if (!DynCGroupMem)
10292 DynCGroupMem = Builder.getInt32(C: 0);
10293
10294 KArgs = OpenMPIRBuilder::TargetKernelArgs(
10295 NumTargetItems, RTArgs, TripCount, NumTeamsC, NumThreadsC, DynCGroupMem,
10296 HasNoWait, /*StrictBlocks=*/false, /*StrictThreads=*/false,
10297 DynCGroupMemFallback);
10298
10299 // Assume no error was returned because TaskBodyCB and
10300 // EmitTargetCallFallbackCB don't produce any.
10301 OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(ValOrErr: [&]() {
10302 // The presence of certain clauses on the target directive require the
10303 // explicit generation of the target task.
10304 if (RequiresOuterTargetTask)
10305 return OMPBuilder.emitTargetTask(TaskBodyCB, DeviceID: RuntimeAttrs.DeviceID,
10306 RTLoc, AllocaIP, Dependencies,
10307 RTArgs: KArgs.RTArgs, HasNoWait: Info.HasNoWait);
10308
10309 return OMPBuilder.emitKernelLaunch(
10310 Loc: Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args&: KArgs,
10311 DeviceID: RuntimeAttrs.DeviceID, RTLoc, AllocaIP);
10312 }());
10313
10314 Builder.restoreIP(IP: AfterIP);
10315 return Error::success();
10316 };
10317
10318 // If we don't have an ID for the target region, it means an offload entry
10319 // wasn't created. In this case we just run the host fallback directly and
10320 // ignore any potential 'if' clauses.
10321 if (!OutlinedFnID) {
10322 cantFail(Err: EmitTargetCallElse(AllocaIP, Builder.saveIP(), DeallocBlocks));
10323 return;
10324 }
10325
10326 // If there's no 'if' clause, only generate the kernel launch code path.
10327 if (!IfCond) {
10328 cantFail(Err: EmitTargetCallThen(AllocaIP, Builder.saveIP(), DeallocBlocks));
10329 return;
10330 }
10331
10332 cantFail(Err: OMPBuilder.emitIfClause(Cond: IfCond, ThenGen: EmitTargetCallThen,
10333 ElseGen: EmitTargetCallElse, AllocaIP));
10334}
10335
10336OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTarget(
10337 const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
10338 InsertPointTy CodeGenIP, ArrayRef<BasicBlock *> DeallocBlocks,
10339 TargetDataInfo &Info, TargetRegionEntryInfo &EntryInfo,
10340 const TargetKernelDefaultAttrs &DefaultAttrs,
10341 const TargetKernelRuntimeAttrs &RuntimeAttrs, Value *IfCond,
10342 SmallVectorImpl<Value *> &Inputs, GenMapInfoCallbackTy GenMapInfoCB,
10343 OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
10344 OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
10345 CustomMapperCallbackTy CustomMapperCB, const DependenciesInfo &Dependencies,
10346 bool HasNowait, Value *DynCGroupMem,
10347 OMPDynGroupprivateFallbackType DynCGroupMemFallback, DebugLoc OutlinedFnLoc,
10348 Value *RTLocOverride) {
10349
10350 if (!updateToLocation(Loc))
10351 return InsertPointTy();
10352
10353 Builder.restoreIP(IP: CodeGenIP);
10354
10355 Function *OutlinedFn;
10356 Constant *OutlinedFnID = nullptr;
10357 // The target region is outlined into its own function. The LLVM IR for
10358 // the target region itself is generated using the callbacks CBFunc
10359 // and ArgAccessorFuncCB
10360 if (Error Err = emitTargetOutlinedFunction(
10361 OMPBuilder&: *this, Builder, IsOffloadEntry, EntryInfo, DefaultAttrs, OutlinedFn,
10362 OutlinedFnID, Inputs, CBFunc, ArgAccessorFuncCB, OutlinedFnLoc))
10363 return Err;
10364
10365 // If we are not on the target device, then we need to generate code
10366 // to make a remote call (offload) to the previously outlined function
10367 // that represents the target region. Do that now.
10368 if (!Config.isTargetDevice())
10369 emitTargetCall(OMPBuilder&: *this, Builder, RTLocOverride, AllocaIP, DeallocBlocks, Info,
10370 DefaultAttrs, RuntimeAttrs, IfCond, OutlinedFn, OutlinedFnID,
10371 Args&: Inputs, GenMapInfoCB, CustomMapperCB, Dependencies,
10372 HasNoWait: HasNowait, DynCGroupMem, DynCGroupMemFallback);
10373 return Builder.saveIP();
10374}
10375
10376std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
10377 StringRef FirstSeparator,
10378 StringRef Separator) {
10379 SmallString<128> Buffer;
10380 llvm::raw_svector_ostream OS(Buffer);
10381 StringRef Sep = FirstSeparator;
10382 for (StringRef Part : Parts) {
10383 OS << Sep << Part;
10384 Sep = Separator;
10385 }
10386 return OS.str().str();
10387}
10388
10389std::string
10390OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {
10391 return OpenMPIRBuilder::getNameWithSeparators(Parts, FirstSeparator: Config.firstSeparator(),
10392 Separator: Config.separator());
10393}
10394
10395GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
10396 Type *Ty, const StringRef &Name, std::optional<unsigned> AddressSpace) {
10397 auto &Elem = *InternalVars.try_emplace(Key: Name, Args: nullptr).first;
10398 if (Elem.second) {
10399 assert(Elem.second->getValueType() == Ty &&
10400 "OMP internal variable has different type than requested");
10401 } else {
10402 // TODO: investigate the appropriate linkage type used for the global
10403 // variable for possibly changing that to internal or private, or maybe
10404 // create different versions of the function for different OMP internal
10405 // variables.
10406 const DataLayout &DL = M.getDataLayout();
10407 // TODO: Investigate why AMDGPU expects AS 0 for globals even though the
10408 // default global AS is 1.
10409 // See double-target-call-with-declare-target.f90 and
10410 // declare-target-vars-in-target-region.f90 libomptarget
10411 // tests.
10412 unsigned AddressSpaceVal = AddressSpace ? *AddressSpace
10413 : M.getTargetTriple().isAMDGPU()
10414 ? 0
10415 : DL.getDefaultGlobalsAddressSpace();
10416 auto Linkage = this->M.getTargetTriple().isWasm()
10417 ? GlobalValue::InternalLinkage
10418 : GlobalValue::CommonLinkage;
10419 auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
10420 Constant::getNullValue(Ty), Elem.first(),
10421 /*InsertBefore=*/nullptr,
10422 GlobalValue::NotThreadLocal, AddressSpaceVal);
10423 const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);
10424 const llvm::Align PtrAlign = DL.getPointerABIAlignment(AS: AddressSpaceVal);
10425 GV->setAlignment(std::max(a: TypeAlign, b: PtrAlign));
10426 Elem.second = GV;
10427 }
10428
10429 return Elem.second;
10430}
10431
10432Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
10433 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
10434 std::string Name = getNameWithSeparators(Parts: {Prefix, "var"}, FirstSeparator: ".", Separator: ".");
10435 return getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
10436}
10437
10438Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {
10439 LLVMContext &Ctx = Builder.getContext();
10440 Value *Null =
10441 Constant::getNullValue(Ty: PointerType::getUnqual(C&: BasePtr->getContext()));
10442 Value *SizeGep =
10443 Builder.CreateGEP(Ty: BasePtr->getType(), Ptr: Null, IdxList: Builder.getInt32(C: 1));
10444 Value *SizePtrToInt = Builder.CreatePtrToInt(V: SizeGep, DestTy: Type::getInt64Ty(C&: Ctx));
10445 return SizePtrToInt;
10446}
10447
10448GlobalVariable *
10449OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
10450 std::string VarName) {
10451 llvm::Constant *MaptypesArrayInit =
10452 llvm::ConstantDataArray::get(Context&: M.getContext(), Elts&: Mappings);
10453 auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
10454 M, MaptypesArrayInit->getType(),
10455 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
10456 VarName);
10457 MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
10458 return MaptypesArrayGlobal;
10459}
10460
10461void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
10462 InsertPointTy AllocaIP,
10463 unsigned NumOperands,
10464 struct MapperAllocas &MapperAllocas) {
10465 if (!updateToLocation(Loc))
10466 return;
10467
10468 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10469 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10470 Builder.restoreIP(IP: AllocaIP);
10471 AllocaInst *ArgsBase = Builder.CreateAlloca(
10472 Ty: ArrI8PtrTy, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
10473 AllocaInst *Args = Builder.CreateAlloca(Ty: ArrI8PtrTy, /* ArraySize = */ nullptr,
10474 Name: ".offload_ptrs");
10475 AllocaInst *ArgSizes = Builder.CreateAlloca(
10476 Ty: ArrI64Ty, /* ArraySize = */ nullptr, Name: ".offload_sizes");
10477 updateToLocation(Loc);
10478 MapperAllocas.ArgsBase = ArgsBase;
10479 MapperAllocas.Args = Args;
10480 MapperAllocas.ArgSizes = ArgSizes;
10481}
10482
10483void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
10484 Function *MapperFunc, Value *SrcLocInfo,
10485 Value *MaptypesArg, Value *MapnamesArg,
10486 struct MapperAllocas &MapperAllocas,
10487 int64_t DeviceID, unsigned NumOperands) {
10488 if (!updateToLocation(Loc))
10489 return;
10490
10491 auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
10492 auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
10493 Value *ArgsBaseGEP =
10494 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.ArgsBase,
10495 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10496 Value *ArgsGEP =
10497 Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.Args,
10498 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10499 Value *ArgSizesGEP =
10500 Builder.CreateInBoundsGEP(Ty: ArrI64Ty, Ptr: MapperAllocas.ArgSizes,
10501 IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
10502 Value *NullPtr =
10503 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Int8Ptr->getContext()));
10504 createRuntimeFunctionCall(Callee: MapperFunc, Args: {SrcLocInfo, Builder.getInt64(C: DeviceID),
10505 Builder.getInt32(C: NumOperands),
10506 ArgsBaseGEP, ArgsGEP, ArgSizesGEP,
10507 MaptypesArg, MapnamesArg, NullPtr});
10508}
10509
10510void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
10511 TargetDataRTArgs &RTArgs,
10512 TargetDataInfo &Info,
10513 bool ForEndCall) {
10514 assert((!ForEndCall || Info.separateBeginEndCalls()) &&
10515 "expected region end call to runtime only when end call is separate");
10516 auto UnqualPtrTy = PointerType::getUnqual(C&: M.getContext());
10517 auto VoidPtrTy = UnqualPtrTy;
10518 auto VoidPtrPtrTy = UnqualPtrTy;
10519 auto Int64Ty = Type::getInt64Ty(C&: M.getContext());
10520 auto Int64PtrTy = UnqualPtrTy;
10521
10522 if (!Info.NumberOfPtrs) {
10523 RTArgs.BasePointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10524 RTArgs.PointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10525 RTArgs.SizesArray = ConstantPointerNull::get(T: Int64PtrTy);
10526 RTArgs.MapTypesArray = ConstantPointerNull::get(T: Int64PtrTy);
10527 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10528 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10529 return;
10530 }
10531
10532 RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
10533 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs),
10534 Ptr: Info.RTArgs.BasePointersArray,
10535 /*Idx0=*/0, /*Idx1=*/0);
10536 RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
10537 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray,
10538 /*Idx0=*/0,
10539 /*Idx1=*/0);
10540 RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
10541 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
10542 /*Idx0=*/0, /*Idx1=*/0);
10543 RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
10544 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs),
10545 Ptr: ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd
10546 : Info.RTArgs.MapTypesArray,
10547 /*Idx0=*/0,
10548 /*Idx1=*/0);
10549
10550 // Only emit the mapper information arrays if debug information is
10551 // requested.
10552 if (!Info.EmitDebug)
10553 RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10554 else
10555 RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
10556 Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.MapNamesArray,
10557 /*Idx0=*/0,
10558 /*Idx1=*/0);
10559 // If there is no user-defined mapper, set the mapper array to nullptr to
10560 // avoid an unnecessary data privatization
10561 if (!Info.HasMapper)
10562 RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
10563 else
10564 RTArgs.MappersArray =
10565 Builder.CreatePointerCast(V: Info.RTArgs.MappersArray, DestTy: VoidPtrPtrTy);
10566}
10567
10568void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
10569 InsertPointTy CodeGenIP,
10570 MapInfosTy &CombinedInfo,
10571 TargetDataInfo &Info) {
10572 MapInfosTy::StructNonContiguousInfo &NonContigInfo =
10573 CombinedInfo.NonContigInfo;
10574
10575 // Build an array of struct descriptor_dim and then assign it to
10576 // offload_args.
10577 //
10578 // struct descriptor_dim {
10579 // uint64_t offset;
10580 // uint64_t count;
10581 // uint64_t stride
10582 // };
10583 Type *Int64Ty = Builder.getInt64Ty();
10584 StructType *DimTy = StructType::create(
10585 Context&: M.getContext(), Elements: ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
10586 Name: "struct.descriptor_dim");
10587
10588 enum { OffsetFD = 0, CountFD, StrideFD };
10589 // We need two index variable here since the size of "Dims" is the same as
10590 // the size of Components, however, the size of offset, count, and stride is
10591 // equal to the size of base declaration that is non-contiguous.
10592 for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
10593 // Skip emitting ir if dimension size is 1 since it cannot be
10594 // non-contiguous.
10595 if (NonContigInfo.Dims[I] == 1)
10596 continue;
10597 Builder.restoreIP(IP: AllocaIP);
10598 ArrayType *ArrayTy = ArrayType::get(ElementType: DimTy, NumElements: NonContigInfo.Dims[I]);
10599 AllocaInst *DimsAddr =
10600 Builder.CreateAlloca(Ty: ArrayTy, /* ArraySize = */ nullptr, Name: "dims");
10601 Builder.restoreIP(IP: CodeGenIP);
10602 for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
10603 unsigned RevIdx = EE - II - 1;
10604 Value *DimsLVal = Builder.CreateInBoundsGEP(
10605 Ty: ArrayTy, Ptr: DimsAddr, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: II)});
10606 // Offset
10607 Value *OffsetLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: OffsetFD);
10608 Builder.CreateAlignedStore(
10609 Val: NonContigInfo.Offsets[L][RevIdx], Ptr: OffsetLVal,
10610 Align: M.getDataLayout().getPrefTypeAlign(Ty: OffsetLVal->getType()));
10611 // Count
10612 Value *CountLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: CountFD);
10613 Builder.CreateAlignedStore(
10614 Val: NonContigInfo.Counts[L][RevIdx], Ptr: CountLVal,
10615 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10616 // Stride
10617 Value *StrideLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: StrideFD);
10618 Builder.CreateAlignedStore(
10619 Val: NonContigInfo.Strides[L][RevIdx], Ptr: StrideLVal,
10620 Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
10621 }
10622 // args[I] = &dims
10623 Builder.restoreIP(IP: CodeGenIP);
10624 Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
10625 V: DimsAddr, DestTy: Builder.getPtrTy());
10626 Value *P = Builder.CreateConstInBoundsGEP2_32(
10627 Ty: ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs),
10628 Ptr: Info.RTArgs.PointersArray, Idx0: 0, Idx1: I);
10629 Builder.CreateAlignedStore(
10630 Val: DAddr, Ptr: P, Align: M.getDataLayout().getPrefTypeAlign(Ty: Builder.getPtrTy()));
10631 ++L;
10632 }
10633}
10634
10635void OpenMPIRBuilder::emitUDMapperArrayInitOrDel(
10636 Function *MapperFn, Value *MapperHandle, Value *Base, Value *Begin,
10637 Value *Size, Value *MapType, Value *MapName, TypeSize ElementSize,
10638 BasicBlock *ExitBB, bool IsInit) {
10639 StringRef Prefix = IsInit ? ".init" : ".del";
10640
10641 // Evaluate if this is an array section.
10642 BasicBlock *BodyBB = BasicBlock::Create(
10643 Context&: M.getContext(), Name: createPlatformSpecificName(Parts: {"omp.array", Prefix}));
10644 Value *IsArray =
10645 Builder.CreateICmpSGT(LHS: Size, RHS: Builder.getInt64(C: 1), Name: "omp.arrayinit.isarray");
10646 Value *DeleteBit = Builder.CreateAnd(
10647 LHS: MapType,
10648 RHS: Builder.getInt64(
10649 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10650 OpenMPOffloadMappingFlags::OMP_MAP_DELETE)));
10651 Value *DeleteCond;
10652 Value *Cond;
10653 if (IsInit) {
10654 // base != begin?
10655 Value *BaseIsBegin = Builder.CreateICmpNE(LHS: Base, RHS: Begin);
10656 Cond = Builder.CreateOr(LHS: IsArray, RHS: BaseIsBegin);
10657 DeleteCond = Builder.CreateIsNull(
10658 Arg: DeleteBit,
10659 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10660 } else {
10661 Cond = IsArray;
10662 DeleteCond = Builder.CreateIsNotNull(
10663 Arg: DeleteBit,
10664 Name: createPlatformSpecificName(Parts: {"omp.array", Prefix, ".delete"}));
10665 }
10666 Cond = Builder.CreateAnd(LHS: Cond, RHS: DeleteCond);
10667 Builder.CreateCondBr(Cond, True: BodyBB, False: ExitBB);
10668
10669 emitBlock(BB: BodyBB, CurFn: MapperFn);
10670 // Get the array size by multiplying element size and element number (i.e., \p
10671 // Size).
10672 Value *ArraySize = Builder.CreateNUWMul(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10673 // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves
10674 // memory allocation/deletion purpose only.
10675 Value *MapTypeArg = Builder.CreateAnd(
10676 LHS: MapType,
10677 RHS: Builder.getInt64(
10678 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10679 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10680 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10681 MapTypeArg = Builder.CreateOr(
10682 LHS: MapTypeArg,
10683 RHS: Builder.getInt64(
10684 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10685 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)));
10686
10687 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
10688 // data structure.
10689 Value *OffloadingArgs[] = {MapperHandle, Base, Begin,
10690 ArraySize, MapTypeArg, MapName};
10691 createRuntimeFunctionCall(
10692 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
10693 Args: OffloadingArgs);
10694}
10695
10696Expected<Function *> OpenMPIRBuilder::emitUserDefinedMapper(
10697 function_ref<MapInfosOrErrorTy(InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10698 llvm::Value *BeginArg)>
10699 GenMapInfoCB,
10700 Type *ElemTy, StringRef FuncName, CustomMapperCallbackTy CustomMapperCB,
10701 bool PreserveMemberOfFlags, bool PropagatePresentToPointee) {
10702 SmallVector<Type *> Params;
10703 Params.emplace_back(Args: Builder.getPtrTy());
10704 Params.emplace_back(Args: Builder.getPtrTy());
10705 Params.emplace_back(Args: Builder.getPtrTy());
10706 Params.emplace_back(Args: Builder.getInt64Ty());
10707 Params.emplace_back(Args: Builder.getInt64Ty());
10708 Params.emplace_back(Args: Builder.getPtrTy());
10709
10710 auto *FnTy =
10711 FunctionType::get(Result: Builder.getVoidTy(), Params, /* IsVarArg */ isVarArg: false);
10712
10713 SmallString<64> TyStr;
10714 raw_svector_ostream Out(TyStr);
10715 Function *MapperFn =
10716 Function::Create(Ty: FnTy, Linkage: GlobalValue::InternalLinkage, N: FuncName, M);
10717 MapperFn->addFnAttr(Kind: Attribute::NoInline);
10718 MapperFn->addFnAttr(Kind: Attribute::NoUnwind);
10719 MapperFn->addParamAttr(ArgNo: 0, Kind: Attribute::NoUndef);
10720 MapperFn->addParamAttr(ArgNo: 1, Kind: Attribute::NoUndef);
10721 MapperFn->addParamAttr(ArgNo: 2, Kind: Attribute::NoUndef);
10722 MapperFn->addParamAttr(ArgNo: 3, Kind: Attribute::NoUndef);
10723 MapperFn->addParamAttr(ArgNo: 4, Kind: Attribute::NoUndef);
10724 MapperFn->addParamAttr(ArgNo: 5, Kind: Attribute::NoUndef);
10725
10726 // Start the mapper function code generation.
10727 BasicBlock *EntryBB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: MapperFn);
10728 IRBuilder<>::InsertPointGuard IPG(Builder);
10729 Builder.SetInsertPoint(EntryBB);
10730 Builder.SetCurrentDebugLocation(llvm::DebugLoc());
10731
10732 Value *MapperHandle = MapperFn->getArg(i: 0);
10733 Value *BaseIn = MapperFn->getArg(i: 1);
10734 Value *BeginIn = MapperFn->getArg(i: 2);
10735 Value *Size = MapperFn->getArg(i: 3);
10736 Value *MapType = MapperFn->getArg(i: 4);
10737 Value *MapName = MapperFn->getArg(i: 5);
10738
10739 // Compute the starting and end addresses of array elements.
10740 // Prepare common arguments for array initiation and deletion.
10741 // Convert the size in bytes into the number of array elements.
10742 TypeSize ElementSize = M.getDataLayout().getTypeStoreSize(Ty: ElemTy);
10743 Size = Builder.CreateExactUDiv(LHS: Size, RHS: Builder.getInt64(C: ElementSize));
10744 Value *PtrBegin = BeginIn;
10745 Value *PtrEnd = Builder.CreateGEP(Ty: ElemTy, Ptr: PtrBegin, IdxList: Size);
10746
10747 // Emit array initiation if this is an array section and \p MapType indicates
10748 // that memory allocation is required.
10749 BasicBlock *HeadBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.head");
10750 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
10751 MapType, MapName, ElementSize, ExitBB: HeadBB,
10752 /*IsInit=*/true);
10753
10754 // Emit a for loop to iterate through SizeArg of elements and map all of them.
10755
10756 // Emit the loop header block.
10757 emitBlock(BB: HeadBB, CurFn: MapperFn);
10758 BasicBlock *BodyBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.body");
10759 BasicBlock *DoneBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.done");
10760 // Evaluate whether the initial condition is satisfied.
10761 Value *IsEmpty =
10762 Builder.CreateICmpEQ(LHS: PtrBegin, RHS: PtrEnd, Name: "omp.arraymap.isempty");
10763 Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
10764
10765 // Emit the loop body block.
10766 emitBlock(BB: BodyBB, CurFn: MapperFn);
10767 BasicBlock *LastBB = BodyBB;
10768 PHINode *PtrPHI =
10769 Builder.CreatePHI(Ty: PtrBegin->getType(), NumReservedValues: 2, Name: "omp.arraymap.ptrcurrent");
10770 PtrPHI->addIncoming(V: PtrBegin, BB: HeadBB);
10771
10772 // Get map clause information. Fill up the arrays with all mapped variables.
10773 MapInfosOrErrorTy Info = GenMapInfoCB(Builder.saveIP(), PtrPHI, BeginIn);
10774 if (!Info)
10775 return Info.takeError();
10776
10777 // Call the runtime API __tgt_mapper_num_components to get the number of
10778 // pre-existing components.
10779 Value *OffloadingArgs[] = {MapperHandle};
10780 Value *PreviousSize = createRuntimeFunctionCall(
10781 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_mapper_num_components),
10782 Args: OffloadingArgs);
10783 Value *ShiftedPreviousSize =
10784 Builder.CreateShl(LHS: PreviousSize, RHS: Builder.getInt64(C: getFlagMemberOffset()));
10785
10786 // Fill up the runtime mapper handle for all components.
10787 for (unsigned I = 0; I < Info->BasePointers.size(); ++I) {
10788 Value *CurBaseArg = Info->BasePointers[I];
10789 Value *CurBeginArg = Info->Pointers[I];
10790 Value *CurSizeArg = Info->Sizes[I];
10791 Value *CurNameArg = Info->Names.size()
10792 ? Info->Names[I]
10793 : Constant::getNullValue(Ty: Builder.getPtrTy());
10794
10795 Value *OriMapType = Builder.getInt64(
10796 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10797 Info->Types[I]));
10798 auto RawType =
10799 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10800 Info->Types[I]);
10801 constexpr uint64_t MemberOfMask =
10802 static_cast<uint64_t>(OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
10803 constexpr uint64_t AttachBit =
10804 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10805 OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
10806
10807 // Add MEMBER_OF (ShiftedPreviousSize) to group this sub-map with the
10808 // current array element (N = __tgt_mapper_num_components() at loop body
10809 // start).
10810 //
10811 // Example 1:
10812 // struct S { int x; int *p; };
10813 //
10814 // mapper: #pragma omp declare mapper(id: S s) map(s.x, s.p[0:10])
10815 // use: S arr[2]; ... map(arr)
10816 // entries per element:
10817 //
10818 // &arr[i], &arr[i].x, sizeof(int), MEMBER_OF(N)|TO|FROM
10819 // &arr[i].p[0], &arr[i].p[0], 10*sizeof(int), TO|FROM (*)
10820 // &arr[i].p, &arr[i].p[0], sizeof(int*), ATTACH (**)
10821 //
10822 // Example 2:
10823 // struct S1 { int x; int y; };
10824 // struct S2 { int z; S1 *s1p; };
10825 //
10826 // mapper: #pragma omp declare mapper(S2 s2) map(s2.z, s2.s1p->x,
10827 // s2.s1p->y)
10828 // use: S2 arr[2]; ... map(arr)
10829 // entries per element:
10830 //
10831 // &arr[i], &arr[i].z, sizeof(int), MEMBER_OF(N)|TO|FROM
10832 // &arr[i].s1p[0], &arr[i].s1p->x, sizeof(s1p->x..y), ALLOC (*)
10833 // &arr[i].s1p[0], &arr[i].s1p->x, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10834 // &arr[i].s1p[0], &arr[i].s1p->y, 4, MEMBER_OF(N+2)|TO|FROM (*)(***)
10835 // &arr[i].s1p, &arr[i].s1p->x, sizeof(ptr), ATTACH (**)
10836 //
10837 // x/y carry inner MEMBER_OF(2)
10838 // which is shifted by N to become MEMBER_OF(N+2).
10839 //
10840 // HasAttachPtr is set on all of the s1p entries except the ATTACH one:
10841 // the combined ALLOC entry for the s1p->x..y block, and the individual
10842 // x/y entries that are MEMBER_OF that block, all describe storage
10843 // reached through the attach ptr arr[i].s1p.
10844 //
10845 // Entries of the following kinds do NOT receive a new outer MEMBER_OF
10846 // linking them to the parent struct:
10847 //
10848 // * (*) Entries with HasAttachPtr: they represent pointee data that
10849 // occupies a different storage block than the struct being mapped, so
10850 // they are not a member of it. They may still be MEMBER_OF an entry
10851 // within that pointee block, in which case those pre-existing bits are
10852 // shifted -- see (***).
10853 // * (**) ATTACH entries: they are not a member of anything — they just
10854 // link a ptr to its ptee.
10855 // * All entries when PreserveMemberOfFlags is set (the Flang/MLIR path):
10856 // its pre-shaped entries already carry their final MEMBER_OF bits.
10857 // TODO: set HasAttachPtr from Flang for entries whose storage is the
10858 // pointee's (e.g. s%p(0:10)) and drop PreserveMemberOfFlags in favor of
10859 // it.
10860 //
10861 // (***) If such an entry already has its own MEMBER_OF bits (e.g. the
10862 // s1p->x/y entries above), those bits are still shifted by N.
10863 Value *MemberMapType;
10864 if (PreserveMemberOfFlags || (RawType & AttachBit) ||
10865 Info->HasAttachPtr[I]) {
10866 if (RawType & MemberOfMask)
10867 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10868 else
10869 MemberMapType = OriMapType;
10870 } else {
10871 MemberMapType = Builder.CreateNUWAdd(LHS: OriMapType, RHS: ShiftedPreviousSize);
10872 }
10873
10874 // Combine the map type inherited from user-defined mapper with that
10875 // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM
10876 // bits of the \a MapType, which is the input argument of the mapper
10877 // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM
10878 // bits of MemberMapType.
10879 // [OpenMP 5.0], 1.2.6. map-type decay.
10880 // | alloc | to | from | tofrom | release | delete
10881 // ----------------------------------------------------------
10882 // alloc | alloc | alloc | alloc | alloc | release | delete
10883 // to | alloc | to | alloc | to | release | delete
10884 // from | alloc | alloc | from | from | release | delete
10885 // tofrom | alloc | to | from | tofrom | release | delete
10886 Value *LeftToFrom = Builder.CreateAnd(
10887 LHS: MapType,
10888 RHS: Builder.getInt64(
10889 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10890 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10891 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10892 BasicBlock *AllocBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc");
10893 BasicBlock *AllocElseBB =
10894 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.alloc.else");
10895 BasicBlock *ToBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to");
10896 BasicBlock *ToElseBB =
10897 BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.to.else");
10898 BasicBlock *FromBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.from");
10899 BasicBlock *EndBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.type.end");
10900 Value *IsAlloc = Builder.CreateIsNull(Arg: LeftToFrom);
10901 Builder.CreateCondBr(Cond: IsAlloc, True: AllocBB, False: AllocElseBB);
10902 // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM.
10903 emitBlock(BB: AllocBB, CurFn: MapperFn);
10904 Value *AllocMapType = Builder.CreateAnd(
10905 LHS: MemberMapType,
10906 RHS: Builder.getInt64(
10907 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10908 OpenMPOffloadMappingFlags::OMP_MAP_TO |
10909 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10910 Builder.CreateBr(Dest: EndBB);
10911 emitBlock(BB: AllocElseBB, CurFn: MapperFn);
10912 Value *IsTo = Builder.CreateICmpEQ(
10913 LHS: LeftToFrom,
10914 RHS: Builder.getInt64(
10915 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10916 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10917 Builder.CreateCondBr(Cond: IsTo, True: ToBB, False: ToElseBB);
10918 // In case of to, clear OMP_MAP_FROM.
10919 emitBlock(BB: ToBB, CurFn: MapperFn);
10920 Value *ToMapType = Builder.CreateAnd(
10921 LHS: MemberMapType,
10922 RHS: Builder.getInt64(
10923 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10924 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10925 Builder.CreateBr(Dest: EndBB);
10926 emitBlock(BB: ToElseBB, CurFn: MapperFn);
10927 Value *IsFrom = Builder.CreateICmpEQ(
10928 LHS: LeftToFrom,
10929 RHS: Builder.getInt64(
10930 C: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10931 OpenMPOffloadMappingFlags::OMP_MAP_FROM)));
10932 Builder.CreateCondBr(Cond: IsFrom, True: FromBB, False: EndBB);
10933 // In case of from, clear OMP_MAP_TO.
10934 emitBlock(BB: FromBB, CurFn: MapperFn);
10935 Value *FromMapType = Builder.CreateAnd(
10936 LHS: MemberMapType,
10937 RHS: Builder.getInt64(
10938 C: ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10939 OpenMPOffloadMappingFlags::OMP_MAP_TO)));
10940 // In case of tofrom, do nothing.
10941 emitBlock(BB: EndBB, CurFn: MapperFn);
10942 LastBB = EndBB;
10943 PHINode *CurMapType =
10944 Builder.CreatePHI(Ty: Builder.getInt64Ty(), NumReservedValues: 4, Name: "omp.maptype");
10945 CurMapType->addIncoming(V: AllocMapType, BB: AllocBB);
10946 CurMapType->addIncoming(V: ToMapType, BB: ToBB);
10947 CurMapType->addIncoming(V: FromMapType, BB: FromBB);
10948 CurMapType->addIncoming(V: MemberMapType, BB: ToElseBB);
10949
10950 // Propagate map-type-modifying bits from the outer map clause to each map
10951 // inserted by the mapper.
10952 //
10953 // OpenMP 6.0:281:34: The effect of the mapper modifier is to remove the
10954 // list item from the map clause and to apply the clauses specified in the
10955 // declared mapper to the construct on which the map clause appears...
10956 // If any modifier with the map-type-modifying property appears in the map
10957 // clause then the effect is as if that modifier appears in each map clause
10958 // specified in the declared mapper.
10959 //
10960 // Map-type-modifying bits: ALWAYS, DELETE, CLOSE, PRESENT.
10961 //
10962 // ALWAYS/DELETE/CLOSE are propagated to every (non-ATTACH) entry.
10963 //
10964 // PRESENT is propagated only to entries that have an attach ptr
10965 // (HasAttachPtr): the pointee data, which occupies a different storage
10966 // block than the struct being mapped and so is not covered by the
10967 // present-check on the struct's own storage. A present modifier on the
10968 // outer clause must still require that pointee to be present on the device.
10969 //
10970 // This is gated on \p PropagatePresentToPointee (set by callers only for
10971 // OpenMP >= 6.0). Before 6.0 the present modifier is treated as not
10972 // applying to the pointee: the spec committee confirmed the divergence
10973 // between the present "motion" modifier (to/from) and the present map-type
10974 // modifier (map) was unintentional, to be fixed as an OpenMP 6.0 erratum,
10975 // so for 5.2 present is ignored for the pointee for both map and to/from.
10976 //
10977 // TODO: PRESENT should also be propagated to the struct's own members
10978 // (e.g. the s.x, s.y of map(present, mapper(id): s)) so that an absent
10979 // member triggers the present-check. We cannot do that yet: while pointer
10980 // members are mapped with PTR_AND_OBJ, a single combined entry allocates
10981 // the whole struct (including the pointer's storage), so propagating
10982 // PRESENT to it would wrongly require the pointer's pointee to be present.
10983 // Enable member propagation once Clang stops emitting PTR_AND_OBJ and uses
10984 // attach-style maps throughout.
10985 uint64_t ModifierBits =
10986 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10987 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
10988 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
10989 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
10990 if (PropagatePresentToPointee && Info->HasAttachPtr[I])
10991 ModifierBits |=
10992 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
10993 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
10994 Value *ImportedModifierBits =
10995 Builder.CreateAnd(LHS: MapType, RHS: Builder.getInt64(C: ModifierBits));
10996 Value *CurMapTypeWithModifiers = Builder.CreateOr(
10997 LHS: CurMapType, RHS: ImportedModifierBits, Name: "omp.maptype.with.modifiers");
10998
10999 // ATTACH entries must not receive map-type-modifying bits: ATTACH|ALWAYS is
11000 // reserved for the attach(always) map-type modifier, and other modifier
11001 // bits (DELETE, CLOSE) have no meaning for an ATTACH entry.
11002 Value *FinalMapType =
11003 (RawType & AttachBit) ? CurMapType : CurMapTypeWithModifiers;
11004
11005 Value *OffloadingArgs[] = {MapperHandle, CurBaseArg, CurBeginArg,
11006 CurSizeArg, FinalMapType, CurNameArg};
11007
11008 auto ChildMapperFn = CustomMapperCB(I);
11009 if (!ChildMapperFn)
11010 return ChildMapperFn.takeError();
11011 if (*ChildMapperFn) {
11012 // Call the corresponding mapper function.
11013 createRuntimeFunctionCall(Callee: *ChildMapperFn, Args: OffloadingArgs)
11014 ->setDoesNotThrow();
11015 } else {
11016 // Call the runtime API __tgt_push_mapper_component to fill up the runtime
11017 // data structure.
11018 createRuntimeFunctionCall(
11019 Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_push_mapper_component),
11020 Args: OffloadingArgs);
11021 }
11022 }
11023
11024 // Update the pointer to point to the next element that needs to be mapped,
11025 // and check whether we have mapped all elements.
11026 Value *PtrNext = Builder.CreateConstGEP1_32(Ty: ElemTy, Ptr: PtrPHI, /*Idx0=*/1,
11027 Name: "omp.arraymap.next");
11028 PtrPHI->addIncoming(V: PtrNext, BB: LastBB);
11029 Value *IsDone = Builder.CreateICmpEQ(LHS: PtrNext, RHS: PtrEnd, Name: "omp.arraymap.isdone");
11030 BasicBlock *ExitBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp.arraymap.exit");
11031 Builder.CreateCondBr(Cond: IsDone, True: ExitBB, False: BodyBB);
11032
11033 emitBlock(BB: ExitBB, CurFn: MapperFn);
11034 // Emit array deletion if this is an array section and \p MapType indicates
11035 // that deletion is required.
11036 emitUDMapperArrayInitOrDel(MapperFn, MapperHandle, Base: BaseIn, Begin: BeginIn, Size,
11037 MapType, MapName, ElementSize, ExitBB: DoneBB,
11038 /*IsInit=*/false);
11039
11040 // Emit the function exit block.
11041 emitBlock(BB: DoneBB, CurFn: MapperFn, /*IsFinished=*/true);
11042
11043 Builder.CreateRetVoid();
11044 return MapperFn;
11045}
11046
11047Error OpenMPIRBuilder::emitOffloadingArrays(
11048 InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
11049 TargetDataInfo &Info, CustomMapperCallbackTy CustomMapperCB,
11050 bool IsNonContiguous,
11051 function_ref<void(unsigned int, Value *)> DeviceAddrCB) {
11052
11053 // Reset the array information.
11054 Info.clearArrayInfo();
11055 Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
11056
11057 if (Info.NumberOfPtrs == 0)
11058 return Error::success();
11059
11060 Builder.restoreIP(IP: AllocaIP);
11061 // Detect if we have any capture size requiring runtime evaluation of the
11062 // size so that a constant array could be eventually used.
11063 ArrayType *PointerArrayType =
11064 ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs);
11065
11066 Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
11067 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
11068
11069 Info.RTArgs.PointersArray = Builder.CreateAlloca(
11070 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_ptrs");
11071 AllocaInst *MappersArray = Builder.CreateAlloca(
11072 Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_mappers");
11073 Info.RTArgs.MappersArray = MappersArray;
11074
11075 // If we don't have any VLA types or other types that require runtime
11076 // evaluation, we can use a constant array for the map sizes, otherwise we
11077 // need to fill up the arrays as we do for the pointers.
11078 Type *Int64Ty = Builder.getInt64Ty();
11079 SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
11080 ConstantInt::get(Ty: Int64Ty, V: 0));
11081 SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
11082 for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
11083 bool IsNonContigEntry =
11084 IsNonContiguous &&
11085 (static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11086 CombinedInfo.Types[I] &
11087 OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG) != 0);
11088 // For NON_CONTIG entries, ArgSizes stores the dimension count (number of
11089 // descriptor_dim records), not the byte size.
11090 if (IsNonContigEntry) {
11091 assert(I < CombinedInfo.NonContigInfo.Dims.size() &&
11092 "Index must be in-bounds for NON_CONTIG Dims array");
11093 const uint64_t DimCount = CombinedInfo.NonContigInfo.Dims[I];
11094 assert(DimCount > 0 && "NON_CONTIG DimCount must be > 0");
11095 ConstSizes[I] = ConstantInt::get(Ty: Int64Ty, V: DimCount);
11096 continue;
11097 }
11098 if (auto *CI = dyn_cast<Constant>(Val: CombinedInfo.Sizes[I])) {
11099 if (!isa<ConstantExpr>(Val: CI) && !isa<GlobalValue>(Val: CI)) {
11100 ConstSizes[I] = CI;
11101 continue;
11102 }
11103 }
11104 RuntimeSizes.set(I);
11105 }
11106
11107 if (RuntimeSizes.all()) {
11108 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
11109 Info.RTArgs.SizesArray = Builder.CreateAlloca(
11110 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
11111 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11112 } else {
11113 auto *SizesArrayInit = ConstantArray::get(
11114 T: ArrayType::get(ElementType: Int64Ty, NumElements: ConstSizes.size()), V: ConstSizes);
11115 std::string Name = createPlatformSpecificName(Parts: {"offload_sizes"});
11116 auto *SizesArrayGbl =
11117 new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
11118 GlobalValue::PrivateLinkage, SizesArrayInit, Name);
11119 SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
11120
11121 if (!RuntimeSizes.any()) {
11122 Info.RTArgs.SizesArray = SizesArrayGbl;
11123 } else {
11124 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11125 Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(BitWidth: 64);
11126 ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
11127 AllocaInst *Buffer = Builder.CreateAlloca(
11128 Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
11129 Buffer->setAlignment(OffloadSizeAlign);
11130 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11131 Builder.CreateMemCpy(
11132 Dst: Buffer, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: Buffer->getType()),
11133 Src: SizesArrayGbl, SrcAlign: OffloadSizeAlign,
11134 Size: Builder.getIntN(
11135 N: IndexSize,
11136 C: Buffer->getAllocationSize(DL: M.getDataLayout())->getFixedValue()));
11137
11138 Info.RTArgs.SizesArray = Buffer;
11139 }
11140 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11141 }
11142
11143 // The map types are always constant so we don't need to generate code to
11144 // fill arrays. Instead, we create an array constant.
11145 SmallVector<uint64_t, 4> Mapping;
11146 for (auto mapFlag : CombinedInfo.Types)
11147 Mapping.push_back(
11148 Elt: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11149 mapFlag));
11150 std::string MaptypesName = createPlatformSpecificName(Parts: {"offload_maptypes"});
11151 auto *MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11152 Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
11153
11154 // The information types are only built if provided.
11155 if (!CombinedInfo.Names.empty()) {
11156 auto *MapNamesArrayGbl = createOffloadMapnames(
11157 Names&: CombinedInfo.Names, VarName: createPlatformSpecificName(Parts: {"offload_mapnames"}));
11158 Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
11159 Info.EmitDebug = true;
11160 } else {
11161 Info.RTArgs.MapNamesArray =
11162 Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext()));
11163 Info.EmitDebug = false;
11164 }
11165
11166 // If there's a present map type modifier, it must not be applied to the end
11167 // of a region, so generate a separate map type array in that case.
11168 if (Info.separateBeginEndCalls()) {
11169 bool EndMapTypesDiffer = false;
11170 for (uint64_t &Type : Mapping) {
11171 if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11172 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
11173 Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
11174 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
11175 EndMapTypesDiffer = true;
11176 }
11177 }
11178 if (EndMapTypesDiffer) {
11179 MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
11180 Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
11181 }
11182 }
11183
11184 PointerType *PtrTy = Builder.getPtrTy();
11185 for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
11186 Value *BPVal = CombinedInfo.BasePointers[I];
11187 Value *BP = Builder.CreateConstInBoundsGEP2_32(
11188 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.BasePointersArray,
11189 Idx0: 0, Idx1: I);
11190 Builder.CreateAlignedStore(Val: BPVal, Ptr: BP,
11191 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11192
11193 if (Info.requiresDevicePointerInfo()) {
11194 if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {
11195 CodeGenIP = Builder.saveIP();
11196 Builder.restoreIP(IP: AllocaIP);
11197 Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(Ty: PtrTy)};
11198 restoreIPandDebugLoc(Builder, IP: CodeGenIP);
11199 if (DeviceAddrCB)
11200 DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);
11201 } else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {
11202 Info.DevicePtrInfoMap[BPVal] = {BP, BP};
11203 if (DeviceAddrCB)
11204 DeviceAddrCB(I, BP);
11205 }
11206 }
11207
11208 Value *PVal = CombinedInfo.Pointers[I];
11209 Value *P = Builder.CreateConstInBoundsGEP2_32(
11210 Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray, Idx0: 0,
11211 Idx1: I);
11212 // TODO: Check alignment correct.
11213 Builder.CreateAlignedStore(Val: PVal, Ptr: P,
11214 Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11215
11216 if (RuntimeSizes.test(Idx: I)) {
11217 Value *S = Builder.CreateConstInBoundsGEP2_32(
11218 Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
11219 /*Idx0=*/0,
11220 /*Idx1=*/I);
11221 Builder.CreateAlignedStore(Val: Builder.CreateIntCast(V: CombinedInfo.Sizes[I],
11222 DestTy: Int64Ty,
11223 /*isSigned=*/true),
11224 Ptr: S, Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
11225 }
11226 // Fill up the mapper array.
11227 unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
11228 Value *MFunc = ConstantPointerNull::get(T: PtrTy);
11229
11230 auto CustomMFunc = CustomMapperCB(I);
11231 if (!CustomMFunc)
11232 return CustomMFunc.takeError();
11233 if (*CustomMFunc)
11234 MFunc = Builder.CreatePointerCast(V: *CustomMFunc, DestTy: PtrTy);
11235
11236 Value *MAddr = Builder.CreateInBoundsGEP(
11237 Ty: PointerArrayType, Ptr: MappersArray,
11238 IdxList: {Builder.getIntN(N: IndexSize, C: 0), Builder.getIntN(N: IndexSize, C: I)});
11239 Builder.CreateAlignedStore(
11240 Val: MFunc, Ptr: MAddr, Align: M.getDataLayout().getPrefTypeAlign(Ty: MAddr->getType()));
11241 }
11242
11243 if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
11244 Info.NumberOfPtrs == 0)
11245 return Error::success();
11246 emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
11247 return Error::success();
11248}
11249
11250void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
11251 BasicBlock *CurBB = Builder.GetInsertBlock();
11252
11253 if (!CurBB || CurBB->hasTerminator()) {
11254 // If there is no insert point or the previous block is already
11255 // terminated, don't touch it.
11256 } else {
11257 // Otherwise, create a fall-through branch.
11258 Builder.CreateBr(Dest: Target);
11259 }
11260
11261 Builder.ClearInsertionPoint();
11262}
11263
11264void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
11265 bool IsFinished) {
11266 BasicBlock *CurBB = Builder.GetInsertBlock();
11267
11268 // Fall out of the current block (if necessary).
11269 emitBranch(Target: BB);
11270
11271 if (IsFinished && BB->use_empty()) {
11272 BB->eraseFromParent();
11273 return;
11274 }
11275
11276 // Place the block after the current block, if possible, or else at
11277 // the end of the function.
11278 if (CurBB && CurBB->getParent())
11279 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
11280 else
11281 CurFn->insert(Position: CurFn->end(), BB);
11282 Builder.SetInsertPoint(BB);
11283}
11284
11285Error OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
11286 BodyGenCallbackTy ElseGen,
11287 InsertPointTy AllocaIP,
11288 ArrayRef<BasicBlock *> DeallocBlocks) {
11289 // If the condition constant folds and can be elided, try to avoid emitting
11290 // the condition and the dead arm of the if/else.
11291 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond)) {
11292 auto CondConstant = CI->getSExtValue();
11293 if (CondConstant)
11294 return ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11295
11296 return ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks);
11297 }
11298
11299 Function *CurFn = Builder.GetInsertBlock()->getParent();
11300
11301 // Otherwise, the condition did not fold, or we couldn't elide it. Just
11302 // emit the conditional branch.
11303 BasicBlock *ThenBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.then");
11304 BasicBlock *ElseBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.else");
11305 BasicBlock *ContBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.end");
11306 Builder.CreateCondBr(Cond, True: ThenBlock, False: ElseBlock);
11307 // Emit the 'then' code.
11308 emitBlock(BB: ThenBlock, CurFn);
11309 if (Error Err = ThenGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
11310 return Err;
11311 emitBranch(Target: ContBlock);
11312 // Emit the 'else' code if present.
11313 // There is no need to emit line number for unconditional branch.
11314 emitBlock(BB: ElseBlock, CurFn);
11315 if (Error Err = ElseGen(AllocaIP, Builder.saveIP(), DeallocBlocks))
11316 return Err;
11317 // There is no need to emit line number for unconditional branch.
11318 emitBranch(Target: ContBlock);
11319 // Emit the continuation block for code after the if.
11320 emitBlock(BB: ContBlock, CurFn, /*IsFinished=*/true);
11321 return Error::success();
11322}
11323
11324bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
11325 const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
11326 assert(!(AO == AtomicOrdering::NotAtomic ||
11327 AO == llvm::AtomicOrdering::Unordered) &&
11328 "Unexpected Atomic Ordering.");
11329
11330 bool Flush = false;
11331 llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
11332
11333 switch (AK) {
11334 case Read:
11335 if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
11336 AO == AtomicOrdering::SequentiallyConsistent) {
11337 FlushAO = AtomicOrdering::Acquire;
11338 Flush = true;
11339 }
11340 break;
11341 case Write:
11342 case Compare:
11343 case Update:
11344 if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
11345 AO == AtomicOrdering::SequentiallyConsistent) {
11346 FlushAO = AtomicOrdering::Release;
11347 Flush = true;
11348 }
11349 break;
11350 case Capture:
11351 switch (AO) {
11352 case AtomicOrdering::Acquire:
11353 FlushAO = AtomicOrdering::Acquire;
11354 Flush = true;
11355 break;
11356 case AtomicOrdering::Release:
11357 FlushAO = AtomicOrdering::Release;
11358 Flush = true;
11359 break;
11360 case AtomicOrdering::AcquireRelease:
11361 case AtomicOrdering::SequentiallyConsistent:
11362 FlushAO = AtomicOrdering::AcquireRelease;
11363 Flush = true;
11364 break;
11365 default:
11366 // do nothing - leave silently.
11367 break;
11368 }
11369 }
11370
11371 if (Flush) {
11372 // Currently Flush RT call still doesn't take memory_ordering, so for when
11373 // that happens, this tries to do the resolution of which atomic ordering
11374 // to use with but issue the flush call
11375 // TODO: pass `FlushAO` after memory ordering support is added
11376 (void)FlushAO;
11377 emitFlush(Loc);
11378 }
11379
11380 // for AO == AtomicOrdering::Monotonic and all other case combinations
11381 // do nothing
11382 return Flush;
11383}
11384
11385OpenMPIRBuilder::InsertPointTy
11386OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
11387 AtomicOpValue &X, AtomicOpValue &V,
11388 AtomicOrdering AO, InsertPointTy AllocaIP) {
11389 if (!updateToLocation(Loc))
11390 return Loc.IP;
11391
11392 assert(X.Var->getType()->isPointerTy() &&
11393 "OMP Atomic expects a pointer to target memory");
11394 Type *XElemTy = X.ElemTy;
11395 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11396 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11397 "OMP atomic read expected a scalar type");
11398
11399 Value *XRead = nullptr;
11400
11401 if (XElemTy->isIntegerTy()) {
11402 LoadInst *XLD =
11403 Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.read");
11404 XLD->setAtomic(Ordering: AO);
11405 XRead = cast<Value>(Val: XLD);
11406 } else if (XElemTy->isStructTy()) {
11407 // FIXME: Add checks to ensure __atomic_load is emitted iff the
11408 // target does not support `atomicrmw` of the size of the struct
11409 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11410 OldVal->setAtomic(Ordering: AO);
11411 const DataLayout &DL = OldVal->getDataLayout();
11412 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11413 OpenMPIRBuilder::AtomicInfo atomicInfo(
11414 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11415 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11416 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11417 XRead = AtomicLoadRes.first;
11418 OldVal->eraseFromParent();
11419 } else {
11420 // We need to perform atomic op as integer
11421 IntegerType *IntCastTy =
11422 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11423 LoadInst *XLoad =
11424 Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.load");
11425 XLoad->setAtomic(Ordering: AO);
11426 if (XElemTy->isFloatingPointTy()) {
11427 XRead = Builder.CreateBitCast(V: XLoad, DestTy: XElemTy, Name: "atomic.flt.cast");
11428 } else {
11429 XRead = Builder.CreateIntToPtr(V: XLoad, DestTy: XElemTy, Name: "atomic.ptr.cast");
11430 }
11431 }
11432 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Read);
11433 Builder.CreateStore(Val: XRead, Ptr: V.Var, isVolatile: V.IsVolatile);
11434 return Builder.saveIP();
11435}
11436
11437OpenMPIRBuilder::InsertPointTy
11438OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
11439 AtomicOpValue &X, Value *Expr,
11440 AtomicOrdering AO, InsertPointTy AllocaIP) {
11441 if (!updateToLocation(Loc))
11442 return Loc.IP;
11443
11444 assert(X.Var->getType()->isPointerTy() &&
11445 "OMP Atomic expects a pointer to target memory");
11446 Type *XElemTy = X.ElemTy;
11447 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11448 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11449 "OMP atomic write expected a scalar type");
11450
11451 if (XElemTy->isIntegerTy()) {
11452 StoreInst *XSt = Builder.CreateStore(Val: Expr, Ptr: X.Var, isVolatile: X.IsVolatile);
11453 XSt->setAtomic(Ordering: AO);
11454 } else if (XElemTy->isStructTy()) {
11455 LoadInst *OldVal = Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, Name: "omp.atomic.read");
11456 const DataLayout &DL = OldVal->getDataLayout();
11457 unsigned LoadSize = DL.getTypeStoreSize(Ty: XElemTy);
11458 OpenMPIRBuilder::AtomicInfo atomicInfo(
11459 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11460 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X.Var);
11461 atomicInfo.EmitAtomicStoreLibcall(AO, Source: Expr);
11462 OldVal->eraseFromParent();
11463 } else {
11464 // We need to bitcast and perform atomic op as integers
11465 IntegerType *IntCastTy =
11466 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11467 Value *ExprCast =
11468 Builder.CreateBitCast(V: Expr, DestTy: IntCastTy, Name: "atomic.src.int.cast");
11469 StoreInst *XSt = Builder.CreateStore(Val: ExprCast, Ptr: X.Var, isVolatile: X.IsVolatile);
11470 XSt->setAtomic(Ordering: AO);
11471 }
11472
11473 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Write);
11474 return Builder.saveIP();
11475}
11476
11477OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
11478 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11479 Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11480 AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
11481 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11482 assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
11483 if (!updateToLocation(Loc))
11484 return Loc.IP;
11485
11486 LLVM_DEBUG({
11487 Type *XTy = X.Var->getType();
11488 assert(XTy->isPointerTy() &&
11489 "OMP Atomic expects a pointer to target memory");
11490 Type *XElemTy = X.ElemTy;
11491 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11492 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11493 "OMP atomic update expected a scalar or struct type");
11494 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11495 (RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&
11496 "OpenMP atomic does not support LT or GT operations");
11497 });
11498
11499 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11500 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp, UpdateOp, VolatileX: X.IsVolatile,
11501 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11502 if (!AtomicResult)
11503 return AtomicResult.takeError();
11504 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Update);
11505 return Builder.saveIP();
11506}
11507
11508// FIXME: Duplicating AtomicExpand
11509Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
11510 AtomicRMWInst::BinOp RMWOp) {
11511 switch (RMWOp) {
11512 case AtomicRMWInst::Add:
11513 return Builder.CreateAdd(LHS: Src1, RHS: Src2);
11514 case AtomicRMWInst::Sub:
11515 return Builder.CreateSub(LHS: Src1, RHS: Src2);
11516 case AtomicRMWInst::And:
11517 return Builder.CreateAnd(LHS: Src1, RHS: Src2);
11518 case AtomicRMWInst::Nand:
11519 return Builder.CreateNeg(V: Builder.CreateAnd(LHS: Src1, RHS: Src2));
11520 case AtomicRMWInst::Or:
11521 return Builder.CreateOr(LHS: Src1, RHS: Src2);
11522 case AtomicRMWInst::Xor:
11523 return Builder.CreateXor(LHS: Src1, RHS: Src2);
11524 case AtomicRMWInst::Xchg:
11525 case AtomicRMWInst::FAdd:
11526 case AtomicRMWInst::FSub:
11527 case AtomicRMWInst::BAD_BINOP:
11528 case AtomicRMWInst::Max:
11529 case AtomicRMWInst::Min:
11530 case AtomicRMWInst::UMax:
11531 case AtomicRMWInst::UMin:
11532 case AtomicRMWInst::FMax:
11533 case AtomicRMWInst::FMin:
11534 case AtomicRMWInst::FMaximum:
11535 case AtomicRMWInst::FMinimum:
11536 case AtomicRMWInst::FMaximumNum:
11537 case AtomicRMWInst::FMinimumNum:
11538 case AtomicRMWInst::UIncWrap:
11539 case AtomicRMWInst::UDecWrap:
11540 case AtomicRMWInst::USubCond:
11541 case AtomicRMWInst::USubSat:
11542 llvm_unreachable("Unsupported atomic update operation");
11543 }
11544 llvm_unreachable("Unsupported atomic update operation");
11545}
11546
11547static AtomicOrdering TransformReleaseAcquireRelease(AtomicOrdering AO) {
11548 // Loads cannot use Release or AcquireRelease ordering. This load is
11549 // just the initial value for the cmpxchg loop; the cmpxchg itself
11550 // retains the original ordering.
11551 AtomicOrdering LoadAO = AO;
11552
11553 if (AO == AtomicOrdering::Release) {
11554 LoadAO = AtomicOrdering::Monotonic;
11555 } else if (AO == AtomicOrdering::AcquireRelease) {
11556 LoadAO = AtomicOrdering::Acquire;
11557 }
11558
11559 return LoadAO;
11560}
11561
11562Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
11563 InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
11564 AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
11565 AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
11566 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11567 // TODO: handle the case where XElemTy is not byte-sized or not a power of 2.
11568 bool emitRMWOp = false;
11569 switch (RMWOp) {
11570 case AtomicRMWInst::Add:
11571 case AtomicRMWInst::And:
11572 case AtomicRMWInst::Nand:
11573 case AtomicRMWInst::Or:
11574 case AtomicRMWInst::Xor:
11575 case AtomicRMWInst::Xchg:
11576 emitRMWOp = XElemTy;
11577 break;
11578 case AtomicRMWInst::Sub:
11579 emitRMWOp = (IsXBinopExpr && XElemTy);
11580 break;
11581 default:
11582 emitRMWOp = false;
11583 }
11584 emitRMWOp &= XElemTy->isIntegerTy();
11585
11586 std::pair<Value *, Value *> Res;
11587 if (emitRMWOp) {
11588 AtomicRMWInst *RMWInst =
11589 Builder.CreateAtomicRMW(Op: RMWOp, Ptr: X, Val: Expr, Align: llvm::MaybeAlign(), Ordering: AO);
11590 if (IsIgnoreDenormalMode)
11591 RMWInst->setMetadata(KindID: llvm::LLVMContext::MD_atomic_ignore_denormal_mode,
11592 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11593 if (T.isAMDGPU()) {
11594 if (!IsFineGrainedMemory)
11595 RMWInst->setMetadata(Kind: "amdgpu.no.fine.grained.memory",
11596 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11597 if (!IsRemoteMemory)
11598 RMWInst->setMetadata(Kind: "amdgpu.no.remote.memory",
11599 Node: llvm::MDNode::get(Context&: Builder.getContext(), MDs: {}));
11600 }
11601 Res.first = RMWInst;
11602 // not needed except in case of postfix captures. Generate anyway for
11603 // consistency with the else part. Will be removed with any DCE pass.
11604 // AtomicRMWInst::Xchg does not have a coressponding instruction.
11605 if (RMWOp == AtomicRMWInst::Xchg)
11606 Res.second = Res.first;
11607 else
11608 Res.second = emitRMWOpAsInstruction(Src1: Res.first, Src2: Expr, RMWOp);
11609 } else if (XElemTy->isStructTy()) {
11610 LoadInst *OldVal =
11611 Builder.CreateLoad(Ty: XElemTy, Ptr: X, Name: X->getName() + ".atomic.load");
11612 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11613 OldVal->setAtomic(Ordering: LoadAO);
11614 const DataLayout &LoadDL = OldVal->getDataLayout();
11615 unsigned LoadSize = LoadDL.getTypeStoreSize(Ty: XElemTy);
11616
11617 OpenMPIRBuilder::AtomicInfo atomicInfo(
11618 &Builder, XElemTy, LoadSize * 8, LoadSize * 8, OldVal->getAlign(),
11619 OldVal->getAlign(), true /* UseLibcall */, AllocaIP, X);
11620 auto AtomicLoadRes = atomicInfo.EmitAtomicLoadLibcall(AO);
11621 BasicBlock *CurBB = Builder.GetInsertBlock();
11622 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11623 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11624 BasicBlock *ExitBB =
11625 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11626 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11627 BBName: X->getName() + ".atomic.cont");
11628 ContBB->getTerminator()->eraseFromParent();
11629 Builder.restoreIP(IP: AllocaIP);
11630 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11631 NewAtomicAddr->setName(X->getName() + "x.new.val");
11632 Builder.SetInsertPoint(ContBB);
11633 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11634 PHI->addIncoming(V: AtomicLoadRes.first, BB: CurBB);
11635 Value *OldExprVal = PHI;
11636 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11637 if (!CBResult)
11638 return CBResult.takeError();
11639 Value *Upd = *CBResult;
11640 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11641 AtomicOrdering Failure =
11642 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11643 auto Result = atomicInfo.EmitAtomicCompareExchangeLibcall(
11644 ExpectedVal: AtomicLoadRes.second, DesiredVal: NewAtomicAddr, Success: AO, Failure);
11645 LoadInst *PHILoad = Builder.CreateLoad(Ty: XElemTy, Ptr: Result.first);
11646 PHI->addIncoming(V: PHILoad, BB: Builder.GetInsertBlock());
11647 Builder.CreateCondBr(Cond: Result.second, True: ExitBB, False: ContBB);
11648 OldVal->eraseFromParent();
11649 Res.first = OldExprVal;
11650 Res.second = Upd;
11651
11652 if (UnreachableInst *ExitTI =
11653 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11654 CurBBTI->eraseFromParent();
11655 Builder.SetInsertPoint(ExitBB);
11656 } else {
11657 Builder.SetInsertPoint(ExitTI);
11658 }
11659 } else {
11660 IntegerType *IntCastTy =
11661 IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
11662 LoadInst *OldVal =
11663 Builder.CreateLoad(Ty: IntCastTy, Ptr: X, Name: X->getName() + ".atomic.load");
11664 AtomicOrdering LoadAO = TransformReleaseAcquireRelease(AO);
11665 OldVal->setAtomic(Ordering: LoadAO);
11666 // CurBB
11667 // | /---\
11668 // ContBB |
11669 // | \---/
11670 // ExitBB
11671 BasicBlock *CurBB = Builder.GetInsertBlock();
11672 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11673 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11674 BasicBlock *ExitBB =
11675 CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
11676 BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
11677 BBName: X->getName() + ".atomic.cont");
11678 ContBB->getTerminator()->eraseFromParent();
11679 Builder.restoreIP(IP: AllocaIP);
11680 AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
11681 NewAtomicAddr->setName(X->getName() + "x.new.val");
11682 Builder.SetInsertPoint(ContBB);
11683 llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
11684 PHI->addIncoming(V: OldVal, BB: CurBB);
11685 bool IsIntTy = XElemTy->isIntegerTy();
11686 Value *OldExprVal = PHI;
11687 if (!IsIntTy) {
11688 if (XElemTy->isFloatingPointTy()) {
11689 OldExprVal = Builder.CreateBitCast(V: PHI, DestTy: XElemTy,
11690 Name: X->getName() + ".atomic.fltCast");
11691 } else {
11692 OldExprVal = Builder.CreateIntToPtr(V: PHI, DestTy: XElemTy,
11693 Name: X->getName() + ".atomic.ptrCast");
11694 }
11695 }
11696
11697 Expected<Value *> CBResult = UpdateOp(OldExprVal, Builder);
11698 if (!CBResult)
11699 return CBResult.takeError();
11700 Value *Upd = *CBResult;
11701 Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
11702 LoadInst *DesiredVal = Builder.CreateLoad(Ty: IntCastTy, Ptr: NewAtomicAddr);
11703 AtomicOrdering Failure =
11704 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11705 AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
11706 Ptr: X, Cmp: PHI, New: DesiredVal, Align: llvm::MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11707 Result->setVolatile(VolatileX);
11708 Value *PreviousVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11709 Value *SuccessFailureVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11710 PHI->addIncoming(V: PreviousVal, BB: Builder.GetInsertBlock());
11711 Builder.CreateCondBr(Cond: SuccessFailureVal, True: ExitBB, False: ContBB);
11712
11713 Res.first = OldExprVal;
11714 Res.second = Upd;
11715
11716 // set Insertion point in exit block
11717 if (UnreachableInst *ExitTI =
11718 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11719 CurBBTI->eraseFromParent();
11720 Builder.SetInsertPoint(ExitBB);
11721 } else {
11722 Builder.SetInsertPoint(ExitTI);
11723 }
11724 }
11725
11726 return Res;
11727}
11728
11729OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
11730 const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
11731 AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
11732 AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
11733 bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
11734 bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
11735 if (!updateToLocation(Loc))
11736 return Loc.IP;
11737
11738 LLVM_DEBUG({
11739 Type *XTy = X.Var->getType();
11740 assert(XTy->isPointerTy() &&
11741 "OMP Atomic expects a pointer to target memory");
11742 Type *XElemTy = X.ElemTy;
11743 assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
11744 XElemTy->isPointerTy() || XElemTy->isStructTy()) &&
11745 "OMP atomic capture expected a scalar or struct type");
11746 assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
11747 "OpenMP atomic does not support LT or GT operations");
11748 });
11749
11750 // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
11751 // 'x' is simply atomically rewritten with 'expr'.
11752 AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
11753 Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
11754 AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp: AtomicOp, UpdateOp, VolatileX: X.IsVolatile,
11755 IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
11756 if (!AtomicResult)
11757 return AtomicResult.takeError();
11758 Value *CapturedVal =
11759 (IsPostfixUpdate ? AtomicResult->first : AtomicResult->second);
11760 Builder.CreateStore(Val: CapturedVal, Ptr: V.Var, isVolatile: V.IsVolatile);
11761
11762 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Capture);
11763 return Builder.saveIP();
11764}
11765
11766OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11767 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11768 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11769 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11770 bool IsFailOnly, bool IsWeak) {
11771
11772 AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
11773 return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
11774 IsPostfixUpdate, IsFailOnly, Failure, IsWeak);
11775}
11776
11777OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
11778 const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
11779 AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
11780 omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
11781 bool IsFailOnly, AtomicOrdering Failure, bool IsWeak) {
11782
11783 if (!updateToLocation(Loc))
11784 return Loc.IP;
11785
11786 assert(X.Var->getType()->isPointerTy() &&
11787 "OMP atomic expects a pointer to target memory");
11788 // compare capture
11789 if (V.Var) {
11790 assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");
11791 assert(V.ElemTy == X.ElemTy && "x and v must be of same type");
11792 }
11793
11794 bool IsInteger = E->getType()->isIntegerTy();
11795
11796 if (Op == OMPAtomicCompareOp::EQ) {
11797 // OldValue and SuccessOrFail are set below and used in the shared V.Var /
11798 // R.Var handling.
11799 Value *OldValue = nullptr;
11800 Value *SuccessOrFail = nullptr;
11801
11802 if (!IsInteger && HandleFPNegZero) {
11803 // IEEE 754 special cases for cmpxchg (which is bitwise):
11804 // 1. -0.0 == +0.0 but they have different bit patterns.
11805 // 2. NaN != NaN but identical NaN bit patterns would match.
11806 //
11807 // CurBB:
11808 // %e_int = bitcast E to intN
11809 // %d_int = bitcast D to intN
11810 // %x_curr = load atomic intN, X
11811 // %x_fp = bitcast %x_curr to FP
11812 // %e_is_nan = fcmp uno E, E
11813 // %x_is_nan = fcmp uno %x_fp, %x_fp
11814 // %either_nan = or %e_is_nan, %x_is_nan
11815 // br %either_nan, NaNBB, NotNaNBB
11816 // NaNBB: ; NaN == anything is always false
11817 // br ExitBB
11818 // NotNaNBB:
11819 // %x_is_zero = fcmp oeq %x_fp, 0.0
11820 // %e_is_zero = fcmp oeq E, 0.0
11821 // %both_zero = and %x_is_zero, %e_is_zero
11822 // br %both_zero, ZeroBB, NormalBB
11823 // ZeroBB: ; both ±0.0 → x = d
11824 // cmpxchg X, %x_curr, %d_int
11825 // br ExitBB
11826 // NormalBB: ; original path
11827 // cmpxchg X, %e_int, %d_int
11828 // br ExitBB
11829 // ExitBB:
11830 // phi merge
11831 IntegerType *IntCastTy =
11832 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11833 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11834 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11835
11836 // Load X atomically.
11837 LoadInst *XCurr = Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var,
11838 Name: X.Var->getName() + ".atomic.load");
11839 XCurr->setAtomic(Ordering: AtomicOrdering::Monotonic);
11840 Value *XFP = Builder.CreateBitCast(V: XCurr, DestTy: X.ElemTy);
11841
11842 // IEEE 754: NaN != NaN, but cmpxchg would succeed if E and X have
11843 // the same NaN bit pattern. Skip cmpxchg when either is NaN.
11844 Value *EIsNaN = Builder.CreateFCmpUNO(LHS: E, RHS: E, Name: "atomic.e.isnan");
11845 Value *XIsNaN = Builder.CreateFCmpUNO(LHS: XFP, RHS: XFP, Name: "atomic.x.isnan");
11846 Value *EitherNaN = Builder.CreateOr(LHS: EIsNaN, RHS: XIsNaN, Name: "atomic.either.nan");
11847
11848 BasicBlock *CurBB = Builder.GetInsertBlock();
11849 Function *F = CurBB->getParent();
11850 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11851 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11852 BasicBlock *ExitBB =
11853 CurBB->splitBasicBlock(I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11854 BasicBlock *NaNBB = BasicBlock::Create(
11855 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.nan", Parent: F, InsertBefore: ExitBB);
11856 BasicBlock *NotNaNBB = BasicBlock::Create(
11857 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.notnan", Parent: F, InsertBefore: ExitBB);
11858 BasicBlock *ZeroBB = BasicBlock::Create(
11859 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.zero", Parent: F, InsertBefore: ExitBB);
11860 BasicBlock *NormalBB = BasicBlock::Create(
11861 Context&: M.getContext(), Name: X.Var->getName() + ".atomic.normal", Parent: F, InsertBefore: ExitBB);
11862
11863 // If either E or X is NaN → NaNBB (always fails), else check for ±0.0.
11864 CurBB->getTerminator()->eraseFromParent();
11865 Builder.SetInsertPoint(CurBB);
11866 Builder.CreateCondBr(Cond: EitherNaN, True: NaNBB, False: NotNaNBB);
11867
11868 // NaNBB: NaN == anything is always false; skip cmpxchg.
11869 Builder.SetInsertPoint(NaNBB);
11870 Builder.CreateBr(Dest: ExitBB);
11871
11872 // NotNaNBB: check both X and E for ±0.0.
11873 Builder.SetInsertPoint(NotNaNBB);
11874 Value *XIsZero =
11875 Builder.CreateFCmpOEQ(LHS: XFP, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11876 Name: X.Var->getName() + ".atomic.xiszero");
11877 Value *EIsZero = Builder.CreateFCmpOEQ(LHS: E, RHS: ConstantFP::getZero(Ty: X.ElemTy),
11878 Name: "atomic.e.iszero");
11879 Value *BothZero = Builder.CreateAnd(LHS: XIsZero, RHS: EIsZero, Name: "atomic.both.zero");
11880 Builder.CreateCondBr(Cond: BothZero, True: ZeroBB, False: NormalBB);
11881
11882 // ZeroBB: cmpxchg with X's loaded bit-pattern.
11883 Builder.SetInsertPoint(ZeroBB);
11884 AtomicCmpXchgInst *ResZero = Builder.CreateAtomicCmpXchg(
11885 Ptr: X.Var, Cmp: XCurr, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11886 ResZero->setWeak(IsWeak);
11887 Value *OldZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/0);
11888 Value *OkZero = Builder.CreateExtractValue(Agg: ResZero, /*Idxs=*/1);
11889 Builder.CreateBr(Dest: ExitBB);
11890
11891 // NormalBB: original bitwise cmpxchg.
11892 Builder.SetInsertPoint(NormalBB);
11893 AtomicCmpXchgInst *ResNormal = Builder.CreateAtomicCmpXchg(
11894 Ptr: X.Var, Cmp: EBCast, New: DBCast, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11895 ResNormal->setWeak(IsWeak);
11896 Value *OldNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/0);
11897 Value *OkNormal = Builder.CreateExtractValue(Agg: ResNormal, /*Idxs=*/1);
11898 Builder.CreateBr(Dest: ExitBB);
11899
11900 // ExitBB: merge results from NaN, Zero, and Normal paths.
11901 Builder.SetInsertPoint(ExitBB->begin());
11902 PHINode *OldIntPHI =
11903 Builder.CreatePHI(Ty: IntCastTy, NumReservedValues: 3, Name: X.Var->getName() + ".atomic.old");
11904 OldIntPHI->addIncoming(V: XCurr, BB: NaNBB);
11905 OldIntPHI->addIncoming(V: OldZero, BB: ZeroBB);
11906 OldIntPHI->addIncoming(V: OldNormal, BB: NormalBB);
11907 PHINode *SuccessPHI = Builder.CreatePHI(Ty: Builder.getInt1Ty(), NumReservedValues: 3,
11908 Name: X.Var->getName() + ".atomic.ok");
11909 SuccessPHI->addIncoming(V: Builder.getFalse(), BB: NaNBB);
11910 SuccessPHI->addIncoming(V: OkZero, BB: ZeroBB);
11911 SuccessPHI->addIncoming(V: OkNormal, BB: NormalBB);
11912
11913 if (isa<UnreachableInst>(Val: ExitBB->getTerminator())) {
11914 CurBBTI->eraseFromParent();
11915 Builder.SetInsertPoint(ExitBB);
11916 } else {
11917 Builder.SetInsertPoint(&*ExitBB->getFirstNonPHIIt());
11918 }
11919
11920 OldValue = Builder.CreateBitCast(V: OldIntPHI, DestTy: X.ElemTy,
11921 Name: X.Var->getName() + ".atomic.old.fp");
11922 SuccessOrFail = SuccessPHI;
11923 } else {
11924 AtomicCmpXchgInst *Result = nullptr;
11925 if (!IsInteger) {
11926 IntegerType *IntCastTy =
11927 IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
11928 Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
11929 Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
11930 Result = Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: EBCast, New: DBCast,
11931 Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11932 } else {
11933 Result =
11934 Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: E, New: D, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
11935 }
11936 Result->setWeak(IsWeak);
11937
11938 if (V.Var) {
11939 OldValue = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
11940 if (!IsInteger)
11941 OldValue = Builder.CreateBitCast(V: OldValue, DestTy: X.ElemTy);
11942 assert(OldValue->getType() == V.ElemTy &&
11943 "OldValue and V must be of same type");
11944 if (IsPostfixUpdate) {
11945 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11946 } else {
11947 SuccessOrFail = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11948 if (IsFailOnly) {
11949 BasicBlock *CurBB = Builder.GetInsertBlock();
11950 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
11951 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
11952 BasicBlock *ExitBB = CurBB->splitBasicBlock(
11953 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
11954 BasicBlock *ContBB = CurBB->splitBasicBlock(
11955 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
11956 ContBB->getTerminator()->eraseFromParent();
11957 CurBB->getTerminator()->eraseFromParent();
11958
11959 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
11960
11961 Builder.SetInsertPoint(ContBB);
11962 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
11963 Builder.CreateBr(Dest: ExitBB);
11964
11965 if (UnreachableInst *ExitTI =
11966 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
11967 CurBBTI->eraseFromParent();
11968 Builder.SetInsertPoint(ExitBB);
11969 } else {
11970 Builder.SetInsertPoint(ExitTI);
11971 }
11972 } else {
11973 Value *CapturedValue =
11974 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
11975 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
11976 }
11977 }
11978 }
11979 // The comparison result has to be stored.
11980 if (R.Var) {
11981 assert(R.Var->getType()->isPointerTy() &&
11982 "r.var must be of pointer type");
11983 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
11984
11985 Value *SuccessFailureVal =
11986 Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
11987 Value *ResultCast =
11988 R.IsSigned ? Builder.CreateSExt(V: SuccessFailureVal, DestTy: R.ElemTy)
11989 : Builder.CreateZExt(V: SuccessFailureVal, DestTy: R.ElemTy);
11990 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
11991 }
11992 }
11993
11994 // For the HandleFPNegZero path, handle V.Var and R.Var using the
11995 // pre-computed OldValue and SuccessOrFail.
11996 if (HandleFPNegZero && !IsInteger) {
11997 if (V.Var) {
11998 assert(OldValue->getType() == V.ElemTy &&
11999 "OldValue and V must be of same type");
12000 if (IsPostfixUpdate) {
12001 Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12002 } else {
12003 if (IsFailOnly) {
12004 BasicBlock *CurBB = Builder.GetInsertBlock();
12005 Instruction *CurBBTI = CurBB->getTerminatorOrNull();
12006 CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
12007 BasicBlock *ExitBB = CurBB->splitBasicBlock(
12008 I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
12009 BasicBlock *ContBB = CurBB->splitBasicBlock(
12010 I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
12011 ContBB->getTerminator()->eraseFromParent();
12012 CurBB->getTerminator()->eraseFromParent();
12013
12014 Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
12015
12016 Builder.SetInsertPoint(ContBB);
12017 Builder.CreateStore(Val: OldValue, Ptr: V.Var);
12018 Builder.CreateBr(Dest: ExitBB);
12019
12020 if (UnreachableInst *ExitTI =
12021 dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
12022 CurBBTI->eraseFromParent();
12023 Builder.SetInsertPoint(ExitBB);
12024 } else {
12025 Builder.SetInsertPoint(ExitTI);
12026 }
12027 } else {
12028 Value *CapturedValue =
12029 Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
12030 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12031 }
12032 }
12033 }
12034 // The comparison result has to be stored.
12035 if (R.Var) {
12036 assert(R.Var->getType()->isPointerTy() &&
12037 "r.var must be of pointer type");
12038 assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
12039
12040 Value *ResultCast = R.IsSigned
12041 ? Builder.CreateSExt(V: SuccessOrFail, DestTy: R.ElemTy)
12042 : Builder.CreateZExt(V: SuccessOrFail, DestTy: R.ElemTy);
12043 Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
12044 }
12045 }
12046 } else {
12047 assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&
12048 "Op should be either max or min at this point");
12049 assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");
12050
12051 // Reverse the ordop as the OpenMP forms are different from LLVM forms.
12052 // Let's take max as example.
12053 // OpenMP form:
12054 // x = x > expr ? expr : x;
12055 // LLVM form:
12056 // *ptr = *ptr > val ? *ptr : val;
12057 // We need to transform to LLVM form.
12058 // x = x <= expr ? x : expr;
12059 AtomicRMWInst::BinOp NewOp;
12060 if (IsXBinopExpr) {
12061 if (IsInteger) {
12062 if (X.IsSigned)
12063 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
12064 : AtomicRMWInst::Max;
12065 else
12066 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
12067 : AtomicRMWInst::UMax;
12068 } else {
12069 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
12070 : AtomicRMWInst::FMax;
12071 }
12072 } else {
12073 if (IsInteger) {
12074 if (X.IsSigned)
12075 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
12076 : AtomicRMWInst::Min;
12077 else
12078 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
12079 : AtomicRMWInst::UMin;
12080 } else {
12081 NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
12082 : AtomicRMWInst::FMin;
12083 }
12084 }
12085
12086 AtomicRMWInst *OldValue =
12087 Builder.CreateAtomicRMW(Op: NewOp, Ptr: X.Var, Val: E, Align: MaybeAlign(), Ordering: AO);
12088 if (V.Var) {
12089 Value *CapturedValue = nullptr;
12090 if (IsPostfixUpdate) {
12091 CapturedValue = OldValue;
12092 } else {
12093 CmpInst::Predicate Pred;
12094 switch (NewOp) {
12095 case AtomicRMWInst::Max:
12096 Pred = CmpInst::ICMP_SGT;
12097 break;
12098 case AtomicRMWInst::UMax:
12099 Pred = CmpInst::ICMP_UGT;
12100 break;
12101 case AtomicRMWInst::FMax:
12102 Pred = CmpInst::FCMP_OGT;
12103 break;
12104 case AtomicRMWInst::Min:
12105 Pred = CmpInst::ICMP_SLT;
12106 break;
12107 case AtomicRMWInst::UMin:
12108 Pred = CmpInst::ICMP_ULT;
12109 break;
12110 case AtomicRMWInst::FMin:
12111 Pred = CmpInst::FCMP_OLT;
12112 break;
12113 default:
12114 llvm_unreachable("unexpected comparison op");
12115 }
12116 Value *NonAtomicCmp = Builder.CreateCmp(Pred, LHS: OldValue, RHS: E);
12117 CapturedValue = Builder.CreateSelect(C: NonAtomicCmp, True: E, False: OldValue);
12118 }
12119 Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
12120 }
12121 }
12122
12123 checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Compare);
12124
12125 return Builder.saveIP();
12126}
12127
12128OpenMPIRBuilder::InsertPointOrErrorTy
12129OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
12130 BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
12131 Value *NumTeamsUpper, Value *ThreadLimit,
12132 Value *IfExpr) {
12133 if (!updateToLocation(Loc))
12134 return InsertPointTy();
12135
12136 uint32_t SrcLocStrSize;
12137 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
12138 Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
12139 Function *CurrentFunction = Builder.GetInsertBlock()->getParent();
12140
12141 // Outer allocation basicblock is the entry block of the current function.
12142 BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();
12143 if (&OuterAllocaBB == Builder.GetInsertBlock()) {
12144 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.entry");
12145 Builder.SetInsertPoint(BodyBB->begin());
12146 }
12147
12148 // The current basic block is split into four basic blocks. After outlining,
12149 // they will be mapped as follows:
12150 // ```
12151 // def current_fn() {
12152 // current_basic_block:
12153 // br label %teams.exit
12154 // teams.exit:
12155 // ; instructions after teams
12156 // }
12157 //
12158 // def outlined_fn() {
12159 // teams.alloca:
12160 // br label %teams.body
12161 // teams.body:
12162 // ; instructions within teams body
12163 // }
12164 // ```
12165 BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.exit");
12166 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.body");
12167 BasicBlock *AllocaBB =
12168 splitBB(Builder, /*CreateBranch=*/true, Name: "teams.alloca");
12169
12170 bool SubClausesPresent =
12171 (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
12172 // Push num_teams
12173 if (!Config.isTargetDevice() && SubClausesPresent) {
12174 assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
12175 "if lowerbound is non-null, then upperbound must also be non-null "
12176 "for bounds on num_teams");
12177
12178 if (NumTeamsUpper == nullptr)
12179 NumTeamsUpper = Builder.getInt32(C: 0);
12180
12181 if (NumTeamsLower == nullptr)
12182 NumTeamsLower = NumTeamsUpper;
12183
12184 if (IfExpr) {
12185 assert(IfExpr->getType()->isIntegerTy() &&
12186 "argument to if clause must be an integer value");
12187
12188 // upper = ifexpr ? upper : 1
12189 if (IfExpr->getType() != Int1)
12190 IfExpr = Builder.CreateICmpNE(LHS: IfExpr,
12191 RHS: ConstantInt::get(Ty: IfExpr->getType(), V: 0));
12192 NumTeamsUpper = Builder.CreateSelect(
12193 C: IfExpr, True: NumTeamsUpper, False: Builder.getInt32(C: 1), Name: "numTeamsUpper");
12194
12195 // lower = ifexpr ? lower : 1
12196 NumTeamsLower = Builder.CreateSelect(
12197 C: IfExpr, True: NumTeamsLower, False: Builder.getInt32(C: 1), Name: "numTeamsLower");
12198 }
12199
12200 if (ThreadLimit == nullptr)
12201 ThreadLimit = Builder.getInt32(C: 0);
12202
12203 // The __kmpc_push_num_teams_51 function expects int32 as the arguments. So,
12204 // truncate or sign extend the passed values to match the int32 parameters.
12205 Value *NumTeamsLowerInt32 =
12206 Builder.CreateSExtOrTrunc(V: NumTeamsLower, DestTy: Builder.getInt32Ty());
12207 Value *NumTeamsUpperInt32 =
12208 Builder.CreateSExtOrTrunc(V: NumTeamsUpper, DestTy: Builder.getInt32Ty());
12209 Value *ThreadLimitInt32 =
12210 Builder.CreateSExtOrTrunc(V: ThreadLimit, DestTy: Builder.getInt32Ty());
12211
12212 Value *ThreadNum = getOrCreateThreadID(Ident);
12213
12214 createRuntimeFunctionCall(
12215 Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_teams_51),
12216 Args: {Ident, ThreadNum, NumTeamsLowerInt32, NumTeamsUpperInt32,
12217 ThreadLimitInt32});
12218 }
12219 // Generate the body of teams.
12220 InsertPointTy AllocaIP(AllocaBB->begin());
12221 InsertPointTy CodeGenIP(BodyBB->begin());
12222 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12223 return Err;
12224
12225 auto OI = std::make_unique<OutlineInfo>();
12226 OI->EntryBB = AllocaBB;
12227 OI->ExitBB = ExitBB;
12228 OI->OuterAllocBB = &OuterAllocaBB;
12229
12230 // Insert fake values for global tid and bound tid.
12231 SmallVector<Instruction *, 8> ToBeDeleted;
12232 InsertPointTy OuterAllocaIP(OuterAllocaBB.begin());
12233 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12234 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "gid", AsPtr: true));
12235 OI->ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
12236 Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "tid", AsPtr: true));
12237
12238 auto HostPostOutlineCB = [this, Ident,
12239 ToBeDeleted](Function &OutlinedFn) mutable {
12240 // The stale call instruction will be replaced with a new call instruction
12241 // for runtime call with the outlined function.
12242
12243 assert(OutlinedFn.hasOneUse() &&
12244 "there must be a single user for the outlined function");
12245 CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
12246 ToBeDeleted.push_back(Elt: StaleCI);
12247
12248 assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
12249 "Outlined function must have two or three arguments only");
12250
12251 bool HasShared = OutlinedFn.arg_size() == 3;
12252
12253 OutlinedFn.getArg(i: 0)->setName("global.tid.ptr");
12254 OutlinedFn.getArg(i: 1)->setName("bound.tid.ptr");
12255 if (HasShared)
12256 OutlinedFn.getArg(i: 2)->setName("data");
12257
12258 // Call to the runtime function for teams in the current function.
12259 assert(StaleCI && "Error while outlining - no CallInst user found for the "
12260 "outlined function.");
12261 Builder.SetInsertPoint(StaleCI);
12262 SmallVector<Value *> Args = {
12263 Ident, Builder.getInt32(C: StaleCI->arg_size() - 2), &OutlinedFn};
12264 if (HasShared)
12265 Args.push_back(Elt: StaleCI->getArgOperand(i: 2));
12266 createRuntimeFunctionCall(
12267 Callee: getOrCreateRuntimeFunctionPtr(
12268 FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
12269 Args);
12270
12271 Builder.ClearInsertionPoint();
12272 for (Instruction *I : llvm::reverse(C&: ToBeDeleted))
12273 I->eraseFromParent();
12274 };
12275
12276 if (!Config.isTargetDevice())
12277 OI->PostOutlineCB = HostPostOutlineCB;
12278
12279 addOutlineInfo(OI: std::move(OI));
12280
12281 Builder.SetInsertPoint(ExitBB);
12282
12283 return Builder.saveIP();
12284}
12285
12286OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createDistribute(
12287 const LocationDescription &Loc, InsertPointTy OuterAllocIP,
12288 ArrayRef<BasicBlock *> OuterDeallocBlocks, BodyGenCallbackTy BodyGenCB) {
12289 if (!updateToLocation(Loc))
12290 return InsertPointTy();
12291
12292 BasicBlock *OuterAllocaBB = OuterAllocIP.getNodeParent();
12293
12294 if (OuterAllocaBB == Builder.GetInsertBlock()) {
12295 BasicBlock *BodyBB =
12296 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.entry");
12297 Builder.SetInsertPoint(BodyBB->begin());
12298 }
12299 BasicBlock *ExitBB =
12300 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.exit");
12301 BasicBlock *BodyBB =
12302 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.body");
12303 BasicBlock *AllocaBB =
12304 splitBB(Builder, /*CreateBranch=*/true, Name: "distribute.alloca");
12305
12306 // Generate the body of distribute clause
12307 InsertPointTy AllocaIP(AllocaBB->begin());
12308 InsertPointTy CodeGenIP(BodyBB->begin());
12309 if (Error Err = BodyGenCB(AllocaIP, CodeGenIP, ExitBB))
12310 return Err;
12311
12312 // When using target we use different runtime functions which require a
12313 // callback.
12314 if (Config.isTargetDevice()) {
12315 auto OI = std::make_unique<OutlineInfo>();
12316 OI->OuterAllocBB = OuterAllocIP.getNodeParent();
12317 OI->EntryBB = AllocaBB;
12318 OI->ExitBB = ExitBB;
12319 OI->OuterDeallocBBs.reserve(N: OuterDeallocBlocks.size());
12320 copy(Range&: OuterDeallocBlocks, Out: OI->OuterDeallocBBs.end());
12321
12322 addOutlineInfo(OI: std::move(OI));
12323 }
12324 Builder.SetInsertPoint(ExitBB);
12325
12326 return Builder.saveIP();
12327}
12328
12329GlobalVariable *
12330OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
12331 std::string VarName) {
12332 llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
12333 T: llvm::ArrayType::get(ElementType: llvm::PointerType::getUnqual(C&: M.getContext()),
12334 NumElements: Names.size()),
12335 V: Names);
12336 auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
12337 M, MapNamesArrayInit->getType(),
12338 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
12339 VarName);
12340 return MapNamesArrayGlobal;
12341}
12342
12343// Create all simple and struct types exposed by the runtime and remember
12344// the llvm::PointerTypes of them for easy access later.
12345void OpenMPIRBuilder::initializeTypes(Module &M) {
12346 LLVMContext &Ctx = M.getContext();
12347 StructType *T;
12348 unsigned DefaultTargetAS = Config.getDefaultTargetAS();
12349 unsigned ProgramAS = M.getDataLayout().getProgramAddressSpace();
12350#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
12351#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
12352 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
12353 VarName##PtrTy = PointerType::get(Ctx, DefaultTargetAS);
12354#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
12355 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
12356 VarName##Ptr = PointerType::get(Ctx, ProgramAS);
12357#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
12358 T = StructType::getTypeByName(Ctx, StructName); \
12359 if (!T) \
12360 T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
12361 VarName = T; \
12362 VarName##Ptr = PointerType::get(Ctx, DefaultTargetAS);
12363#include "llvm/Frontend/OpenMP/OMPKinds.def"
12364}
12365
12366void OpenMPIRBuilder::OutlineInfo::collectBlocks(
12367 SmallPtrSetImpl<BasicBlock *> &BlockSet,
12368 SmallVectorImpl<BasicBlock *> &BlockVector) {
12369 SmallVector<BasicBlock *, 32> Worklist;
12370 BlockSet.insert(Ptr: EntryBB);
12371 BlockSet.insert(Ptr: ExitBB);
12372
12373 Worklist.push_back(Elt: EntryBB);
12374 while (!Worklist.empty()) {
12375 BasicBlock *BB = Worklist.pop_back_val();
12376 BlockVector.push_back(Elt: BB);
12377 for (BasicBlock *SuccBB : successors(BB))
12378 if (BlockSet.insert(Ptr: SuccBB).second)
12379 Worklist.push_back(Elt: SuccBB);
12380 }
12381}
12382
12383std::unique_ptr<CodeExtractor>
12384OpenMPIRBuilder::OutlineInfo::createCodeExtractor(ArrayRef<BasicBlock *> Blocks,
12385 bool ArgsInZeroAddressSpace,
12386 Twine Suffix) {
12387 return std::make_unique<CodeExtractor>(
12388 args&: Blocks, /* DominatorTree */ args: nullptr,
12389 /* AggregateArgs */ args: true,
12390 /* BlockFrequencyInfo */ args: nullptr,
12391 /* BranchProbabilityInfo */ args: nullptr,
12392 /* AssumptionCache */ args: nullptr,
12393 /* AllowVarArgs */ args: true,
12394 /* AllowAlloca */ args: true,
12395 /* AllocationBlock*/ args&: OuterAllocBB,
12396 /* DeallocationBlocks */ args: ArrayRef<BasicBlock *>(),
12397 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12398}
12399
12400std::unique_ptr<CodeExtractor> DeviceSharedMemOutlineInfo::createCodeExtractor(
12401 ArrayRef<BasicBlock *> Blocks, bool ArgsInZeroAddressSpace, Twine Suffix) {
12402 return std::make_unique<DeviceSharedMemCodeExtractor>(
12403 args&: OMPBuilder, args&: Blocks, /* DominatorTree */ args: nullptr,
12404 /* AggregateArgs */ args: true,
12405 /* BlockFrequencyInfo */ args: nullptr,
12406 /* BranchProbabilityInfo */ args: nullptr,
12407 /* AssumptionCache */ args: nullptr,
12408 /* AllowVarArgs */ args: true,
12409 /* AllowAlloca */ args: true,
12410 /* AllocationBlock*/ args&: OuterAllocBB,
12411 /* DeallocationBlocks */ args: OuterDeallocBBs.empty()
12412 ? SmallVector<BasicBlock *>{ExitBB}
12413 : OuterDeallocBBs,
12414 /* Suffix */ args: Suffix.str(), args&: ArgsInZeroAddressSpace);
12415}
12416
12417void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
12418 uint64_t Size, int32_t Flags,
12419 GlobalValue::LinkageTypes,
12420 StringRef Name) {
12421 if (!Config.isGPU()) {
12422 llvm::offloading::emitOffloadingEntry(
12423 M, Kind: object::OffloadKind::OFK_OpenMP, Addr: ID,
12424 Name: Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0);
12425 return;
12426 }
12427 // TODO: Add support for global variables on the device after declare target
12428 // support.
12429 Function *Fn = dyn_cast<Function>(Val: Addr);
12430 if (!Fn)
12431 return;
12432
12433 // Add a function attribute for the kernel.
12434 Fn->addFnAttr(Kind: "kernel");
12435 if (T.isAMDGCN())
12436 Fn->addFnAttr(Kind: "uniform-work-group-size");
12437 Fn->addFnAttr(Kind: Attribute::MustProgress);
12438}
12439
12440// We only generate metadata for function that contain target regions.
12441void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
12442 EmitMetadataErrorReportFunctionTy &ErrorFn) {
12443
12444 // If there are no entries, we don't need to do anything.
12445 if (OffloadInfoManager.empty())
12446 return;
12447
12448 LLVMContext &C = M.getContext();
12449 SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,
12450 TargetRegionEntryInfo>,
12451 16>
12452 OrderedEntries(OffloadInfoManager.size());
12453
12454 // Auxiliary methods to create metadata values and strings.
12455 auto &&GetMDInt = [this](unsigned V) {
12456 return ConstantAsMetadata::get(C: ConstantInt::get(Ty: Builder.getInt32Ty(), V));
12457 };
12458
12459 auto &&GetMDString = [&C](StringRef V) { return MDString::get(Context&: C, Str: V); };
12460
12461 // Create the offloading info metadata node.
12462 NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "omp_offload.info");
12463 auto &&TargetRegionMetadataEmitter =
12464 [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
12465 const TargetRegionEntryInfo &EntryInfo,
12466 const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {
12467 // Generate metadata for target regions. Each entry of this metadata
12468 // contains:
12469 // - Entry 0 -> Kind of this type of metadata (0).
12470 // - Entry 1 -> Device ID of the file where the entry was identified.
12471 // - Entry 2 -> File ID of the file where the entry was identified.
12472 // - Entry 3 -> Mangled name of the function where the entry was
12473 // identified.
12474 // - Entry 4 -> Line in the file where the entry was identified.
12475 // - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.
12476 // - Entry 6 -> Order the entry was created.
12477 // The first element of the metadata node is the kind.
12478 Metadata *Ops[] = {
12479 GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
12480 GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
12481 GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
12482 GetMDInt(E.getOrder())};
12483
12484 // Save this entry in the right position of the ordered entries array.
12485 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y: EntryInfo);
12486
12487 // Add metadata to the named metadata node.
12488 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12489 };
12490
12491 OffloadInfoManager.actOnTargetRegionEntriesInfo(Action: TargetRegionMetadataEmitter);
12492
12493 // Create function that emits metadata for each device global variable entry;
12494 auto &&DeviceGlobalVarMetadataEmitter =
12495 [&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
12496 StringRef MangledName,
12497 const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {
12498 // Generate metadata for global variables. Each entry of this metadata
12499 // contains:
12500 // - Entry 0 -> Kind of this type of metadata (1).
12501 // - Entry 1 -> Mangled name of the variable.
12502 // - Entry 2 -> Declare target kind.
12503 // - Entry 3 -> Order the entry was created.
12504 // The first element of the metadata node is the kind.
12505 Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
12506 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
12507
12508 // Save this entry in the right position of the ordered entries array.
12509 TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);
12510 OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y&: varInfo);
12511
12512 // Add metadata to the named metadata node.
12513 MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
12514 };
12515
12516 OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(
12517 Action: DeviceGlobalVarMetadataEmitter);
12518
12519 for (const auto &E : OrderedEntries) {
12520 assert(E.first && "All ordered entries must exist!");
12521 if (const auto *CE =
12522 dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
12523 Val: E.first)) {
12524 if (!CE->getID() || !CE->getAddress()) {
12525 // Do not blame the entry if the parent funtion is not emitted.
12526 TargetRegionEntryInfo EntryInfo = E.second;
12527 StringRef FnName = EntryInfo.ParentName;
12528 if (!M.getNamedValue(Name: FnName))
12529 continue;
12530 ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);
12531 continue;
12532 }
12533 createOffloadEntry(ID: CE->getID(), Addr: CE->getAddress(),
12534 /*Size=*/0, Flags: CE->getFlags(),
12535 GlobalValue::WeakAnyLinkage);
12536 } else if (const auto *CE = dyn_cast<
12537 OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(
12538 Val: E.first)) {
12539 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =
12540 static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12541 CE->getFlags());
12542 switch (Flags) {
12543 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:
12544 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:
12545 if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())
12546 continue;
12547 if (!CE->getAddress()) {
12548 ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);
12549 continue;
12550 }
12551 // The vaiable has no definition - no need to add the entry.
12552 if (CE->getVarSize() == 0)
12553 continue;
12554 break;
12555 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:
12556 assert(((Config.isTargetDevice() && !CE->getAddress()) ||
12557 (!Config.isTargetDevice() && CE->getAddress())) &&
12558 "Declaret target link address is set.");
12559 if (Config.isTargetDevice())
12560 continue;
12561 if (!CE->getAddress()) {
12562 ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());
12563 continue;
12564 }
12565 break;
12566 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect:
12567 case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable:
12568 if (!CE->getAddress()) {
12569 ErrorFn(EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR, E.second);
12570 continue;
12571 }
12572 break;
12573 default:
12574 break;
12575 }
12576
12577 // Hidden or internal symbols on the device are not externally visible.
12578 // We should not attempt to register them by creating an offloading
12579 // entry. Indirect variables are handled separately on the device.
12580 if (auto *GV = dyn_cast<GlobalValue>(Val: CE->getAddress()))
12581 if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
12582 (Flags !=
12583 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect &&
12584 Flags != OffloadEntriesInfoManager::
12585 OMPTargetGlobalVarEntryIndirectVTable))
12586 continue;
12587
12588 // Indirect globals need to use a special name that doesn't match the name
12589 // of the associated host global.
12590 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
12591 Flags ==
12592 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
12593 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12594 Flags, CE->getLinkage(), Name: CE->getVarName());
12595 else
12596 createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
12597 Flags, CE->getLinkage());
12598
12599 } else {
12600 llvm_unreachable("Unsupported entry kind.");
12601 }
12602 }
12603
12604 // Emit requires directive globals to a special entry so the runtime can
12605 // register them when the device image is loaded.
12606 // TODO: This reduces the offloading entries to a 32-bit integer. Offloading
12607 // entries should be redesigned to better suit this use-case.
12608 if (Config.hasRequiresFlags() && !Config.isTargetDevice())
12609 offloading::emitOffloadingEntry(
12610 M, Kind: object::OffloadKind::OFK_OpenMP,
12611 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: M.getContext())),
12612 Name: ".requires", /*Size=*/0,
12613 Flags: OffloadEntriesInfoManager::OMPTargetGlobalRegisterRequires,
12614 Data: Config.getRequiresFlags());
12615}
12616
12617void TargetRegionEntryInfo::getTargetRegionEntryFnName(
12618 SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,
12619 unsigned FileID, unsigned Line, unsigned Count) {
12620 raw_svector_ostream OS(Name);
12621 OS << KernelNamePrefix << llvm::format(Fmt: "%x", Vals: DeviceID)
12622 << llvm::format(Fmt: "_%x_", Vals: FileID) << ParentName << "_l" << Line;
12623 if (Count)
12624 OS << "_" << Count;
12625}
12626
12627void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
12628 SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {
12629 unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
12630 TargetRegionEntryInfo::getTargetRegionEntryFnName(
12631 Name, ParentName: EntryInfo.ParentName, DeviceID: EntryInfo.DeviceID, FileID: EntryInfo.FileID,
12632 Line: EntryInfo.Line, Count: NewCount);
12633}
12634
12635TargetRegionEntryInfo
12636OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
12637 vfs::FileSystem &VFS,
12638 StringRef ParentName) {
12639 sys::fs::UniqueID ID(0xdeadf17e, 0);
12640 auto FileIDInfo = CallBack();
12641 uint64_t FileID = 0;
12642 if (ErrorOr<vfs::Status> Status = VFS.status(Path: std::get<0>(t&: FileIDInfo))) {
12643 ID = Status->getUniqueID();
12644 FileID = Status->getUniqueID().getFile();
12645 } else {
12646 // If the inode ID could not be determined, create a hash value
12647 // the current file name and use that as an ID.
12648 FileID = hash_value(arg: std::get<0>(t&: FileIDInfo));
12649 }
12650
12651 return TargetRegionEntryInfo(ParentName, ID.getDevice(), FileID,
12652 std::get<1>(t&: FileIDInfo));
12653}
12654
12655unsigned OpenMPIRBuilder::getFlagMemberOffset() {
12656 unsigned Offset = 0;
12657 for (uint64_t Remain =
12658 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12659 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
12660 !(Remain & 1); Remain = Remain >> 1)
12661 Offset++;
12662 return Offset;
12663}
12664
12665omp::OpenMPOffloadMappingFlags
12666OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {
12667 // Rotate by getFlagMemberOffset() bits.
12668 return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
12669 << getFlagMemberOffset());
12670}
12671
12672void OpenMPIRBuilder::setCorrectMemberOfFlag(
12673 omp::OpenMPOffloadMappingFlags &Flags,
12674 omp::OpenMPOffloadMappingFlags MemberOfFlag) {
12675 // If the entry is PTR_AND_OBJ but has not been marked with the special
12676 // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
12677 // marked as MEMBER_OF.
12678 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12679 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&
12680 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12681 (Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
12682 omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))
12683 return;
12684
12685 // Entries with ATTACH are not members-of anything. They are handled
12686 // separately by the runtime after other maps have been handled.
12687 if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
12688 Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH))
12689 return;
12690
12691 // Reset the placeholder value to prepare the flag for the assignment of the
12692 // proper MEMBER_OF value.
12693 Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
12694 Flags |= MemberOfFlag;
12695}
12696
12697Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(
12698 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12699 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12700 bool IsDeclaration, bool IsExternallyVisible,
12701 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12702 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12703 std::vector<Triple> TargetTriple, Type *LlvmPtrTy,
12704 std::function<Constant *()> GlobalInitializer,
12705 std::function<GlobalValue::LinkageTypes()> VariableLinkage) {
12706 // TODO: convert this to utilise the IRBuilder Config rather than
12707 // a passed down argument.
12708 if (OpenMPSIMD)
12709 return nullptr;
12710
12711 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||
12712 ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12713 CaptureClause ==
12714 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12715 Config.hasRequiresUnifiedSharedMemory())) {
12716 SmallString<64> PtrName;
12717 {
12718 raw_svector_ostream OS(PtrName);
12719 OS << MangledName;
12720 if (!IsExternallyVisible)
12721 OS << format(Fmt: "_%x", Vals: EntryInfo.FileID);
12722 OS << "_decl_tgt_ref_ptr";
12723 }
12724
12725 Value *Ptr = M.getNamedValue(Name: PtrName);
12726
12727 if (!Ptr) {
12728 GlobalValue *GlobalValue = M.getNamedValue(Name: MangledName);
12729 Ptr = getOrCreateInternalVariable(Ty: LlvmPtrTy, Name: PtrName);
12730
12731 auto *GV = cast<GlobalVariable>(Val: Ptr);
12732 GV->setLinkage(GlobalValue::WeakAnyLinkage);
12733
12734 if (!Config.isTargetDevice()) {
12735 if (GlobalInitializer)
12736 GV->setInitializer(GlobalInitializer());
12737 else
12738 GV->setInitializer(GlobalValue);
12739 }
12740
12741 registerTargetGlobalVariable(
12742 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12743 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12744 GlobalInitializer, VariableLinkage, LlvmPtrTy, Addr: cast<Constant>(Val: Ptr));
12745 }
12746
12747 return cast<Constant>(Val: Ptr);
12748 }
12749
12750 return nullptr;
12751}
12752
12753void OpenMPIRBuilder::registerTargetGlobalVariable(
12754 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
12755 OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
12756 bool IsDeclaration, bool IsExternallyVisible,
12757 TargetRegionEntryInfo EntryInfo, StringRef MangledName,
12758 std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
12759 std::vector<Triple> TargetTriple,
12760 std::function<Constant *()> GlobalInitializer,
12761 std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,
12762 Constant *Addr) {
12763 if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||
12764 (TargetTriple.empty() && !Config.isTargetDevice()))
12765 return;
12766
12767 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;
12768 StringRef VarName;
12769 int64_t VarSize;
12770 GlobalValue::LinkageTypes Linkage;
12771
12772 if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
12773 CaptureClause ==
12774 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
12775 !Config.hasRequiresUnifiedSharedMemory()) {
12776 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12777 VarName = MangledName;
12778 GlobalValue *LlvmVal = M.getNamedValue(Name: VarName);
12779
12780 if (!IsDeclaration)
12781 VarSize = divideCeil(
12782 Numerator: M.getDataLayout().getTypeSizeInBits(Ty: LlvmVal->getValueType()), Denominator: 8);
12783 else
12784 VarSize = 0;
12785 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();
12786
12787 // This is a workaround carried over from Clang which prevents undesired
12788 // optimisation of internal variables.
12789 if (Config.isTargetDevice() &&
12790 (!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {
12791 // Do not create a "ref-variable" if the original is not also available
12792 // on the host.
12793 if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))
12794 return;
12795
12796 std::string RefName = createPlatformSpecificName(Parts: {VarName, "ref"});
12797
12798 if (!M.getNamedValue(Name: RefName)) {
12799 Constant *AddrRef =
12800 getOrCreateInternalVariable(Ty: Addr->getType(), Name: RefName);
12801 auto *GvAddrRef = cast<GlobalVariable>(Val: AddrRef);
12802 GvAddrRef->setConstant(true);
12803 GvAddrRef->setLinkage(GlobalValue::InternalLinkage);
12804 GvAddrRef->setInitializer(Addr);
12805 GeneratedRefs.push_back(x: GvAddrRef);
12806 }
12807 }
12808 } else {
12809 if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)
12810 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
12811 else
12812 Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
12813
12814 if (Config.isTargetDevice()) {
12815 VarName = (Addr) ? Addr->getName() : "";
12816 Addr = nullptr;
12817 } else {
12818 Addr = getAddrOfDeclareTargetVar(
12819 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
12820 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
12821 LlvmPtrTy, GlobalInitializer, VariableLinkage);
12822 VarName = (Addr) ? Addr->getName() : "";
12823 }
12824 VarSize = M.getDataLayout().getPointerSize();
12825 Linkage = GlobalValue::WeakAnyLinkage;
12826 }
12827
12828 OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,
12829 Flags, Linkage);
12830}
12831
12832/// Loads all the offload entries information from the host IR
12833/// metadata.
12834void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {
12835 // If we are in target mode, load the metadata from the host IR. This code has
12836 // to match the metadata creation in createOffloadEntriesAndInfoMetadata().
12837
12838 NamedMDNode *MD = M.getNamedMetadata(Name: ompOffloadInfoName);
12839 if (!MD)
12840 return;
12841
12842 for (MDNode *MN : MD->operands()) {
12843 auto &&GetMDInt = [MN](unsigned Idx) {
12844 auto *V = cast<ConstantAsMetadata>(Val: MN->getOperand(I: Idx));
12845 return cast<ConstantInt>(Val: V->getValue())->getZExtValue();
12846 };
12847
12848 auto &&GetMDString = [MN](unsigned Idx) {
12849 auto *V = cast<MDString>(Val: MN->getOperand(I: Idx));
12850 return V->getString();
12851 };
12852
12853 switch (GetMDInt(0)) {
12854 default:
12855 llvm_unreachable("Unexpected metadata!");
12856 break;
12857 case OffloadEntriesInfoManager::OffloadEntryInfo::
12858 OffloadingEntryInfoTargetRegion: {
12859 TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),
12860 /*DeviceID=*/GetMDInt(1),
12861 /*FileID=*/GetMDInt(2),
12862 /*Line=*/GetMDInt(4),
12863 /*Count=*/GetMDInt(5));
12864 OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,
12865 /*Order=*/GetMDInt(6));
12866 break;
12867 }
12868 case OffloadEntriesInfoManager::OffloadEntryInfo::
12869 OffloadingEntryInfoDeviceGlobalVar:
12870 OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(
12871 /*MangledName=*/Name: GetMDString(1),
12872 Flags: static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
12873 /*Flags=*/GetMDInt(2)),
12874 /*Order=*/GetMDInt(3));
12875 break;
12876 }
12877 }
12878}
12879
12880void OpenMPIRBuilder::loadOffloadInfoMetadata(vfs::FileSystem &VFS,
12881 StringRef HostFilePath) {
12882 if (HostFilePath.empty())
12883 return;
12884
12885 auto Buf = VFS.getBufferForFile(Name: HostFilePath);
12886 if (std::error_code Err = Buf.getError()) {
12887 report_fatal_error(reason: ("error opening host file from host file path inside of "
12888 "OpenMPIRBuilder: " +
12889 Err.message())
12890 .c_str());
12891 }
12892
12893 LLVMContext Ctx;
12894 auto M = expectedToErrorOrAndEmitErrors(
12895 Ctx, Val: parseBitcodeFile(Buffer: Buf.get()->getMemBufferRef(), Context&: Ctx));
12896 if (std::error_code Err = M.getError()) {
12897 report_fatal_error(
12898 reason: ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())
12899 .c_str());
12900 }
12901
12902 loadOffloadInfoMetadata(M&: *M.get());
12903}
12904
12905OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createIteratorLoop(
12906 LocationDescription Loc, llvm::Value *TripCount, IteratorBodyGenTy BodyGen,
12907 llvm::StringRef Name) {
12908 Builder.restoreIP(IP: Loc.IP);
12909
12910 BasicBlock *CurBB = Builder.GetInsertBlock();
12911 assert(CurBB &&
12912 "expected a valid insertion block for creating an iterator loop");
12913 Function *F = CurBB->getParent();
12914
12915 InsertPointTy SplitIP = Builder.saveIP();
12916 if (SplitIP == CurBB->end())
12917 if (Instruction *Terminator = CurBB->getTerminatorOrNull())
12918 SplitIP = Terminator->getIterator();
12919
12920 BasicBlock *ContBB =
12921 splitBB(IP: SplitIP, /*CreateBranch=*/false,
12922 DL: Builder.getCurrentDebugLocation(), Name: "omp.it.cont");
12923
12924 CanonicalLoopInfo *CLI =
12925 createLoopSkeleton(DL: Builder.getCurrentDebugLocation(), TripCount, F,
12926 /*PreInsertBefore=*/ContBB,
12927 /*PostInsertBefore=*/ContBB, Name);
12928
12929 // Enter loop from original block.
12930 redirectTo(Source: CurBB, Target: CLI->getPreheader(), DL: Builder.getCurrentDebugLocation());
12931
12932 // Remove the unconditional branch inserted by createLoopSkeleton in the body
12933 if (Instruction *T = CLI->getBody()->getTerminatorOrNull())
12934 T->eraseFromParent();
12935
12936 InsertPointTy BodyIP = CLI->getBodyIP();
12937 // Blocks that exist before BodyGen, other than the body and latch, are
12938 // outside the loop body.
12939 SmallPtrSet<BasicBlock *, 32> ExistingBlocks;
12940 for (BasicBlock &Block : *F)
12941 ExistingBlocks.insert(Ptr: &Block);
12942 if (llvm::Error Err = BodyGen(BodyIP, CLI->getIndVar()))
12943 return Err;
12944
12945 // The body may span several blocks. Branch its single unterminated block to
12946 // the latch; otherwise some block must already branch there.
12947 BasicBlock *Latch = CLI->getLatch();
12948 BasicBlock *OpenBB = nullptr;
12949 bool ReachesLatch = false;
12950 SmallVector<BasicBlock *> Worklist{CLI->getBody()};
12951 SmallPtrSet<BasicBlock *, 8> Visited{CLI->getBody()};
12952 while (!Worklist.empty()) {
12953 BasicBlock *BB = Worklist.pop_back_val();
12954 if (!BB->hasTerminator()) {
12955 if (OpenBB)
12956 return make_error<StringError>(
12957 Args: "iterator bodygen must leave at most one unterminated block",
12958 Args: inconvertibleErrorCode());
12959 OpenBB = BB;
12960 continue;
12961 }
12962 for (BasicBlock *Succ : successors(BB)) {
12963 if (Succ == Latch) {
12964 ReachesLatch = true;
12965 continue;
12966 }
12967 if (Succ != CLI->getBody() && ExistingBlocks.contains(Ptr: Succ))
12968 return make_error<StringError>(
12969 Args: "iterator bodygen must not branch out of the loop body",
12970 Args: inconvertibleErrorCode());
12971 if (Visited.insert(Ptr: Succ).second)
12972 Worklist.push_back(Elt: Succ);
12973 }
12974 }
12975
12976 if (OpenBB) {
12977 Builder.SetInsertPoint(OpenBB);
12978 Builder.CreateBr(Dest: Latch);
12979 } else if (!ReachesLatch) {
12980 return make_error<StringError>(Args: "iterator bodygen must reach the loop latch",
12981 Args: inconvertibleErrorCode());
12982 }
12983
12984 // Link After -> ContBB
12985 Builder.SetInsertPoint(CLI->getAfter()->begin());
12986 if (!CLI->getAfter()->hasTerminator())
12987 Builder.CreateBr(Dest: ContBB);
12988
12989 return ContBB->begin();
12990}
12991
12992/// Mangle the parameter part of the vector function name according to
12993/// their OpenMP classification. The mangling function is defined in
12994/// section 4.5 of the AAVFABI(2021Q1).
12995static std::string mangleVectorParameters(
12996 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12997 SmallString<256> Buffer;
12998 llvm::raw_svector_ostream Out(Buffer);
12999 for (const auto &ParamAttr : ParamAttrs) {
13000 switch (ParamAttr.Kind) {
13001 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear:
13002 Out << 'l';
13003 break;
13004 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef:
13005 Out << 'R';
13006 break;
13007 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal:
13008 Out << 'U';
13009 break;
13010 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal:
13011 Out << 'L';
13012 break;
13013 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform:
13014 Out << 'u';
13015 break;
13016 case llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector:
13017 Out << 'v';
13018 break;
13019 }
13020 if (ParamAttr.HasVarStride)
13021 Out << "s" << ParamAttr.StrideOrArg;
13022 else if (ParamAttr.Kind ==
13023 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
13024 ParamAttr.Kind ==
13025 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef ||
13026 ParamAttr.Kind ==
13027 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
13028 ParamAttr.Kind ==
13029 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) {
13030 // Don't print the step value if it is not present or if it is
13031 // equal to 1.
13032 if (ParamAttr.StrideOrArg < 0)
13033 Out << 'n' << -ParamAttr.StrideOrArg;
13034 else if (ParamAttr.StrideOrArg != 1)
13035 Out << ParamAttr.StrideOrArg;
13036 }
13037
13038 if (!!ParamAttr.Alignment)
13039 Out << 'a' << ParamAttr.Alignment;
13040 }
13041
13042 return std::string(Out.str());
13043}
13044
13045void OpenMPIRBuilder::emitX86DeclareSimdFunction(
13046 llvm::Function *Fn, unsigned NumElts, const llvm::APSInt &VLENVal,
13047 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch) {
13048 struct ISADataTy {
13049 char ISA;
13050 unsigned VecRegSize;
13051 };
13052 ISADataTy ISAData[] = {
13053 {.ISA: 'b', .VecRegSize: 128}, // SSE
13054 {.ISA: 'c', .VecRegSize: 256}, // AVX
13055 {.ISA: 'd', .VecRegSize: 256}, // AVX2
13056 {.ISA: 'e', .VecRegSize: 512}, // AVX512
13057 };
13058 llvm::SmallVector<char, 2> Masked;
13059 switch (Branch) {
13060 case DeclareSimdBranch::Undefined:
13061 Masked.push_back(Elt: 'N');
13062 Masked.push_back(Elt: 'M');
13063 break;
13064 case DeclareSimdBranch::Notinbranch:
13065 Masked.push_back(Elt: 'N');
13066 break;
13067 case DeclareSimdBranch::Inbranch:
13068 Masked.push_back(Elt: 'M');
13069 break;
13070 }
13071 for (char Mask : Masked) {
13072 for (const ISADataTy &Data : ISAData) {
13073 llvm::SmallString<256> Buffer;
13074 llvm::raw_svector_ostream Out(Buffer);
13075 Out << "_ZGV" << Data.ISA << Mask;
13076 if (!VLENVal) {
13077 assert(NumElts && "Non-zero simdlen/cdtsize expected");
13078 Out << llvm::APSInt::getUnsigned(X: Data.VecRegSize / NumElts);
13079 } else {
13080 Out << VLENVal;
13081 }
13082 Out << mangleVectorParameters(ParamAttrs);
13083 Out << '_' << Fn->getName();
13084 Fn->addFnAttr(Kind: Out.str());
13085 }
13086 }
13087}
13088
13089// Function used to add the attribute. The parameter `VLEN` is templated to
13090// allow the use of `x` when targeting scalable functions for SVE.
13091template <typename T>
13092static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix,
13093 char ISA, StringRef ParSeq,
13094 StringRef MangledName, bool OutputBecomesInput,
13095 llvm::Function *Fn) {
13096 SmallString<256> Buffer;
13097 llvm::raw_svector_ostream Out(Buffer);
13098 Out << Prefix << ISA << LMask << VLEN;
13099 if (OutputBecomesInput)
13100 Out << 'v';
13101 Out << ParSeq << '_' << MangledName;
13102 Fn->addFnAttr(Kind: Out.str());
13103}
13104
13105// Helper function to generate the Advanced SIMD names depending on the value
13106// of the NDS when simdlen is not present.
13107static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask,
13108 StringRef Prefix, char ISA,
13109 StringRef ParSeq, StringRef MangledName,
13110 bool OutputBecomesInput,
13111 llvm::Function *Fn) {
13112 switch (NDS) {
13113 case 8:
13114 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13115 OutputBecomesInput, Fn);
13116 addAArch64VectorName(VLEN: 16, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13117 OutputBecomesInput, Fn);
13118 break;
13119 case 16:
13120 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13121 OutputBecomesInput, Fn);
13122 addAArch64VectorName(VLEN: 8, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13123 OutputBecomesInput, Fn);
13124 break;
13125 case 32:
13126 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13127 OutputBecomesInput, Fn);
13128 addAArch64VectorName(VLEN: 4, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13129 OutputBecomesInput, Fn);
13130 break;
13131 case 64:
13132 case 128:
13133 addAArch64VectorName(VLEN: 2, LMask: Mask, Prefix, ISA, ParSeq, MangledName,
13134 OutputBecomesInput, Fn);
13135 break;
13136 default:
13137 llvm_unreachable("Scalar type is too wide.");
13138 }
13139}
13140
13141/// Emit vector function attributes for AArch64, as defined in the AAVFABI.
13142void OpenMPIRBuilder::emitAArch64DeclareSimdFunction(
13143 llvm::Function *Fn, unsigned UserVLEN,
13144 llvm::ArrayRef<DeclareSimdAttrTy> ParamAttrs, DeclareSimdBranch Branch,
13145 char ISA, unsigned NarrowestDataSize, bool OutputBecomesInput) {
13146 assert((ISA == 'n' || ISA == 's') && "Expected ISA either 's' or 'n'.");
13147
13148 // Sort out parameter sequence.
13149 const std::string ParSeq = mangleVectorParameters(ParamAttrs);
13150 StringRef Prefix = "_ZGV";
13151 StringRef MangledName = Fn->getName();
13152
13153 // Generate simdlen from user input (if any).
13154 if (UserVLEN) {
13155 if (ISA == 's') {
13156 // SVE generates only a masked function.
13157 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13158 OutputBecomesInput, Fn);
13159 return;
13160 }
13161
13162 switch (Branch) {
13163 case DeclareSimdBranch::Undefined:
13164 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13165 OutputBecomesInput, Fn);
13166 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13167 OutputBecomesInput, Fn);
13168 break;
13169 case DeclareSimdBranch::Inbranch:
13170 addAArch64VectorName(VLEN: UserVLEN, LMask: "M", Prefix, ISA, ParSeq, MangledName,
13171 OutputBecomesInput, Fn);
13172 break;
13173 case DeclareSimdBranch::Notinbranch:
13174 addAArch64VectorName(VLEN: UserVLEN, LMask: "N", Prefix, ISA, ParSeq, MangledName,
13175 OutputBecomesInput, Fn);
13176 break;
13177 }
13178 return;
13179 }
13180
13181 if (ISA == 's') {
13182 // SVE, section 3.4.1, item 1.
13183 addAArch64VectorName(VLEN: "x", LMask: "M", Prefix, ISA, ParSeq, MangledName,
13184 OutputBecomesInput, Fn);
13185 return;
13186 }
13187
13188 switch (Branch) {
13189 case DeclareSimdBranch::Undefined:
13190 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13191 MangledName, OutputBecomesInput, Fn);
13192 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13193 MangledName, OutputBecomesInput, Fn);
13194 break;
13195 case DeclareSimdBranch::Inbranch:
13196 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "M", Prefix, ISA, ParSeq,
13197 MangledName, OutputBecomesInput, Fn);
13198 break;
13199 case DeclareSimdBranch::Notinbranch:
13200 addAArch64AdvSIMDNDSNames(NDS: NarrowestDataSize, Mask: "N", Prefix, ISA, ParSeq,
13201 MangledName, OutputBecomesInput, Fn);
13202 break;
13203 }
13204}
13205
13206//===----------------------------------------------------------------------===//
13207// OffloadEntriesInfoManager
13208//===----------------------------------------------------------------------===//
13209
13210bool OffloadEntriesInfoManager::empty() const {
13211 return OffloadEntriesTargetRegion.empty() &&
13212 OffloadEntriesDeviceGlobalVar.empty();
13213}
13214
13215unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
13216 const TargetRegionEntryInfo &EntryInfo) const {
13217 auto It = OffloadEntriesTargetRegionCount.find(
13218 x: getTargetRegionEntryCountKey(EntryInfo));
13219 if (It == OffloadEntriesTargetRegionCount.end())
13220 return 0;
13221 return It->second;
13222}
13223
13224void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
13225 const TargetRegionEntryInfo &EntryInfo) {
13226 OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
13227 EntryInfo.Count + 1;
13228}
13229
13230/// Initialize target region entry.
13231void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(
13232 const TargetRegionEntryInfo &EntryInfo, unsigned Order) {
13233 OffloadEntriesTargetRegion[EntryInfo] =
13234 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
13235 OMPTargetRegionEntryTargetRegion);
13236 ++OffloadingEntriesNum;
13237}
13238
13239void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(
13240 TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,
13241 OMPTargetRegionEntryKind Flags) {
13242 assert(EntryInfo.Count == 0 && "expected default EntryInfo");
13243
13244 // Update the EntryInfo with the next available count for this location.
13245 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13246
13247 // If we are emitting code for a target, the entry is already initialized,
13248 // only has to be registered.
13249 if (OMPBuilder->Config.isTargetDevice()) {
13250 // This could happen if the device compilation is invoked standalone.
13251 if (!hasTargetRegionEntryInfo(EntryInfo)) {
13252 return;
13253 }
13254 auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
13255 Entry.setAddress(Addr);
13256 Entry.setID(ID);
13257 Entry.setFlags(Flags);
13258 } else {
13259 if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&
13260 hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))
13261 return;
13262 assert(!hasTargetRegionEntryInfo(EntryInfo) &&
13263 "Target region entry already registered!");
13264 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
13265 OffloadEntriesTargetRegion[EntryInfo] = Entry;
13266 ++OffloadingEntriesNum;
13267 }
13268 incrementTargetRegionEntryInfoCount(EntryInfo);
13269}
13270
13271bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(
13272 TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {
13273
13274 // Update the EntryInfo with the next available count for this location.
13275 EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
13276
13277 auto It = OffloadEntriesTargetRegion.find(x: EntryInfo);
13278 if (It == OffloadEntriesTargetRegion.end()) {
13279 return false;
13280 }
13281 // Fail if this entry is already registered.
13282 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
13283 return false;
13284 return true;
13285}
13286
13287void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(
13288 const OffloadTargetRegionEntryInfoActTy &Action) {
13289 // Scan all target region entries and perform the provided action.
13290 for (const auto &It : OffloadEntriesTargetRegion) {
13291 Action(It.first, It.second);
13292 }
13293}
13294
13295void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(
13296 StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {
13297 OffloadEntriesDeviceGlobalVar.try_emplace(Key: Name, Args&: Order, Args&: Flags);
13298 ++OffloadingEntriesNum;
13299}
13300
13301void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(
13302 StringRef VarName, Constant *Addr, int64_t VarSize,
13303 OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {
13304 if (OMPBuilder->Config.isTargetDevice()) {
13305 // This could happen if the device compilation is invoked standalone.
13306 if (!hasDeviceGlobalVarEntryInfo(VarName))
13307 return;
13308 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13309 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
13310 if (Entry.getVarSize() == 0) {
13311 Entry.setVarSize(VarSize);
13312 Entry.setLinkage(Linkage);
13313 }
13314 return;
13315 }
13316 Entry.setVarSize(VarSize);
13317 Entry.setLinkage(Linkage);
13318 Entry.setAddress(Addr);
13319 } else {
13320 if (hasDeviceGlobalVarEntryInfo(VarName)) {
13321 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
13322 assert(Entry.isValid() && Entry.getFlags() == Flags &&
13323 "Entry not initialized!");
13324 if (Entry.getVarSize() == 0) {
13325 Entry.setVarSize(VarSize);
13326 Entry.setLinkage(Linkage);
13327 }
13328 return;
13329 }
13330 if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect ||
13331 Flags ==
13332 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable)
13333 OffloadEntriesDeviceGlobalVar.try_emplace(Key: VarName, Args&: OffloadingEntriesNum,
13334 Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage,
13335 Args: VarName.str());
13336 else
13337 OffloadEntriesDeviceGlobalVar.try_emplace(
13338 Key: VarName, Args&: OffloadingEntriesNum, Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage, Args: "");
13339 ++OffloadingEntriesNum;
13340 }
13341}
13342
13343void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(
13344 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
13345 // Scan all target region entries and perform the provided action.
13346 for (const auto &E : OffloadEntriesDeviceGlobalVar)
13347 Action(E.getKey(), E.getValue());
13348}
13349
13350//===----------------------------------------------------------------------===//
13351// CanonicalLoopInfo
13352//===----------------------------------------------------------------------===//
13353
13354void CanonicalLoopInfo::collectControlBlocks(
13355 SmallVectorImpl<BasicBlock *> &BBs) {
13356 // We only count those BBs as control block for which we do not need to
13357 // reverse the CFG, i.e. not the loop body which can contain arbitrary control
13358 // flow. For consistency, this also means we do not add the Body block, which
13359 // is just the entry to the body code.
13360 BBs.reserve(N: BBs.size() + 6);
13361 BBs.append(IL: {getPreheader(), Header, Cond, Latch, Exit, getAfter()});
13362}
13363
13364BasicBlock *CanonicalLoopInfo::getPreheader() const {
13365 assert(isValid() && "Requires a valid canonical loop");
13366 for (BasicBlock *Pred : predecessors(BB: Header)) {
13367 if (Pred != Latch)
13368 return Pred;
13369 }
13370 llvm_unreachable("Missing preheader");
13371}
13372
13373void CanonicalLoopInfo::setTripCount(Value *TripCount) {
13374 assert(isValid() && "Requires a valid canonical loop");
13375
13376 Instruction *CmpI = &getCond()->front();
13377 assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
13378 CmpI->setOperand(i: 1, Val: TripCount);
13379
13380#ifndef NDEBUG
13381 assertOK();
13382#endif
13383}
13384
13385void CanonicalLoopInfo::mapIndVar(
13386 llvm::function_ref<Value *(Instruction *)> Updater) {
13387 assert(isValid() && "Requires a valid canonical loop");
13388
13389 Instruction *OldIV = getIndVar();
13390
13391 // Record all uses excluding those introduced by the updater. Uses by the
13392 // CanonicalLoopInfo itself to keep track of the number of iterations are
13393 // excluded.
13394 SmallVector<Use *> ReplacableUses;
13395 for (Use &U : OldIV->uses()) {
13396 auto *User = dyn_cast<Instruction>(Val: U.getUser());
13397 if (!User)
13398 continue;
13399 if (User->getParent() == getCond())
13400 continue;
13401 if (User->getParent() == getLatch())
13402 continue;
13403 ReplacableUses.push_back(Elt: &U);
13404 }
13405
13406 // Run the updater that may introduce new uses
13407 Value *NewIV = Updater(OldIV);
13408
13409 // Replace the old uses with the value returned by the updater.
13410 for (Use *U : ReplacableUses)
13411 U->set(NewIV);
13412
13413#ifndef NDEBUG
13414 assertOK();
13415#endif
13416}
13417
13418void CanonicalLoopInfo::assertOK() const {
13419#ifndef NDEBUG
13420 // No constraints if this object currently does not describe a loop.
13421 if (!isValid())
13422 return;
13423
13424 BasicBlock *Preheader = getPreheader();
13425 BasicBlock *Body = getBody();
13426 BasicBlock *After = getAfter();
13427
13428 // Verify standard control-flow we use for OpenMP loops.
13429 assert(Preheader);
13430 assert(isa<UncondBrInst>(Preheader->getTerminator()) &&
13431 "Preheader must terminate with unconditional branch");
13432 assert(Preheader->getSingleSuccessor() == Header &&
13433 "Preheader must jump to header");
13434
13435 assert(Header);
13436 assert(isa<UncondBrInst>(Header->getTerminator()) &&
13437 "Header must terminate with unconditional branch");
13438 assert(Header->getSingleSuccessor() == Cond &&
13439 "Header must jump to exiting block");
13440
13441 assert(Cond);
13442 assert(Cond->getSinglePredecessor() == Header &&
13443 "Exiting block only reachable from header");
13444
13445 assert(isa<CondBrInst>(Cond->getTerminator()) &&
13446 "Exiting block must terminate with conditional branch");
13447 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
13448 "Exiting block's first successor jump to the body");
13449 assert(cast<CondBrInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
13450 "Exiting block's second successor must exit the loop");
13451
13452 assert(Body);
13453 assert(Body->getSinglePredecessor() == Cond &&
13454 "Body only reachable from exiting block");
13455 assert(!isa<PHINode>(Body->front()));
13456
13457 assert(Latch);
13458 assert(isa<UncondBrInst>(Latch->getTerminator()) &&
13459 "Latch must terminate with unconditional branch");
13460 assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
13461 // TODO: To support simple redirecting of the end of the body code that has
13462 // multiple; introduce another auxiliary basic block like preheader and after.
13463 assert(Latch->getSinglePredecessor() != nullptr);
13464 assert(!isa<PHINode>(Latch->front()));
13465
13466 assert(Exit);
13467 assert(isa<UncondBrInst>(Exit->getTerminator()) &&
13468 "Exit block must terminate with unconditional branch");
13469 assert(Exit->getSingleSuccessor() == After &&
13470 "Exit block must jump to after block");
13471
13472 assert(After);
13473 assert(After->getSinglePredecessor() == Exit &&
13474 "After block only reachable from exit block");
13475 assert(After->empty() || !isa<PHINode>(After->front()));
13476
13477 Instruction *IndVar = getIndVar();
13478 assert(IndVar && "Canonical induction variable not found?");
13479 assert(isa<IntegerType>(IndVar->getType()) &&
13480 "Induction variable must be an integer");
13481 assert(cast<PHINode>(IndVar)->getParent() == Header &&
13482 "Induction variable must be a PHI in the loop header");
13483 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
13484 assert(
13485 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());
13486 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
13487
13488 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
13489 assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
13490 assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
13491 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
13492 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
13493 ->isOne());
13494
13495 Value *TripCount = getTripCount();
13496 assert(TripCount && "Loop trip count not found?");
13497 assert(IndVar->getType() == TripCount->getType() &&
13498 "Trip count and induction variable must have the same type");
13499
13500 auto *CmpI = cast<CmpInst>(&Cond->front());
13501 assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
13502 "Exit condition must be a signed less-than comparison");
13503 assert(CmpI->getOperand(0) == IndVar &&
13504 "Exit condition must compare the induction variable");
13505 assert(CmpI->getOperand(1) == TripCount &&
13506 "Exit condition must compare with the trip count");
13507#endif
13508}
13509
13510void CanonicalLoopInfo::invalidate() {
13511 Header = nullptr;
13512 Cond = nullptr;
13513 Latch = nullptr;
13514 Exit = nullptr;
13515}
13516